From 7244282fceb012006c773cf7dd9a88faad82b7de Mon Sep 17 00:00:00 2001 From: Rickylabs Date: Sat, 11 Jul 2026 14:17:18 +0200 Subject: [PATCH] docs: reconcile capability pages into pillars --- .../context-pack.md | 3 + .../docs-433-ia-reconcile--codex/drift.md | 6 + .../runs/docs-433-ia-reconcile--codex/plan.md | 30 ++ .../docs-433-ia-reconcile--codex/research.md | 9 + .../supervisor.md | 10 + .../docs-433-ia-reconcile--codex/worklog.md | 28 + docs/site/_data.ts | 2 + docs/site/_data/xref.ts | 32 +- docs/site/_includes/layouts/redirect.vto | 13 + docs/site/ai/mcp.md | 9 + docs/site/background-processing/index.md | 2 +- .../background-processing/polyglot-tasks.md | 237 ++++++++ docs/site/background-processing/workers.md | 475 ++++++++++++++++ docs/site/capabilities/ai.md | 81 +-- docs/site/capabilities/auth.md | 351 +----------- docs/site/capabilities/background-jobs.md | 475 +--------------- docs/site/capabilities/database.md | 392 +------------- docs/site/capabilities/durable-sagas.md | 508 +----------------- docs/site/capabilities/fresh-framework.md | 312 +---------- docs/site/capabilities/fresh-ui.md | 264 +-------- docs/site/capabilities/index.md | 150 +----- docs/site/capabilities/kv-queues-cron.md | 378 +------------ docs/site/capabilities/polyglot-tasks.md | 237 +------- docs/site/capabilities/runtime-config.md | 283 +--------- docs/site/capabilities/sdk.md | 228 +------- docs/site/capabilities/services.md | 452 +--------------- docs/site/capabilities/streams.md | 410 +------------- docs/site/capabilities/telemetry.md | 436 +-------------- docs/site/capabilities/triggers.md | 485 +---------------- docs/site/data-persistence/database.md | 392 ++++++++++++++ docs/site/data-persistence/index.md | 2 +- docs/site/data-persistence/kv-queues-cron.md | 378 +++++++++++++ docs/site/durable-workflows/sagas.md | 508 ++++++++++++++++++ docs/site/durable-workflows/streams.md | 410 ++++++++++++++ docs/site/durable-workflows/triggers.md | 485 +++++++++++++++++ docs/site/identity-access/auth.md | 351 ++++++++++++ docs/site/observability/index.md | 2 +- docs/site/observability/telemetry.md | 436 +++++++++++++++ .../orchestration-runtime/cli-scaffold.md | 9 + .../orchestration-runtime/runtime-config.md | 283 ++++++++++ docs/site/services-sdk/sdk.md | 228 ++++++++ docs/site/services-sdk/services.md | 452 ++++++++++++++++ docs/site/web-layer/fresh-ui.md | 264 +++++++++ 43 files changed, 5069 insertions(+), 5429 deletions(-) create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/context-pack.md create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/drift.md create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/plan.md create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/research.md create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/supervisor.md create mode 100644 .llm/runs/docs-433-ia-reconcile--codex/worklog.md create mode 100644 docs/site/_includes/layouts/redirect.vto create mode 100644 docs/site/ai/mcp.md create mode 100644 docs/site/background-processing/polyglot-tasks.md create mode 100644 docs/site/background-processing/workers.md create mode 100644 docs/site/data-persistence/database.md create mode 100644 docs/site/data-persistence/kv-queues-cron.md create mode 100644 docs/site/durable-workflows/sagas.md create mode 100644 docs/site/durable-workflows/streams.md create mode 100644 docs/site/durable-workflows/triggers.md create mode 100644 docs/site/identity-access/auth.md create mode 100644 docs/site/observability/telemetry.md create mode 100644 docs/site/orchestration-runtime/cli-scaffold.md create mode 100644 docs/site/orchestration-runtime/runtime-config.md create mode 100644 docs/site/services-sdk/sdk.md create mode 100644 docs/site/services-sdk/services.md create mode 100644 docs/site/web-layer/fresh-ui.md diff --git a/.llm/runs/docs-433-ia-reconcile--codex/context-pack.md b/.llm/runs/docs-433-ia-reconcile--codex/context-pack.md new file mode 100644 index 000000000..0325dfecc --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/context-pack.md @@ -0,0 +1,3 @@ +# Context pack + +Issue #433 S0 is implemented as a docs-only structural IA reconciliation. Full site verify and explicit redirect/nav/scope checks pass. PLAN-EVAL was owner-waived; separate supervisor slice review/IMPL-EVAL remains external to this implementation lane. diff --git a/.llm/runs/docs-433-ia-reconcile--codex/drift.md b/.llm/runs/docs-433-ia-reconcile--codex/drift.md new file mode 100644 index 000000000..f42a2a5cb --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/drift.md @@ -0,0 +1,6 @@ +# Drift + +## D1 — PLAN-EVAL waiver + +Owner explicitly waived PLAN-EVAL for this structural S0 slice. The implementation plan and Design checkpoint were recorded before changes. + diff --git a/.llm/runs/docs-433-ia-reconcile--codex/plan.md b/.llm/runs/docs-433-ia-reconcile--codex/plan.md new file mode 100644 index 000000000..3980cb863 --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/plan.md @@ -0,0 +1,30 @@ +# Plan + +## Locked decisions + +1. Move the 13 one-to-one capability pages byte-for-byte to the proposal targets. +2. Treat the two fold mappings as redirects to existing pillar indexes; do not merge or rewrite prose in S0. +3. Generate static HTML redirect documents at all 15 old leaf URLs plus the retired capabilities index URL, using a dedicated Vento layout with canonical, meta-refresh, and JavaScript replacement targets. +4. Retarget `cap:*`, the three landing cards, and hard-coded internal capability links to canonical pillar URLs while leaving tutorial nav anchors unchanged. +5. Add minimal titled stubs for CLI/scaffold and MCP and expose them in their pillar nav sections. + +## Open decisions + +- HTTP 301 at GitHub Pages hosting: safe to defer; the static host cannot emit per-page status codes. Redirect documents preserve URLs client-side and canonical metadata. + +## Slices and gates + +1. IA + redirects + nav/xrefs/stubs — `deno task verify`, built redirect inspection, grep/nav assertions; docs-site files and run artifacts only. + +## Risks + +- Stale hard-coded links: mitigate with site verify and a full `/capabilities/` scan. +- Base-path mistakes: redirect layout uses Lume's `url` filter. +- Prose churn: verify moved blobs with git rename detection/diff review. +- Lock churn: compare `deno.lock` before commit. + +## Deferred scope + +- All prose rewriting/merging, tutorial chapter anchors, framework code, and deployment authoring. +- JSR audit: N/A, docs-only and no package/plugin public surface changes. + diff --git a/.llm/runs/docs-433-ia-reconcile--codex/research.md b/.llm/runs/docs-433-ia-reconcile--codex/research.md new file mode 100644 index 000000000..1f28de01d --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/research.md @@ -0,0 +1,9 @@ +# Research + +- Re-baselined issue #433 and proposal §2.2 against the current clean branch. +- `docs/site/capabilities/` contains 15 leaf pages plus `index.md`; proposal maps all leaves into the nine pillar folders. +- `_data/xref.ts` owns all `cap:*` destinations. Three pillar landing cards currently route into `capabilities/`: Background Processing, Data & Persistence, and Observability. +- No redirect plugin or redirect-frontmatter convention exists in `_config.ts`; static redirect documents are therefore required for GitHub Pages compatibility. +- The nine pillar nav sections and order are Web Layer, Services & SDK, Background Processing, Durable Workflows, AI & Agents, Data & Persistence, Identity & Access, Orchestration & Runtime, Observability. +- The two fold mappings target existing canonical pillar indexes. Structural-only scope forbids merging prose, so the old URLs redirect to those indexes and the existing indexes remain canonical. + diff --git a/.llm/runs/docs-433-ia-reconcile--codex/supervisor.md b/.llm/runs/docs-433-ia-reconcile--codex/supervisor.md new file mode 100644 index 000000000..e972294de --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/supervisor.md @@ -0,0 +1,10 @@ +# Supervisor + +- Run: `docs-433-ia-reconcile--codex` +- Supervisor: beta-7 Claude session `df71d36c` +- Implementation lane: WSL Codex, `/home/codex/repos/ns-wt-433` +- Branch: `docs/433-ia-reconcile-capabilities` +- Scope: docs-only, `SCOPE-docs` +- PLAN-EVAL: owner-waived (D1) +- IMPL-EVAL / slice review: separate supervisor session required + diff --git a/.llm/runs/docs-433-ia-reconcile--codex/worklog.md b/.llm/runs/docs-433-ia-reconcile--codex/worklog.md new file mode 100644 index 000000000..620fcc5b9 --- /dev/null +++ b/.llm/runs/docs-433-ia-reconcile--codex/worklog.md @@ -0,0 +1,28 @@ +# Worklog + +## Design + +- Public surface: canonical pillar URLs, legacy redirect URLs, sidebar navigation, and stable `cap:*` xrefs. +- Domain vocabulary: 15 legacy capability slugs mapped to canonical pillar paths; nine ordered pillars; two navigable stubs. +- Ports: Lume page rendering and the existing `url` filter/base-path behavior. +- Constants: the mapping is locked by proposal §2.2 and represented by redirect frontmatter + xref entries. +- Commit slices: one structural docs slice, proven by `deno task verify`, redirect inspection, grep, nav/stub assertions, and scope/lock checks. +- Deferred: prose rewrites/merges, tutorial nav anchors, framework/package/plugin changes. +- Contributor path: add concepts inside the owning pillar, register stable xrefs in `_data/xref.ts`, and add nav leaves in `_data.ts`. + +## Plan gate + +- PLAN-EVAL owner-waived; recorded as drift D1 before implementation. + +## Evidence + +- `deno task verify` in `docs/site`: exit 0; 23,016 internal links across 162 pages resolve; 27 caveat markers across 22 pages resolve. +- Built redirect inspection: all 15 `/capabilities//` documents exist and contain their mapped `/netscript//` canonical/refresh target (`redirects=15 PASS`). +- `rg 'capabilities/' docs/site/_data.ts docs/site/_data/xref.ts`: no matches. +- Nine pillar headings remain in the original order; exact `_data.ts` heading lines verified. +- Stub nav: `/ai/mcp/` and `/orchestration-runtime/cli-scaffold/` both present in `_data.ts`; built pages exist. +- Scope/lock check: no path under `packages/` or `plugins/`, and no `deno.lock`, appears in the diff (`scope-lock=PASS`). + +## Reconcile + +- Issue #433 remains open with the expected docs/refactor taxonomy and beta.7 milestone. No PR was opened per owner instruction. diff --git a/docs/site/_data.ts b/docs/site/_data.ts index 07e1f2cb8..6987426d5 100644 --- a/docs/site/_data.ts +++ b/docs/site/_data.ts @@ -146,6 +146,7 @@ export const navSections: NavSection[] = [ label: "AI & Agents", items: [ { href: "/ai/", label: "Overview & Concepts", icon: "O" }, + { href: "/ai/mcp/", label: "Guide: MCP", icon: "G" }, { href: "/tutorials/chat/", label: "Quickstart: AI chat", icon: "Q" }, { href: "/how-to/build-a-durable-chat/", label: "How-To: build a durable chat", icon: "H" }, { href: "/ai/durable-chat/", label: "Guide: durable chat", icon: "G" }, @@ -187,6 +188,7 @@ export const navSections: NavSection[] = [ label: "Orchestration & Runtime", items: [ { href: "/orchestration-runtime/", label: "Overview & Concepts", icon: "O" }, + { href: "/orchestration-runtime/cli-scaffold/", label: "Guide: CLI & scaffold", icon: "G" }, { href: "/quickstart/", label: "Quickstart: run the workspace", icon: "Q" }, { href: "/how-to/deno-lsp-code-intelligence/", label: "How-To: Deno LSP intelligence", icon: "H" }, { href: "/how-to/deploy-local-aspire/", label: "How-To: deploy locally with Aspire", icon: "H" }, diff --git a/docs/site/_data/xref.ts b/docs/site/_data/xref.ts index 08999de89..b05e0fc58 100644 --- a/docs/site/_data/xref.ts +++ b/docs/site/_data/xref.ts @@ -6,7 +6,7 @@ * and FAILS on an unknown key, so the build doubles as a link checker. * * Locked key namespaces (OD3): - * cap: capability hub (cap:services -> /capabilities/services/) + * cap: capability hub (cap:services -> /services-sdk/services/) * howto: how-to recipe (howto:add-a-plugin) * tut: tutorial chapter/track (tut:first-workspace) * explain: explanation essay (explain:architecture) @@ -77,22 +77,22 @@ function refEntries(): Record { export const xref: Record = { // ─── Capability hubs (cap:) ──────────────────────────────────────────────── - "cap:services": { href: "/capabilities/services/", label: "Services & contracts" }, - "cap:background-jobs": { href: "/capabilities/background-jobs/", label: "Background jobs" }, - "cap:durable-sagas": { href: "/capabilities/durable-sagas/", label: "Durable sagas" }, - "cap:triggers": { href: "/capabilities/triggers/", label: "Triggers & ingress" }, - "cap:streams": { href: "/capabilities/streams/", label: "Durable streams" }, - "cap:database": { href: "/capabilities/database/", label: "Database & Prisma" }, - "cap:kv-queues-cron": { href: "/capabilities/kv-queues-cron/", label: "KV, queues & cron" }, - "cap:telemetry": { href: "/capabilities/telemetry/", label: "Telemetry & logging" }, - "cap:auth": { href: "/capabilities/auth/", label: "Authentication" }, - "cap:fresh-ui": { href: "/capabilities/fresh-ui/", label: "Fresh UI & design" }, - "cap:index": { href: "/capabilities/", label: "Capabilities" }, + "cap:services": { href: "/services-sdk/services/", label: "Services & contracts" }, + "cap:background-jobs": { href: "/background-processing/workers/", label: "Background jobs" }, + "cap:durable-sagas": { href: "/durable-workflows/sagas/", label: "Durable sagas" }, + "cap:triggers": { href: "/durable-workflows/triggers/", label: "Triggers & ingress" }, + "cap:streams": { href: "/durable-workflows/streams/", label: "Durable streams" }, + "cap:database": { href: "/data-persistence/database/", label: "Database & Prisma" }, + "cap:kv-queues-cron": { href: "/data-persistence/kv-queues-cron/", label: "KV, queues & cron" }, + "cap:telemetry": { href: "/observability/telemetry/", label: "Telemetry & logging" }, + "cap:auth": { href: "/identity-access/auth/", label: "Authentication" }, + "cap:fresh-ui": { href: "/web-layer/fresh-ui/", label: "Fresh UI & design" }, + "cap:index": { href: "/", label: "NetScript" }, // v3 NEW capability hubs - "cap:fresh-framework": { href: "/capabilities/fresh-framework/", label: "Fresh meta-framework" }, - "cap:sdk": { href: "/capabilities/sdk/", label: "Typed SDK & client" }, - "cap:polyglot-tasks": { href: "/capabilities/polyglot-tasks/", label: "Polyglot tasks" }, - "cap:runtime-config": { href: "/capabilities/runtime-config/", label: "Runtime configuration" }, + "cap:fresh-framework": { href: "/web-layer/", label: "Fresh meta-framework" }, + "cap:sdk": { href: "/services-sdk/sdk/", label: "Typed SDK & client" }, + "cap:polyglot-tasks": { href: "/background-processing/polyglot-tasks/", label: "Polyglot tasks" }, + "cap:runtime-config": { href: "/orchestration-runtime/runtime-config/", label: "Runtime configuration" }, // ─── How-to recipes (howto:) ─────────────────────────────────────────────── "howto:index": { href: "/how-to/", label: "How-to guides" }, diff --git a/docs/site/_includes/layouts/redirect.vto b/docs/site/_includes/layouts/redirect.vto new file mode 100644 index 000000000..525ec096c --- /dev/null +++ b/docs/site/_includes/layouts/redirect.vto @@ -0,0 +1,13 @@ + + + + + + + Moved + + +

This page moved to {{ redirectTo }}.

+ + + diff --git a/docs/site/ai/mcp.md b/docs/site/ai/mcp.md new file mode 100644 index 000000000..d09775f25 --- /dev/null +++ b/docs/site/ai/mcp.md @@ -0,0 +1,9 @@ +--- +layout: layouts/base.vto +title: MCP +templateEngine: [vento, md] +--- + +# MCP + +_Story page forthcoming._ diff --git a/docs/site/background-processing/index.md b/docs/site/background-processing/index.md index c8887319a..e2451a196 100644 --- a/docs/site/background-processing/index.md +++ b/docs/site/background-processing/index.md @@ -11,7 +11,7 @@ per-task permissions. Start here when work should run outside the request path b durable saga state model. {{ comp.cardsGrid({ columns: 3, cards: [ - { eyebrow: "Overview & Concepts", title: "Workers and queues", body: "Worker tasks consume queued work with provider and runtime choices kept inside the leaf.", href: "/capabilities/background-jobs/", icon: "O" }, + { eyebrow: "Overview & Concepts", title: "Workers and queues", body: "Worker tasks consume queued work with provider and runtime choices kept inside the leaf.", href: "/background-processing/workers/", icon: "O" }, { eyebrow: "Quickstart", title: "Polyglot transform", body: "Run a task step from the ERP Sync tutorial.", href: "/tutorials/erp-sync/03-polyglot-transform/", icon: "Q" }, { eyebrow: "How-To", title: "Queue, KV, and cron", body: "Create the queue and scheduler loop used by background work.", href: "/how-to/queue-kv-cron/", icon: "H" }, { eyebrow: "How-To", title: "Choose a queue provider", body: "Select the provider that fits your local and deployed runtime.", href: "/how-to/choose-a-queue-provider/", icon: "H" }, diff --git a/docs/site/background-processing/polyglot-tasks.md b/docs/site/background-processing/polyglot-tasks.md new file mode 100644 index 000000000..a21bf23c9 --- /dev/null +++ b/docs/site/background-processing/polyglot-tasks.md @@ -0,0 +1,237 @@ +--- +layout: layouts/base.vto +title: Polyglot tasks +templateEngine: [vento, md] +prev: { label: "Typed SDK & client", href: "/services-sdk/sdk/" } +next: { label: "Runtime configuration", href: "/orchestration-runtime/runtime-config/" } +--- + +# Polyglot tasks + +A **polyglot task** runs non-TypeScript work — a Python script, a .NET program, a +shell or PowerShell script, or any executable — as a **managed subprocess** spawned by the +worker runtime. NetScript hands the task its input as command-line arguments and +environment variables, captures every line of `stdout`/`stderr`, parses a final JSON line +into a structured result, and normalizes the exit code into a `TaskResult`. It is the +escape hatch for the moments your platform is otherwise all-TypeScript: an ML model in +Python, a legacy .NET DLL, a system `pwsh` script. {{ comp.badge({ status: "alpha" }) }} + +{{ comp.diagram({ src: "/assets/diagrams/polyglot-task-execution.svg", alt: "The worker runtime resolves a TaskDefinition to a runtime adapter, which builds an argv and spawns a python/node/dotnet subprocess; input flows in as args and env, the subprocess streams stdout/stderr back, and the last JSON line of stdout becomes the structured result returned to the queue and database.", caption: "A task is dispatched to a runtime adapter that spawns a subprocess. Input arrives as argv + env; the last JSON line of stdout is parsed into the result; the exit code, captured logs, and duration become a TaskResult." }) }} + +## What it is + +A **job** runs in-process TypeScript on a worker; a **task** runs a *subprocess* in +another runtime. Tasks share the worker plugin's queue, retry, and telemetry machinery +with [background jobs](/background-processing/workers/) — the difference is purely the +execution surface. The `MultiRuntimeTaskExecutor` keeps a map of **runtime adapters** +(one per `TaskType`) and dispatches a `TaskDefinition` to the adapter that supports its +`type`. Each adapter builds an `argv` for its runtime (e.g. `python3 -u script.py …`, +`pwsh -File script.ps1 …`) and runs it through a Dax-backed process runner that streams +output and times the process out. This page covers that subprocess seam; for in-process +TS handlers, runtime modes, and the queue lifecycle, start at +[background jobs](/background-processing/workers/). + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for a polyglot task when the work must run in another runtime — a +Python data/ML script, a .NET tool, an existing shell or PowerShell automation, or a +prebuilt binary — and you want it queued, retried, traced, and result-captured like any +other unit of work. If the work is plain TypeScript, use an in-process +background job instead — it avoids a process +spawn entirely. +{{ /comp }} + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Workers track", + body: "Tutorial Track C builds the worker plugin end to end; lesson 03 adds a polyglot task with explicit permissions.", + href: "/tutorials/erp-sync/", + icon: "→" + }, + { + title: "Do — Run a polyglot task", + body: "Task recipe: define a python (or shell) task, wire its entrypoint and permissions, and execute it through the runtime.", + href: "/how-to/run-a-polyglot-task/", + icon: "◆" + }, + { + title: "Understand — Background jobs", + body: "The queue, worker-runtime modes, retry, and shutdown machinery that polyglot tasks share with in-process jobs.", + href: "/background-processing/workers/", + icon: "◎" + } +] }) }} + +## Minimal example + +A task is authored with the `defineTask` typestate builder, then run through the default +multi-runtime executor. The builder's default `runtime` is `'deno'`; call `.runtime(type)` +to target another language. Input reaches the script as **argv** (`.args(...)`) and +**environment variables** (`.env({...})`); the script returns a result by writing a single +JSON object as the **last line of `stdout`**. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Define + run a python task", + lang: "ts", + code: "// workers/tasks/score-batch.ts\nimport { defineTask } from '@netscript/plugin-workers-core/builders';\nimport { createDefaultTaskExecutor } from '@netscript/plugin-workers-core/executor';\n\n// Build a task definition: python runtime, script entrypoint, explicit inputs.\nconst scoreBatch = defineTask('score-batch')\n .runtime('python')\n .entrypoint('./scripts/score.py')\n .env({ MODEL_PATH: './models/scorer.pkl' })\n .args('--threshold', '0.8')\n .timeout(120_000) // ms; defaults to 300_000\n .build();\n\n// The executor resolves the python adapter and spawns: python3 -u ./scripts/score.py --threshold 0.8\nconst executor = createDefaultTaskExecutor();\nconst result = await executor.execute(scoreBatch, {\n onStdout: (line) => console.log('[score]', line),\n});\n\nif (result.success) {\n // result.result is the parsed JSON object from the LAST stdout line, or null.\n console.log('scored', result.result);\n} else {\n console.error('task failed', result.exitCode, result.error);\n}" + }, + { + label: "score.py (the subprocess)", + lang: "python", + code: "# scripts/score.py\nimport json, os, sys\n\n# Input arrives as argv + env (NOT stdin).\nthreshold = float(sys.argv[sys.argv.index('--threshold') + 1])\nmodel_path = os.environ['MODEL_PATH']\n\n# ... do the work ...\nscored = {'kept': 42, 'dropped': 3, 'threshold': threshold}\n\n# Any prior prints become captured logs. The result is the LAST line of stdout,\n# and must be a single JSON object (not an array) to populate result.result.\nprint('scoring complete', file=sys.stderr)\nprint(json.dumps(scored))" + } +] }) }} + +{{ comp callout { type: "important", title: "How input and output cross the process boundary" } }} +Input is passed as command-line arguments (args) and +environment variables (env) — there is no JSON-over-stdin +channel. The runtime merges Deno.env, the task's env, and the +call's options.env, and injects TRACEPARENT, +TRACESTATE, and CORRELATION_ID for trace propagation. Output is +the reverse: the subprocess returns a structured value by printing one JSON object +as the final line of stdout; the runtime parses that line into +result.result (a non-object or non-final-line value yields null). +All other lines are captured into stdout/stderr and streamed to +the onStdout/onStderr/onLog callbacks. +{{ /comp }} + +## Key types first — TaskResult + +Every task — whatever its runtime — resolves to one `TaskResult`. This is the shape your +calling code reads; check it before writing per-runtime branches. + +{{ comp.apiTable({ + caption: "TaskResult — returned by executor.execute()", + rows: [ + { name: "taskId", type: "string", desc: "The TaskDefinition.id this result belongs to." }, + { name: "status", type: "string", desc: "Lifecycle status: 'completed' | 'failed' | 'timeout' | 'cancelled' (running/pending used in-flight)." }, + { name: "success", type: "boolean", desc: "True only when status is 'completed' (exit code 0). Branch on this." }, + { name: "exitCode", type: "number", desc: "Subprocess exit code; -1 when the process never started (spawn error, cancelled, timeout)." }, + { name: "stdout", type: "string", desc: "All captured stdout lines, joined by newlines." }, + { name: "stderr", type: "string", desc: "All captured stderr lines (and the error message on a spawn failure)." }, + { name: "result", type: "Record | null", desc: "The JSON object parsed from the LAST line of stdout, or null if that line is not a JSON object." }, + { name: "error", type: "string | null", desc: "Human-readable failure message (includes exit code and first stderr line); null on success." }, + { name: "duration", type: "number", desc: "Wall-clock execution time in milliseconds." }, + { name: "startedAt / completedAt", type: "string", desc: "ISO-8601 timestamps bracketing the run." }, + { name: "attempt", type: "number", desc: "Attempt index for this execution (0 from the executor itself)." } + ] +}) }} + +## TaskDefinition & runtimes + +`defineTask(id)` returns a typestate builder; `.build()` is only callable once an +`.entrypoint(path)` (subprocess) or `.handler(fn)` (in-process) is set. The resulting +`TaskDefinition` carries the runtime `type`, entrypoint, args, env, timeout, and +permissions. The seven supported runtimes (`TaskType`) and how each is launched: + +{{ comp.apiTable({ + caption: "TaskType — built-in runtime adapters", + rows: [ + { name: "deno", type: "default", desc: "Spawns `deno run` with permission flags built from .permissions(...). The only runtime that is sandboxed by per-task Deno permissions." }, + { name: "python", type: "subprocess", desc: "Spawns `python3 -u ` (or `py` on Windows); honors metadata.pythonConfig.venvPath / pythonPath and NETSCRIPT_PYTHON_PATH." }, + { name: "dotnet", type: "subprocess", desc: "Runs a .cs via `dotnet run`, a project via `dotnet run --project`, or a built executable directly; metadata.dotnetConfig.runtimeArgs/useDotnetRun." }, + { name: "shell", type: "subprocess", desc: "Runs the entrypoint under bash (metadata.shellConfig.shell / loginShell); resolves Git Bash util paths on Windows." }, + { name: "powershell", type: "subprocess", desc: "Spawns `powershell` (Windows) or `pwsh` with -NoProfile -NonInteractive -ExecutionPolicy Bypass -File." }, + { name: "cmd", type: "subprocess", desc: "Spawns `cmd.exe /c ` (Windows)." }, + { name: "executable", type: "subprocess", desc: "Runs the entrypoint directly as a prebuilt binary with the task's args." } + ] +}) }} + +The per-task permission set passed to `.permissions(...)` mirrors the Deno permission +model. **It is enforced only for the `deno` runtime** — those keys are translated into +`--allow-*` flags on the `deno run` command line. For `python`, `shell`, and the other +external runtimes there is no Deno sandbox; the subprocess inherits the OS-level access of +the worker process. + +{{ comp.apiTable({ + caption: "permissions() options — applied to the deno runtime", + rows: [ + { name: "net", type: "boolean | string[]", desc: "--allow-net (true) or --allow-net=HOSTS (array). Network access." }, + { name: "read", type: "boolean | string[]", desc: "--allow-read or --allow-read=PATHS. Filesystem read." }, + { name: "write", type: "boolean | string[]", desc: "--allow-write or --allow-write=PATHS. Filesystem write." }, + { name: "env", type: "boolean | string[]", desc: "--allow-env or --allow-env=VARS. Environment-variable access." }, + { name: "run", type: "boolean | string[]", desc: "--allow-run or --allow-run=CMDS. Subprocess spawn." }, + { name: "ffi", type: "boolean", desc: "--allow-ffi. FFI access to native libraries." }, + { name: "import", type: "string[]", desc: "--allow-import=SPECIFIERS. Allowed dynamic-import sources." } + ] +}) }} + +{{ comp callout { type: "note", title: "Omitting permissions widens the Deno sandbox" } }} +For a deno task, calling .build() without +.permissions(...) produces an --allow-all command line. Always +pass an explicit, least-privilege permission set for untrusted or third-party Deno task +code. Non-Deno runtimes ignore these keys entirely — gate those at the OS level instead. +{{ /comp }} + +## Production notes + +{{ comp callout { type: "warning", title: "The result contract is the LAST stdout line" } }} +A subprocess returns structured data only by printing a single JSON +object as its final stdout line. A trailing log line, a pretty-print, +a JSON array, or a stray newline after the payload all cause +result.result to be null even though the task +succeeded. Print diagnostics to stderr, and emit the JSON result +last with no trailing output. Buffering can also reorder this — for Python the runtime runs +python3 -u (unbuffered) for exactly this reason; flush your own output in other +runtimes. +{{ /comp }} + +{{ comp callout { type: "important", title: "Runtimes must exist on the host; exit codes are diagnostic" } }} +Each non-Deno runtime requires its toolchain on the worker host: a Python +interpreter (or a configured venv), the .NET SDK, pwsh/powershell, +or bash. A missing interpreter surfaces as a failed task — exit code +127 is reported as command not found and 126 as +command not executable. cmd and powershell.exe are +Windows-only; shell uses bash and resolves Git Bash utility paths on Windows. +A task that exceeds its timeout resolves with status timeout and +exit code -1; passing an already-aborted signal yields +cancelled. +{{ /comp }} + +{{ comp callout { type: "note", title: "Tasks run external code — treat them as a trust boundary" } }} +Polyglot tasks spawn arbitrary processes with the worker's OS privileges. Pin entrypoints +to known scripts, prefer a pinned interpreter / venv (pythonConfig.venvPath) +over $PATH discovery, and avoid interpolating untrusted input into +args or the entrypoint path. Trace context +(TRACEPARENT/CORRELATION_ID) is injected into the subprocess +environment so cross-runtime spans stitch together in telemetry. +{{ /comp }} + +## Reference → + +The full generated API — `defineTask`, `MultiRuntimeTaskExecutor`, +`createDefaultTaskExecutor`, every runtime adapter, and the executor option types — lives +in the workers reference. + +{{ comp.xref({ key: "ref:workers" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — Workers reference", + body: "defineTask, the multi-runtime executor, runtime adapters, TaskDefinition / TaskResult / TaskExecutionOptions, and permission types.", + href: "/reference/workers/", + icon: "≡" + }, + { + title: "Do — Run a polyglot task", + body: "Step-by-step recipe for a python and a shell task with explicit permissions and result capture.", + href: "/how-to/run-a-polyglot-task/", + icon: "◆" + }, + { + title: "Understand — Background jobs", + body: "The queue, worker-runtime modes, retry, and graceful-shutdown drain shared with in-process jobs.", + href: "/background-processing/workers/", + icon: "◎" + }, + { + title: "Next — Runtime configuration", + body: "How worker, service, and adapter settings resolve from schema defaults, config files, env vars, and overrides.", + href: "/orchestration-runtime/runtime-config/", + icon: "→" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Typed SDK & client", href: "/services-sdk/sdk/" }, next: { label: "Runtime configuration", href: "/orchestration-runtime/runtime-config/" } }) }} diff --git a/docs/site/background-processing/workers.md b/docs/site/background-processing/workers.md new file mode 100644 index 000000000..bc9b95f89 --- /dev/null +++ b/docs/site/background-processing/workers.md @@ -0,0 +1,475 @@ +--- +layout: layouts/base.vto +title: Background jobs +templateEngine: [vento, md] +prev: { label: "Services & contracts", href: "/services-sdk/services/" } +next: { label: "Durable sagas", href: "/durable-workflows/sagas/" } +--- + +# Background jobs + +A NetScript **background job** is a durable, KV-backed TypeScript handler that runs in +its own thread-isolated worker, separate from your request-serving services. You author a +job as one `defineJobHandler(...)` callable, give it an `id`, and the runtime takes care of +registration, dispatch, retry, execution tracking, scheduling, and an HTTP API to enqueue +and inspect runs. It is the unit you reach for whenever work should happen *after* a request +returns — charging a payment, sending a welcome email, processing an upload — without +blocking the caller. + +{{ comp.diagram({ src: "/assets/diagrams/queue-worker-scheduler.svg", alt: "An enqueue call (from a trigger, an HTTP POST to the workers API, or the scheduler) places a job on the durable queue; the worker runtime pulls it and runs the handler in one of three runner modes — in-process, web-worker (one V8 isolate per worker), or subprocess — then writes a JobResult to the KV-backed execution store, which streams updates back over SSE.", caption: "Enqueue → durable queue → worker runtime (in-process / web-worker / subprocess) → result store. The scheduler fires cron-defined jobs onto the same queue; graceful shutdown drains in-flight runs before the runner stops." }) }} + +## What it is + +A job is plain TypeScript that runs **in the worker process, not the request process**. +The headline surface — `defineJobHandler`, `createSuccessResult`, and `createFailureResult` +from `@netscript/plugin-workers-core` — lets you write a typed handler over a `ctx`, do the +work, and return a structured `JobResult`. Job dispatch and execution are instrumented with +real OpenTelemetry spans that show up in the [Aspire dashboard](/explanation/aspire/) +automatically, so a queued run is observable end to end without wiring. The runner mode +(how the handler is isolated) is a **user-tunable** — see the runtime-mode table below — and +the same queue and scheduler also drive [polyglot tasks](/background-processing/polyglot-tasks/) when +the work is owned by another runtime. The why-behind-the-choreography lives in +[Durability model](/explanation/durability-model/). + +{{ comp callout { type: "tip", title: "Jobs vs. tasks vs. sagas vs. services" } }} +Reach for a background job when the work is fire-and-forget or +deferrable and written in TypeScript: it should survive the request that started +it, run on its own schedule or trigger, and be retried and observed independently. If the +work must run in another runtime (Python, .NET, a shell script, a binary), use a +polyglot task — it shares this same queue, +retry, and telemetry machinery but spawns a subprocess. If the work coordinates +several steps across time (waiting for messages, compensating on failure), model it +as a durable saga; jobs and sagas compose. For a +synchronous request/response API, author a +service instead. +{{ /comp }} + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — ERP Sync, lesson 02", + body: "The tutorial rung: add the worker plugin to the running app, author an import job, and trigger it over :8091 as part of the ERP-sync narrative.", + href: "/tutorials/erp-sync/02-import-job/", + icon: "→" + }, + { + title: "Do — Tune the worker runtime", + body: "Recipe: pick the in-process / web-worker / subprocess runner, set WORKERS_CONCURRENCY, and choose a queue provider for your deployment.", + href: "/how-to/tune-worker-runtime/", + icon: "◆" + }, + { + title: "Understand — Polyglot tasks", + body: "Run non-TypeScript work (Python, .NET, shell) on the same durable queue and scheduler as a managed subprocess.", + href: "/background-processing/polyglot-tasks/", + icon: "◎" + } +] }) }} + +## Minimal example + +Add the workers plugin to a published workspace with the public package install flow: + +```bash +netscript plugin install @netscript/plugin-workers +``` + +For local-source contributor work inside this monorepo, use the maintainer binary when you need +first-party samples: + +```bash +deno run -A packages/cli/bin/netscript-dev.ts plugin install worker --name workers --samples +``` + +That local path lands real, compiling modules you can read and trigger immediately — including +`plugins/workers/jobs/health-check.ts` (a job handler) and +`plugins/workers/tasks/validate-payload.ts` (a polyglot task). The plugin's API service comes up +on **port 8091**. + +A job handler is an async callable over a `ctx` object that returns a `JobResult`. Parse the +payload with a Zod schema, do the work, and return `createSuccessResult(...)` or +`createFailureResult(...)`. The job's identity is attached with `Object.assign(handler, { id })`. + +```ts +// plugins/workers/jobs/process-payment.ts +import { + createFailureResult, + createSuccessResult, + defineJobHandler, +} from '@netscript/plugin-workers-core'; +import { z } from 'zod'; + +// Payload contract — parse ctx.payload before doing any work. +const ProcessPaymentPayloadSchema = z.object({ + orderId: z.string().min(1), + amountCents: z.number().int().positive(), +}); + +const handler = defineJobHandler(async (ctx) => { + const { orderId, amountCents } = ProcessPaymentPayloadSchema.parse(ctx.payload ?? {}); + + // Optional progress callback — the runtime wires reportProgress on real runs. + ctx.reportProgress?.(10, 'charging customer'); + + const charge = await chargeCustomer(orderId, amountCents); + if (!charge.ok) { + // A failure result is recorded and feeds the retry policy. + return createFailureResult(`charge declined: ${charge.reason}`); + } + + ctx.reportProgress?.(100, 'charged'); + // The success payload (data) is persisted on the execution record. + return createSuccessResult({ orderId, chargeId: charge.id, amountCents }); +}); + +// The id is how the runtime registers, lists, and triggers the job. +export default Object.assign(handler, { id: 'process-payment' }); +``` + +Once the workers API is up (Aspire first — see Production notes), enqueue a run by `id`: + +```bash +# Enqueue the process-payment job (workers API on port 8091). +curl -X POST http://localhost:8091/api/v1/workers/jobs/process-payment/trigger \ + -H 'content-type: application/json' \ + -d '{"orderId":"o_42","amountCents":4999}' + +# Watch it land in the KV-backed execution history. +curl http://localhost:8091/api/v1/workers/executions?limit=10 +``` + +## Key types first — `JobHandlerContext` & `JobResult` + +A handler is `(ctx: JobHandlerContext) => JobResult | Promise<…>`. These +two shapes are the contract you write against; read them before the option tables. + +{{ comp.apiTable({ + caption: "JobHandlerContext — the ctx passed to defineJobHandler", + rows: [ + { name: "id", type: "string", desc: "The execution id for this run. Stable per dispatched run." }, + { name: "job", type: "{ id: string } | undefined", desc: "The job definition reference (its id), when available." }, + { name: "payload", type: "TPayload", desc: "The enqueued payload. Parse it with your Zod schema before use." }, + { name: "correlationId", type: "string | undefined", desc: "Correlation id propagated across the dispatch for tracing." }, + { name: "traceparent", type: "string | undefined", desc: "W3C traceparent of the dispatching span; child spans nest under it." }, + { name: "tracestate", type: "string | undefined", desc: "W3C tracestate accompanying the traceparent." }, + { name: "reportProgress", type: "(percent, message?) => void | Promise", desc: "Optional progress callback wired by the runtime on real runs; emits job.progress events." } + ] +}) }} + +{{ comp.apiTable({ + caption: "JobResult — return createSuccessResult() or createFailureResult()", + rows: [ + { name: "success", type: "true | false", desc: "Discriminant. createSuccessResult sets true; createFailureResult sets false. Branch on this." }, + { name: "data", type: "TResult | undefined", desc: "The result payload, persisted on the execution record (present on success, optional on failure)." }, + { name: "error", type: "string", desc: "Failure message — required when success is false (the first arg to createFailureResult)." } + ] +}) }} + +## Worker runtime modes (`WORKER_RUNTIMES`) + +How a handler is isolated from the API process is a tunable: `WORKER_RUNTIMES` enumerates the +three runner modes the worker runtime supports. The scaffold default is **web-worker**, where +each worker is its own V8 isolate sized by the `WORKERS_CONCURRENCY` env var. Pick the mode +that matches your isolation, memory, and parallelism needs. + +{{ comp.apiTable({ + caption: "WORKER_RUNTIMES — runner isolation modes (WorkerRuntime type)", + rows: [ + { name: "in-process", type: "WorkerRuntime", desc: "Runs the handler in the same process via the in-process runner (registry-first). Lowest overhead, no isolation — best for tests, compiled binaries, and single-tenant local composition." }, + { name: "web-worker", type: "WorkerRuntime", desc: "Runs each worker in its own Web Worker / V8 isolate (~20-40 MB each). The scaffold default; WORKERS_CONCURRENCY sets the process pool size for parallel job execution. Keep it low to bound memory." }, + { name: "subprocess", type: "WorkerRuntime", desc: "Runs the handler in a spawned subprocess. Strongest process isolation; only Deno tasks get permission sandboxing through .permissions(). Python, .NET, shell, PowerShell, and cmd inherit the worker process's OS permissions." } + ] +}) }} + +{{ comp.apiTable({ + caption: "Deployment & scaling knobs (workers config)", + rows: [ + { name: "WORKERS_CONCURRENCY", type: "env (number)", desc: "Runtime worker process pool size. The entrypoint reads this plural variable; current Aspire metadata also emits WORKER_CONCURRENCY, but the runtime does not consume it." }, + { name: "concurrency", type: "number", desc: "Per-topic max concurrent workers (WorkersConfigData.concurrency / per-group scaling)." }, + { name: "mode", type: "'combined' | 'distributed'", desc: "Per-topic deployment mode: one combined runner vs. distributed runners. Defaults to 'combined'." }, + { name: "queueProvider", type: "'auto' | 'deno-kv' | 'redis' | 'postgres' | 'amqp'", desc: "Queue backend. 'auto' resolves a provider; see Choose a queue provider." }, + { name: "jobsDir / tasksDir", type: "string", desc: "Directories scanned for default-exported job and task modules." } + ] +}) }} + +{{ comp callout { type: "note", title: "Tune it without touching code" } }} +The runner mode and pool size are deployment settings, not handler concerns — the same +process-payment handler runs unchanged under any +WORKER_RUNTIMES mode. Start on the +web-worker default with a small WORKERS_CONCURRENCY, move to +subprocess when you need hard isolation, and drop to in-process for +tests and compiled single-binary deployments. Resolution precedence (schema default → config +file → env → override) is covered in +runtime configuration. +{{ /comp }} + +## Enqueue from a trigger + +The HTTP `…/trigger` endpoint is one way in; the other is **declaratively, from a trigger +handler**, which returns an `enqueueJob(...)` action. `enqueueJob(job, options)` comes from +`@netscript/plugin-triggers-core` and binds an imported job definition to a payload, so an +inbound webhook or a scheduled trigger drops work onto this same runtime. + +```ts +// plugins/triggers/triggers/payment-webhook.ts +import { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core'; +// Import the job definition you want to dispatch (its default export carries the id). +import processPayment from '../../workers/jobs/process-payment.ts'; + +const handler = defineWebhook( + // The handler returns trigger actions; enqueueJob is the most common one. + (event) => [ + enqueueJob(processPayment, { + payload: { orderId: event.payload.orderId, amountCents: event.payload.amountCents }, + idempotencyKey: event.payload.orderId, // optional: collapse duplicate deliveries + }), + ], + { id: 'payment-webhook', path: '/webhooks/payment', verifier: 'hmac-sha256', secretEnv: 'WEBHOOK_SECRET' }, +); + +export default handler; +``` + +{{ comp.apiTable({ + caption: "enqueueJob(job, options) — EnqueueJobOptions (from @netscript/plugin-triggers-core)", + rows: [ + { name: "payload", type: "TPayload", desc: "The payload handed to the job's ctx.payload. Parse it with the job's Zod schema." }, + { name: "idempotencyKey", type: "string", desc: "Collapses duplicate deliveries — at-most-once effect per key." }, + { name: "concurrencyKey", type: "string", desc: "Serializes runs that share a key (e.g. per-order) so they do not overlap." }, + { name: "priority", type: "number", desc: "Dispatch priority for this enqueue action." } + ] +}) }} + +## Trigger a job from a typed client + +The `…/trigger` curl above is the OpenAPI/REST way in. Because the workers API is a **plugin +service**, the same trigger is reachable with a generated typed client over the RPC route +`/api/rpc/*` — no OpenAPI-only fallback and no transport `404`. A first-party plugin API mounts +its router under a named segment, so the client takes a `routerName` alongside `serviceName`; +Aspire injects the URL under `services__workers-api__http__0`. + +```ts +// Reach the workers plugin API with a generated typed client. +import { createServiceClient } from '@netscript/sdk/client'; +import { workersContract } from '@netscript/plugin-workers/contracts'; + +const workers = createServiceClient({ + contract: workersContract, + serviceName: 'workers-api', + routerName: 'workers', +}); + +// triggerJob is served over /api/rpc/* and returns { jobId, triggered }. +// Over REST the {id} path segment carries the target job; over RPC (no path +// segment) it travels in this input object. Either way it resolves to +// input.id — with no id at all the call fails with a 422 VALIDATION_ERROR. +const { jobId, triggered } = await workers.triggerJob({ + id: 'process-payment', + payload: { orderId: 'o_42', amountCents: 4999 }, +}); + +// triggerTask is the polyglot-task twin, returning { taskId, triggered }. +const task = await workers.triggerTask({ id: 'validate-payload', payload: {} }); +``` + +{{ comp.apiTable({ + caption: "Trigger procedures — served over /api/rpc/* (typed client) and /api/v1/... (OpenAPI/REST)", + rows: [ + { name: "triggerJob({ id, payload?, priority?, delay?, correlationId? })", type: "{ jobId, triggered }", desc: "POST /jobs/{id}/trigger — enqueue a run of the job named by the {id} path segment; resolves the jobId and a triggered flag." }, + { name: "triggerTask({ id, payload?, priority?, delay?, correlationId? })", type: "{ taskId, triggered }", desc: "POST /tasks/{id}/trigger — enqueue a run of the polyglot task named by the {id} path segment; resolves the taskId and a triggered flag." } + ] +}) }} + +{{ comp callout { type: "note", title: "Declare the plugin dependency with pluginReferences" } }} +When a Service depends on a plugin API — the workers plugin, say — declare it in that Service's +config section with pluginReferences?: string[], a list of the plugin resource names it +consumes (alongside the existing dependsOn). The Aspire helper generation wires those +references so the Service can discover and load the user jobs the plugin API exposes at runtime, and +so the typed client above resolves the plugin's injected URL +(services__workers-api__http__0). It is the config seam behind Service→plugin-API job +discovery. +{{ /comp }} + +{{ comp callout { type: "caution", title: "The {id} path segment is authoritative" } }} +triggerJob / triggerTask resolve the target id from +input.id. On the REST route (POST /jobs/{id}/trigger) oRPC merges the +{id} URL path segment into input.id, so the path is authoritative and a +body id is a redundant fallback that can never disagree with it; on the RPC transport +there is no path segment, so the id you pass in the input object is used directly. If +neither resolves an id, the handler short-circuits to a typed VALIDATION_ERROR +(HTTP 422) through the centralized validationFailed contract helper before any +KV write — it never persists an undefined-keyed run. Handle it as the contract's +typed error on the client, not a generic 500. +{{ /comp }} + +## Graceful shutdown + +Background runners must drain in flight work before they exit, or a redeploy loses jobs +mid-run. The `@netscript/plugin-workers-core/shutdown` subpath provides a `ShutdownManager` +that registers stoppable resources and stops them in priority order, with a timeout, when a +shutdown is requested. + +```ts +// plugins/workers/bin/with-shutdown.ts +import { ShutdownManager } from '@netscript/plugin-workers-core/shutdown'; +import { startWorkers } from '@netscript/plugin-workers-core'; + +const runtime = await startWorkers({ autoStart: true }); +const shutdown = new ShutdownManager({ timeoutMs: 10_000 }); + +// Register the runtime so a drain stops it gracefully (lower priority stops first). +shutdown.register({ id: 'workers-runtime', priority: 100, stop: (reason) => runtime.stop(reason) }); + +// Tie OS signals to the drain, then report what stopped / failed / timed out. +Deno.addSignalListener('SIGTERM', async () => { + const report = await shutdown.shutdown('SIGTERM'); + console.info('shutdown', report.state, { stopped: report.stopped, timedOut: report.timedOut }); + Deno.exit(0); +}); +``` + +{{ comp.apiTable({ + caption: "ShutdownManager — @netscript/plugin-workers-core/shutdown", + rows: [ + { name: "register(resource)", type: "void", desc: "Register a ShutdownResource ({ id, stop(reason?), priority? }) to be stopped on drain." }, + { name: "unregister(id)", type: "void", desc: "Remove a resource from the drain set." }, + { name: "shutdown(reason?, { timeoutMs })", type: "Promise", desc: "Stop registered resources in priority order; returns { state, stopped, failed, timedOut }." }, + { name: "waitForShutdown()", type: "Promise", desc: "Resolves once shutdown has started — await it in long-running loops." }, + { name: "createAbortController()", type: "AbortController", desc: "An AbortController that aborts when shutdown begins; pass its signal into in-flight async work." }, + { name: "state", type: "'running' | 'shutting-down' | 'stopped'", desc: "Current lifecycle state of the manager." } + ] +}) }} + +## Workers API & where jobs come from + +Once Aspire is up and the schema is wired, the workers API on **`:8091`** registers your job +by `id` and exposes HTTP endpoints to seed, trigger, and inspect runs. These are the +endpoints the CLI E2E suite validates live. + +{{ comp.apiTable({ + caption: "Workers API — port 8091 (full generated surface in the workers reference)", + rows: [ + { name: "GET /health", type: "liveness", desc: "Health probe for the workers API service." }, + { name: "GET /api/v1/workers/jobs", type: "list", desc: "All registered job definitions (id, name, topic) discovered from the jobs directories." }, + { name: "POST /api/v1/workers/jobs/{id}/trigger", type: "enqueue", desc: "Enqueue a run of the job with this id, passing a JSON payload body." }, + { name: "GET /api/v1/workers/executions?limit=10", type: "history", desc: "Recent executions and outcomes (KV-backed execution state)." }, + { name: "GET /api/v1/workers/tasks", type: "list", desc: "Task registry view (polyglot defineTask entries)." }, + { name: "POST /api/v1/workers/seed", type: "seed", desc: "Seed the workers store with the registered jobs." }, + { name: "GET /api/v1/workers/subscribe", type: "SSE", desc: "Server-sent-events stream of execution updates (KV-watch)." } + ] +}) }} + +{{ comp callout { type: "note", title: "Where jobs come from" } }} +Each handler file under your jobs directory (workers/jobs by default, +*.ts) is discovered and compiled into one generated registry at +.netscript/generated/plugin-workers/job-registry.ts, keyed by the source filename — +so workers/jobs/process-payment.ts registers as process-payment. Both +the :8091 API service and the background runner load that single registry at +startup: the API service registers the generated user job definitions before it serves, +so your jobs — not just the built-in workers-plugin-health-check — appear in +GET /api/v1/workers/jobs and resolve on trigger. Background execution runs from +plugins/workers/bin/combined.ts, a separate process from the API service — +the API enqueues, the runner executes. A missing generated registry is tolerated as an empty set, +so a fresh workspace boots before you author any job. Set WORKERS_CONCURRENCY on the +worker background process when you need a specific process pool size. Current Aspire metadata also +emits WORKER_CONCURRENCY, but the runtime entrypoint does not consume it; use +Tune the worker runtime for the mismatch details. +{{ /comp }} + +## Observability: real job traces out of the box + +Job-level observability is **not** a stub. The workers runtime instruments the +scheduler → queue → worker → subprocess path with real OpenTelemetry spans, and those traces +appear in the [Aspire dashboard](/explanation/aspire/) automatically once Aspire is up. + +{{ comp.apiTable({ + caption: "What the worker runtime traces automatically (framework layer — real today)", + rows: [ + { name: "Job dispatch + execution", type: "span", desc: "Each enqueued run gets a span with attributes, duration, status, and job.started / job.completed / job.failed / job.exception events." }, + { name: "Step + progress events", type: "event", desc: "job.step.* and job.progress (current / total / percentage) events are emitted on real job runs." }, + { name: "Subprocess trace continuation", type: "context", desc: "W3C traceparent / tracestate is propagated into the subprocess runner, so the child trace links back to the dispatching span." }, + { name: "Scheduler + cron spans", type: "span", desc: "Scheduler-start, schedule-job, dispatch, and cron-run spans cover the timer/cron path that fires scheduled jobs." } + ] +}) }} + +For spans you author *inside* a handler, import directly from **`@netscript/telemetry`** +(e.g. `@netscript/telemetry/instrumentation` for `withChildSpan`). These nest correctly under +the automatic dispatch span. See [Observability](/explanation/observability/) for the model and +[Add OpenTelemetry](/how-to/add-opentelemetry/) for the recipe. + +## Production notes + +{{ comp callout { type: "important", title: "Aspire first, then jobs" } }} +The workers plugin persists job definitions to Postgres and uses Deno KV for execution state, +so bring orchestration up before you exercise it. Step 1 is the database service; +step 2 is Aspire: cd aspire && aspire start provisions Postgres and +Redis, then netscript db init --name init / netscript db generate +wire the schema. Only after Aspire is up will :8091 resolve jobs and record +executions. See Database & migration. +{{ /comp }} + +{{ comp callout { type: "warning", title: "Drain on redeploy, or lose in-flight runs" } }} +A runner that exits without draining abandons jobs mid-execution. Wire a +ShutdownManager to SIGTERM (above), bound it with a +timeoutMs, and check report.timedOut in your logs. Pair the drain +with an idempotencyKey on enqueue so a retried delivery after a forced exit does +not double-charge. For the full recipe see +Tune the worker runtime. +{{ /comp }} + +{{ comp.badge({ status: "partial" }) }} + +{{ comp callout { type: "note", title: "Deferred: the scaffold createJobTools helper is a no-op" } }} +The scaffold's createJobTools(ctx) helper (in +plugins/workers/jobs/job-tools.ts) exposes log, +progress, and trace shims — but in the generated copy its +progress and trace methods are no-op stubs (only log writes to +the console), and it is not a published @netscript/plugin-workers-core +export. This is a known, tracked limitation with a fix planned, not a permanent design choice, +and it is only in that scaffold-facing helper: job dispatch and execution themselves +emit real spans automatically (above). For custom handler spans today, call +@netscript/telemetry helpers directly. Do not say "worker tracing is a no-op" — +that is false for the framework layer. + +{{ /comp }} + +## Reference → + +This hub is intentionally thin — the full generated API for `@netscript/plugin-workers` +(`defineJobHandler`, the job/task builders, the runtime, the shutdown manager, and every +exported type and subpath) lives in the reference. + +{{ comp.xref({ key: "ref:workers" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Learn — ERP Sync, lesson 02", + body: "Author an import job and trigger it over :8091 as part of the continuous ERP-sync tutorial narrative.", + href: "/tutorials/erp-sync/02-import-job/", + icon: "→" + }, + { + title: "Do — Tune the worker runtime", + body: "Pick the runner mode, set WORKERS_CONCURRENCY, choose a queue provider, and wire graceful shutdown.", + href: "/how-to/tune-worker-runtime/", + icon: "◆" + }, + { + title: "Look up — workers reference", + body: "The full generated deno doc API for @netscript/plugin-workers — every exported symbol, type, and subpath.", + href: "/reference/workers/", + icon: "≡" + }, + { + title: "Understand — Durability model", + body: "How jobs compose with sagas: a job publishes a message a saga consumes. The why behind the choreography.", + href: "/explanation/durability-model/", + icon: "◎" + } +] }) }} + +{{ comp.nextPrev({ + prev: { label: "Services & contracts", href: "/services-sdk/services/" }, + next: { label: "Durable sagas", href: "/durable-workflows/sagas/" } +}) }} diff --git a/docs/site/capabilities/ai.md b/docs/site/capabilities/ai.md index 64c2e289c..fcdd9c97f 100644 --- a/docs/site/capabilities/ai.md +++ b/docs/site/capabilities/ai.md @@ -1,81 +1,4 @@ --- -layout: layouts/base.vto -title: AI & agents -templateEngine: [vento, md] +layout: layouts/redirect.vto +redirectTo: /ai/ --- - -# AI & agents - -The AI capability is NetScript's stack for **agentic chat and tool-calling**: a -provider-agnostic engine at the design center, a durable-chat runtime that makes a chat -survive reload and reconnect, and an app-owned chat UI you copy into your workspace. -{{ comp.badge({ status: "alpha" }) }} - -It is layered so each piece stands alone. Two layers ship and install today — the -**durable-chat runtime** (`@netscript/fresh/ai`) and the **chat UI** -(`@netscript/fresh-ui` `ai` copy registry). The provider-agnostic **engine** -(`@netscript/ai`) and the **AI plugin** (`@netscript/plugin-ai` + -`@netscript/plugin-ai-core`) arrive in the next cut. - -{{ comp callout { type: "important", title: "Available from 0.0.1-beta.2" } }} -The engine (@netscript/ai) and the AI plugin -(@netscript/plugin-ai, @netscript/plugin-ai-core, and the -netscript generate ai codegen — ai-tools and -ai-agents registries) are not installable today — they -ship in 0.0.1-beta.2. Do not -deno add jsr:@netscript/ai yet. What you can build on now is the -durable-chat runtime and the chat UI, both self-contained and already on JSR. -{{ /comp }} - -## The two planes - -NetScript models live state in two distinct planes, and the headline rule of this -capability is **never build chat on the wrong primitive**. A **StreamDB shape** is the -right tool for live lists, boards, and tables — CRUD rows reconciled last-writer-wins. A -**durable chat session** is an append-only, replayable chunk log keyed by `sessionId`, -because a tool call is a multi-chunk mid-stream event that cannot collapse into a single -row without losing its `pending → streaming → complete` states. Reach for -[durable streams](/capabilities/streams/) for data; reach for -[durable chat](/ai/durable-chat/) for conversation. - -## Minimal taste — a durable chat proxy - -Every durable chat needs one route that proxies the browser's chat-stream request to the -session's durable-stream URL. That route is a single call: - -```ts -// routes/api/chat/[sessionId].ts -import { createNetScriptChatStreamProxy } from "@netscript/fresh/ai"; - -const proxy = createNetScriptChatStreamProxy({ - target: (req) => ({ - sessionId: new URL(req.url).pathname.split("/").pop()!, - }), -}); - -export const handler = { POST: proxy, GET: proxy }; -``` - -Authorization lives where you produce the session response (`toNetScriptChatResponse` -with an `authorize` hook — required in production), and the transcript is rendered by the -copy-registry chat UI. The [durable chat page](/ai/durable-chat/) walks the full recipe. - -## The section - -{{ comp.featureGrid({ items: [ - { title: "Overview & the two planes", icon: "◎", body: "The stack story, the plane rule, the engine as design center, and the plugin thinness laws.", href: "/ai/" }, - { title: "Durable chat", icon: "💬", body: "@netscript/fresh/ai: the chat-stream proxy, the one-projection law, authorize, and the ui:// sandbox. Ships today.", href: "/ai/durable-chat/" }, - { title: "Chat UI", icon: "🎨", body: "The fresh-ui `ai` copy registry: composer, message thread, tool cards, and the generative-UI block renderer. Ships today.", href: "/ai/chat-ui/" }, - { title: "AI engine", icon: "⚙️", body: "@netscript/ai: runtime, contracts vocabulary, ports, tools, agent loop, MCP transports, provider adapters. Beta.2.", href: "/ai/engine/" } -] }) }} - -## Reference - -The section pages orient; the generated reference enumerates every symbol. The shipping -layers live under [`@netscript/fresh`](/reference/fresh/) (the `/ai` and `/ai/sandbox` -subpaths) and [`@netscript/fresh-ui`](/reference/fresh-ui/) (the copy-registry manifest). -The engine and the AI plugin have their own generated reference pages: - -{{ comp.xref({ key: "ref:ai" }) }} -{{ comp.xref({ key: "ref:plugin-ai" }) }} -{{ comp.xref({ key: "ref:plugin-ai-core" }) }} diff --git a/docs/site/capabilities/auth.md b/docs/site/capabilities/auth.md index ebab54c79..62bccee8e 100644 --- a/docs/site/capabilities/auth.md +++ b/docs/site/capabilities/auth.md @@ -1,351 +1,4 @@ --- -layout: layouts/base.vto -title: Authentication -templateEngine: [vento, md] -prev: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" } -next: null +layout: layouts/redirect.vto +redirectTo: /identity-access/auth/ --- - -# Authentication - -NetScript ships authentication as a first-class official plugin: `netscript plugin install @netscript/plugin-auth` -adds an `auth-api` oRPC service that boots on port **8094** alongside your workers, sagas, and -triggers. The plugin is **pure-backend** — it owns the session lifecycle, OAuth/OIDC redirect -flow, and a stable five-endpoint REST/RPC surface, but it does **not** ship UI. You compose -exactly **one active backend** (single-active-backend), chosen by an environment variable, and -the service exposes the same contract no matter which backend is wired in. - -{{ comp.badge({ status: "alpha" }) }} - -{{ comp.diagram({ - src: "/assets/diagrams/auth-flow.svg", - alt: "A browser sign-in request enters the auth-api service, which resolves the single active backend from an env var; the kv-oauth backend redirects to the OAuth/OIDC provider, handles the callback, mints a normalized session in Deno KV, sets the __Host-ns_session cookie, and maps the session to a NetScript Principal that downstream services trust.", - caption: "Auth flow: browser → auth-api → resolved backend → provider redirect/callback → session store → __Host-ns_session cookie → Principal." -}) }} - -The design follows the framework's contracts-first doctrine: a core seam package -(`@netscript/plugin-auth-core`) defines the `AuthBackendPort`, three interchangeable adapter -packages implement it, and the `@netscript/plugin-auth` plugin composes the selected one into a -running service. Of the three adapters, only the KV-OAuth backend implements the full -**interactive** sign-in/callback flow today; the WorkOS and better-auth adapters are -non-interactive by design. - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for the auth plugin when you need session-based authentication backed by an OAuth -or OIDC provider — a typed /api/v1/auth/* surface your services and Fresh -UI can call to sign users in, resolve the current session, and sign out. The default -kv-oauth backend gives you a complete interactive redirect flow with KV-stored -sessions and an opaque HMAC-signed session token. To wire it into a workspace step by step, see -Add authentication; to understand the pure-backend -port model and why only one backend is active, see -The authentication model. -{{ /comp }} - -## What it is - -Auth is a **port-and-adapter seam, not a monolith**. The core package -`@netscript/plugin-auth-core` is contract-only: it defines the `AuthBackendPort` (and the four -sub-ports it composes — provider registry, session store, token crypto, and principal mapper), -the normalized `AuthSession` and `Principal` shapes, the `AUTH_SESSION_STATES`, the auth oRPC -contract, the config schemas, and the stream-event schemas. It contains **no provider SDK code**. -Each backend adapter (`@netscript/auth-kv-oauth`, `@netscript/auth-workos`, -`@netscript/auth-better-auth`) implements `AuthBackendPort` against a concrete identity provider, -and the `@netscript/plugin-auth` plugin selects exactly one of them at boot via -`NETSCRIPT_AUTH_BACKEND` and serves it as the `auth-api` service. Because every backend produces -the same `Principal`, downstream services authorize identically regardless of which provider is -wired in. The full rationale — why auth is pure-backend, what the single-active boundary buys you, -and how `InteractiveFlowPort` gates the redirect flow — is in -{{ comp.xref({ key: "explain:auth-model", text: "The authentication model" }) }}. - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Do — Add authentication", - body: "Task recipe: add the auth plugin, set NETSCRIPT_AUTH_BACKEND, run the auth.prisma migration, and wire kv-oauth with a provider preset.", - href: "/how-to/add-authentication/", - icon: "◆" - }, - { - title: "Understand — The authentication model", - body: "Why auth is pure-backend: the AuthBackendPort seam, the single-active-backend boundary, InteractiveFlowPort, and the three adapters' capability matrices.", - href: "/explanation/auth-model/", - icon: "✦" - }, - { - title: "Compose — Service authn / authz", - body: "Gate a service's own routes with the provider-agnostic /auth middleware seam — verify a credential or trust a header, complementary to this plugin.", - href: "/capabilities/services/", - icon: "→" - } -] }) }} - -## Add it in two shapes - -The fastest path is the **provider preset**: add the plugin, set `NETSCRIPT_AUTH_BACKEND` -(it already defaults to `kv-oauth`), and hand a preset like `providers.google({...})` to -`createKvOAuthBackend`. The **advanced** shape uses the same factory but defines a custom OIDC -provider explicitly with `defineOAuthProvider` when your identity provider is not one of the -fourteen built-in presets. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — provider preset", - lang: "ts", - code: "// Add the plugin from the workspace root:\n// netscript plugin install @netscript/plugin-auth\n// Default backend is already kv-oauth (NETSCRIPT_AUTH_BACKEND=kv-oauth).\n\nimport { createKvOAuthBackend, getRequiredEnv, providers } from '@netscript/auth-kv-oauth';\n\n// One of 14 presets: github, google, gitlab, discord, slack, spotify,\n// facebook, twitter, auth0, okta, awsCognito, azureAd, logto, clerk.\nexport const backend = await createKvOAuthBackend({\n provider: providers.google({\n clientId: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_ID'),\n clientSecret: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_SECRET'),\n redirectUri: getRequiredEnv('NETSCRIPT_AUTH_REDIRECT_URI'),\n }),\n});\n\n// backend.name === 'kv-oauth' — the only adapter with backend.interactive." - }, - { - label: "Advanced — explicit provider", - lang: "ts", - code: "// Same factory, but define the OIDC provider yourself when your IdP\n// is not one of the built-in presets.\nimport { createKvOAuthBackend, defineOAuthProvider, getRequiredEnv } from '@netscript/auth-kv-oauth';\n\nconst provider = defineOAuthProvider({\n id: 'my-idp',\n clientId: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_ID'),\n clientSecret: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_SECRET'),\n authorizationEndpoint: getRequiredEnv('NETSCRIPT_AUTH_AUTHORIZATION_ENDPOINT'),\n tokenEndpoint: getRequiredEnv('NETSCRIPT_AUTH_TOKEN_ENDPOINT'),\n userInfoEndpoint: getRequiredEnv('NETSCRIPT_AUTH_USERINFO_ENDPOINT'),\n redirectUri: getRequiredEnv('NETSCRIPT_AUTH_REDIRECT_URI'),\n scopes: ['openid', 'profile', 'email'],\n});\n\nexport const backend = await createKvOAuthBackend({ provider });" - } -] }) }} - -{{ comp callout { type: "note", title: "Alpha package pins" } }} -The scaffold pins jsr:@netscript/plugin-auth-core{{ releaseSpecifier }} and siblings -at the exact aligned NetScript version. Add auth through -netscript plugin install @netscript/plugin-auth, which wires the workspace correctly for -the alpha instead of hand-editing imports. -{{ /comp }} - -## Sign in, resolve the session, sign out - -Once the plugin is wired, your front end or a typed client drives the five-endpoint surface. A -sign-in begins the redirect, the provider returns to your callback, the service mints a session -and sets the cookie, and every subsequent request resolves the current session from that cookie. -The snippet below is a copy-ready client flow against the REST surface. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Browser / typed client flow", - lang: "ts", - code: "// app/sign-in.ts — drive the auth-api REST surface from the browser\nconst AUTH = 'http://localhost:8094/api/v1/auth';\n\n// 1. Begin sign-in. The kv-oauth backend responds with a provider redirect;\n// follow it to authenticate with the IdP.\nlocation.href = `${AUTH}/signin`;\n\n// 2. The IdP redirects back to /api/v1/auth/callback, which mints the\n// session and sets the __Host-ns_session cookie automatically.\n\n// 3. Resolve the current session on any later request (cookie is sent\n// automatically; the response is { authenticated, user, session }).\nconst me = await fetch(`${AUTH}/me`, { credentials: 'include' })\n .then((r) => r.json());\nif (me.authenticated) {\n console.log('signed in as', me.user.subject, '— state', me.session.state);\n}\n\n// 4. Sign out — revokes the session and clears the cookie.\nawait fetch(`${AUTH}/signout`, { method: 'POST', credentials: 'include' });" - }, - { - label: "Server-side backend port", - lang: "ts", - code: "// server/auth-guard.ts — use the resolved backend directly in app code\nimport { AuthBackendOperationUnsupportedError } from '@netscript/plugin-auth-core/ports';\nimport { backend } from './backend.ts';\n\n// Resolve the current session from an incoming Request via the session store.\nconst session = await backend.sessions.getSession({ request: incoming });\nif (session?.state === 'active') {\n // Map the normalized session to a NetScript Principal for authorization.\n const { principal } = backend.principalMapper.mapSessionToPrincipal(session);\n console.log(principal.subject, principal.scopes, principal.roles);\n}\n\n// Interactive sign-in is optional capability — guard before you call it.\nif (!backend.interactive) {\n throw new AuthBackendOperationUnsupportedError(\n backend.name,\n 'interactive.signIn',\n 'The active backend authenticates sessions only.',\n );\n}\nconst redirect = await backend.interactive.signIn(incoming);" - } -] }) }} - -## Key types first — the core auth contract - -Every backend normalizes to the same two shapes the rest of NetScript depends on: an -`AuthSession` (what the store persists) and a `Principal` (what services authorize against). The -`AuthSession` is mapped to a `Principal` by the backend's `principalMapper`, and the -`authenticate()` method on every backend returns an `AuthnResult`. These are the contract-only -types from `@netscript/plugin-auth-core` — confirmed against the live export surface. - -{{ comp.apiTable({ - caption: "Principal (@netscript/plugin-auth-core) — the identity services authorize against", - rows: [ - { name: "subject", type: "string", desc: "Stable subject identifier — a user id, service id, or API-key id." }, - { name: "scopes", type: "readonly string[]", desc: "Granted scopes for per-operation RPC/REST permission checks." }, - { name: "roles", type: "readonly string[]", desc: "Granted roles for role-based access checks." }, - { name: "scheme", type: "'api-key' | 'bearer' | 'trusted-header' | 'custom'", desc: "How the principal was established. Auth-plugin backends use 'custom' with the claim bag below." }, - { name: "claims", type: "Readonly>", desc: "Opaque verified claims — organization/tenant id, session id, provider permissions, or normalized WorkOS/better-auth metadata." } - ] -}) }} - -{{ comp.apiTable({ - caption: "AuthSession (@netscript/plugin-auth-core) — the normalized session the store persists", - rows: [ - { name: "id", type: "string", desc: "Stable session id; the value an opaque session token resolves to." }, - { name: "userId / subject", type: "string", desc: "User id and stable subject the session belongs to." }, - { name: "state", type: "AuthSessionState", desc: "Lifecycle state: 'active' | 'expired' | 'revoked' (from AUTH_SESSION_STATES)." }, - { name: "scopes / roles", type: "readonly string[]", desc: "Scopes and roles carried into the mapped Principal." }, - { name: "claims", type: "Readonly>", desc: "Verified provider claims preserved on the session." }, - { name: "issuedAt / expiresAt", type: "string (ISO)", desc: "Issue and expiry timestamps; refreshedAt / revokedAt are set on those transitions." }, - { name: "accountId / providerId", type: "string?", desc: "Optional linked provider account and provider id." }, - { name: "traceparent / tracestate", type: "string?", desc: "Optional W3C trace context carried for audit correlation." } - ] -}) }} - -{{ comp.apiTable({ - caption: "AuthnResult (@netscript/plugin-auth-core) — what backend.authenticate() returns", - rows: [ - { name: "{ ok: true, principal }", type: "success", desc: "A resolved Principal; optional responseHeaders and setCookies the service flushes to the client." }, - { name: "{ ok: false, reason }", type: "rejection", desc: "A typed rejection reason — fail-loud, never a silent anonymous principal." } - ] -}) }} - -## The service surface - -The plugin's service is named `auth-api` and is built with `@netscript/service`'s oRPC builder -(`createService(...).withRPC()`), not raw Hono. It mounts a public REST surface at -`/api/v1/auth/*` and the equivalent oRPC surface at `/api/rpc/v1/auth/*`, plus standard -`/health/live` and `/health/ready` probes. - -{{ comp.apiTable({ - caption: "auth-api endpoints (REST at /api/v1/auth/*, oRPC at /api/rpc/v1/auth/*)", - rows: [ - { name: "signin", type: "POST", desc: "Begins the interactive sign-in. Requires backend.interactive; on WorkOS / better-auth it returns AUTH_PROVIDER_ERROR (502) because those backends are non-interactive." }, - { name: "callback", type: "POST", desc: "Completes the OAuth/OIDC redirect, mints the session, sets the session cookie. Interactive-only — same non-interactive caveat as signin." }, - { name: "signout", type: "POST", desc: "Revokes the current session and clears the cookie." }, - { name: "session", type: "GET", desc: "Resolves the current AuthSession from the cookie (active | expired | revoked), refreshing on read when policy allows." }, - { name: "me", type: "GET", desc: "Returns { authenticated: true, user, session } when a valid active session exists, or { authenticated: false } (HTTP 200) when there is none." } - ] -}) }} - -{{ comp callout { type: "note", title: "Backend selection is one env var" } }} -The service picks one active backend from NETSCRIPT_AUTH_BACKEND -(or appsettings auth.backend): valid values are kv-oauth | -workos | better-auth, defaulting to kv-oauth. -This is the single-active-backend boundary — there is no multi-active routing, -cross-backend account linking, or global logout across backends in v1. Switching providers means -changing the env var and the backend's credentials, not running two backends at once. - -{{ /comp }} - -## The three backends - -All three implement the same `AuthBackendPort`, so the `auth-api` contract is identical across -them. They differ in one decisive capability: whether they expose an interactive -sign-in/callback flow. Each ships a single factory that returns an `AuthBackendPort`. - -{{ comp.apiTable({ - caption: "Backend adapters: factory, capability, and what each one needs", - rows: [ - { name: "@netscript/auth-kv-oauth", type: "createKvOAuthBackend(options)", desc: "Interactive (default). Full OAuth/OIDC redirect flow, Deno KV session store, real createSession / refreshSession / revokeSession, refresh-on-read (refreshMode / refreshSkewMs), opaque session token, __Host-ns_session cookie, AES-256-GCM token-at-rest. Needs a provider preset or defineOAuthProvider config." }, - { name: "@netscript/auth-workos", type: "createWorkosBackend({ workos, cookiePassword })", desc: "Non-interactive. WorkOS AuthKit sealed sessions, stateless verification. Needs a WorkOS SDK client and a cookie password. signin / callback return AUTH_PROVIDER_ERROR; session mutations throw AuthBackendOperationUnsupportedError." }, - { name: "@netscript/auth-better-auth", type: "createBetterAuthBackend({ auth, sessionTokenSecret })", desc: "Non-interactive. Validates externally-issued sessions via auth.api.getSession over a Prisma store (createNetscriptBetterAuth wires the better-auth Prisma adapter). Needs a better-auth instance and a token secret." } - ] -}) }} - -{{ comp callout { type: "important", title: "Only kv-oauth is interactive" } }} -The signin and callback endpoints require a backend that implements the -optional InteractiveFlowPort (signIn / handleCallback / -getSessionId / signOut). Today that is only -kv-oauth. On workos and better-auth those two -endpoints return a typed AUTH_PROVIDER_ERROR ("does not expose an interactive flow"), -and direct session mutations throw AuthBackendOperationUnsupportedError — a -deliberate, fail-loud capability boundary rather than a silent no-op. Use those adapters when -sign-in is handled by WorkOS or an external better-auth deployment and NetScript only verifies the -resulting session. -{{ /comp }} - -## How auth integrates with services - -The auth **plugin** (this page) signs human users in and resolves their sessions. A -NetScript **service** gates its own routes with the separate, provider-agnostic -`@netscript/service/auth` seam — `.withAuthn()` resolves a `Principal` and `.withAuthz()` makes -an authorization decision from it. Both layers speak the same `Principal` type, so they compose: -the auth plugin establishes identity, and a service's `.withAuthn({ authenticator })` can trust a -backend's authenticator to turn a request into that `Principal`. By default a service protects -`/api` and leaves `/health` anonymous. The full builder seam — `createStaticCredentialAuthenticator`, -`createTrustedHeaderAuthenticator`, `createScopeAuthorizer`, and the preset -`defineService(router, { auth: { authn, authz } })` form — is documented on the services hub. - -{{ comp callout { type: "note", title: "Plugin vs. service-auth seam" } }} -The auth plugin owns interactive sign-in, OAuth callbacks, and session cookies. -The service-auth seam is for -machine-to-machine and gateway-fronted gating — static credentials, trusted upstream headers, -scope checks — without an interactive identity provider. They are complementary and can run -together: sign users in with the plugin, then gate individual service routes with -.withAuthn() / .withAuthz(). See -{{ comp.xref({ key: "cap:services", text: "Services & contracts" }) }}. -{{ /comp }} - -## Database & runtime events - -The plugin contributes a package-provided `auth.prisma` schema with four -better-auth-shaped models — `User` → `auth_users`, `Session` → `auth_sessions`, -`Account` → `auth_accounts`, `Verification` → `auth_verifications`. These tables back the -better-auth adapter; `kv-oauth` keeps its sessions in Deno KV, and WorkOS is stateless. As with -every plugin schema, you bring the tables to life by running the database workflow **after Aspire -is up** — see [Database migrations](/how-to/database-migration/). - -The plugin also emits five durable `auth.*` runtime events through the durable-streams runtime — -the `AUTH_STREAM_EVENT_TYPES`: `auth.signin.started`, `auth.signin.failed`, -`auth.token.refreshed`, `auth.session.revoked`, and `auth.oidc.completed`. The session projection -is described by the `authStreamSchema` entity stream. For the streams runtime itself see -{{ comp.xref({ key: "cap:streams", text: "Durable streams" }) }}. - -{{ comp callout { type: "warning", title: "Scope: events are best-effort, no audit surface" } }} -The auth.* events are best-effort: they are no-ops unless the -durable-streams service is wired (DURABLE_STREAMS_URL / -services__streams__http__0 set), so they are not a guaranteed audit -trail. Only oidc.completed, token.refreshed, and -session.revoked write the session projection; signin.started / -signin.failed are diagnostic-only. There is no dedicated auth telemetry or -audit-observability surface yet — do not build dashboards expecting one. For general -tracing and structured logs see Telemetry & logging. -{{ /comp }} - -## Endpoints & ports - -{{ comp.apiTable({ - caption: "Authentication runtime surface", - rows: [ - { name: ":8094", type: "port", desc: "auth-api default port (AUTH_API_DEFAULT_PORT). Family: workers :8091, sagas :8092, triggers :8093, auth :8094." }, - { name: "auth-api", type: "service name", desc: "AUTH_API_SERVICE_NAME — the service contribution the auth plugin (AUTH_PLUGIN_ID 'auth') adds." }, - { name: "/api/v1/auth/*", type: "REST", desc: "Public REST surface: signin, callback, signout, session, me." }, - { name: "/api/rpc/v1/auth/*", type: "oRPC", desc: "The oRPC surface for the same five operations, for typed NetScript clients." }, - { name: "/health/live", type: "HTTP", desc: "Liveness probe; /health/ready for readiness. OpenAPI + docs served via .withOpenAPI()/.withDocs()." }, - { name: "NETSCRIPT_AUTH_BACKEND", type: "env", desc: "Selects the single active backend: kv-oauth (default) | workos | better-auth." } - ] -}) }} - -## Production notes - -{{ comp callout { type: "warning", title: "Footguns before you ship" } }} -
    -
  • Only kv-oauth signs users in. If -NETSCRIPT_AUTH_BACKEND is workos or better-auth, the -signin / callback endpoints return AUTH_PROVIDER_ERROR by -design — those adapters only verify externally-issued sessions. Pick the backend that matches who -owns the sign-in.
    • -
  • The session cookie is __Host-ns_session. The -__Host- prefix requires Path=/, no Domain, and -HTTPS — a misconfigured domain or a plain-HTTP origin makes the cookie refuse to -set. Use a real TLS origin in production.
    • -
  • Switching providers is an env-var + credentials change, not a runtime toggle. -There is no multi-active routing, cross-backend account linking, or global logout across backends -in v1 (single-active-backend).
    • -
  • Auth audit is opt-in, and not a compliance-grade ledger. A real, structured, -redacted auth audit surface ships via createAuthTelemetry, but it only records when a -subjectHashSalt is configured (resolved from NETSCRIPT_AUTH_AUDIT_SALT) — -with no salt it runs as a no-op. The raw auth.* stream events are themselves best-effort -and no-op until the durable-streams service is wired. Treat the audit surface as a strong, queryable -trail for operational and security review, not a tamper-evident, write-once compliance log. See -{{ comp.xref({ key: "explain:observability" }) }}.
    • -
  • Run the auth.prisma migration after Aspire is up. The -better-auth backend's tables (and any DB-backed flow) need your database provisioned first -(Postgres is the recommended engine; or mysql / mssql / sqlite via ---db — better-auth persists through Prisma, so it is not Postgres-specific) — the same -Aspire-first ordering every plugin schema follows.
    • -
-{{ /comp }} - -## Reference - -The auth runtime is a `@netscript/service`; the auth plugin, core contract package, and backend -adapters now have dedicated generated reference pages. - -{{ comp.xref({ key: "ref:auth" }) }} -{{ comp.xref({ key: "ref:service" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Do — Add authentication", - body: "Task recipe: add the auth plugin, set NETSCRIPT_AUTH_BACKEND, run the auth.prisma migration, and wire kv-oauth with a provider preset.", - href: "/how-to/add-authentication/", - icon: "◆" - }, - { - title: "Understand — The authentication model", - body: "Why auth is pure-backend: the AuthBackendPort seam, the single-active-backend boundary, InteractiveFlowPort, and the three adapters' capability matrices.", - href: "/explanation/auth-model/", - icon: "✦" - }, - { - title: "Look up — @netscript/service reference", - body: "The oRPC service builder behind auth-api: createService().withRPC(), the contract surface, the /auth authn-authz seam, and the runtime that serves /api/v1/auth/*.", - href: "/reference/service/", - icon: "≡" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" }, next: null }) }} - - diff --git a/docs/site/capabilities/background-jobs.md b/docs/site/capabilities/background-jobs.md index a5cdc4c3e..29e21c285 100644 --- a/docs/site/capabilities/background-jobs.md +++ b/docs/site/capabilities/background-jobs.md @@ -1,475 +1,4 @@ --- -layout: layouts/base.vto -title: Background jobs -templateEngine: [vento, md] -prev: { label: "Services & contracts", href: "/capabilities/services/" } -next: { label: "Durable sagas", href: "/capabilities/durable-sagas/" } +layout: layouts/redirect.vto +redirectTo: /background-processing/workers/ --- - -# Background jobs - -A NetScript **background job** is a durable, KV-backed TypeScript handler that runs in -its own thread-isolated worker, separate from your request-serving services. You author a -job as one `defineJobHandler(...)` callable, give it an `id`, and the runtime takes care of -registration, dispatch, retry, execution tracking, scheduling, and an HTTP API to enqueue -and inspect runs. It is the unit you reach for whenever work should happen *after* a request -returns — charging a payment, sending a welcome email, processing an upload — without -blocking the caller. - -{{ comp.diagram({ src: "/assets/diagrams/queue-worker-scheduler.svg", alt: "An enqueue call (from a trigger, an HTTP POST to the workers API, or the scheduler) places a job on the durable queue; the worker runtime pulls it and runs the handler in one of three runner modes — in-process, web-worker (one V8 isolate per worker), or subprocess — then writes a JobResult to the KV-backed execution store, which streams updates back over SSE.", caption: "Enqueue → durable queue → worker runtime (in-process / web-worker / subprocess) → result store. The scheduler fires cron-defined jobs onto the same queue; graceful shutdown drains in-flight runs before the runner stops." }) }} - -## What it is - -A job is plain TypeScript that runs **in the worker process, not the request process**. -The headline surface — `defineJobHandler`, `createSuccessResult`, and `createFailureResult` -from `@netscript/plugin-workers-core` — lets you write a typed handler over a `ctx`, do the -work, and return a structured `JobResult`. Job dispatch and execution are instrumented with -real OpenTelemetry spans that show up in the [Aspire dashboard](/explanation/aspire/) -automatically, so a queued run is observable end to end without wiring. The runner mode -(how the handler is isolated) is a **user-tunable** — see the runtime-mode table below — and -the same queue and scheduler also drive [polyglot tasks](/capabilities/polyglot-tasks/) when -the work is owned by another runtime. The why-behind-the-choreography lives in -[Durability model](/explanation/durability-model/). - -{{ comp callout { type: "tip", title: "Jobs vs. tasks vs. sagas vs. services" } }} -Reach for a background job when the work is fire-and-forget or -deferrable and written in TypeScript: it should survive the request that started -it, run on its own schedule or trigger, and be retried and observed independently. If the -work must run in another runtime (Python, .NET, a shell script, a binary), use a -polyglot task — it shares this same queue, -retry, and telemetry machinery but spawns a subprocess. If the work coordinates -several steps across time (waiting for messages, compensating on failure), model it -as a durable saga; jobs and sagas compose. For a -synchronous request/response API, author a -service instead. -{{ /comp }} - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — ERP Sync, lesson 02", - body: "The tutorial rung: add the worker plugin to the running app, author an import job, and trigger it over :8091 as part of the ERP-sync narrative.", - href: "/tutorials/erp-sync/02-import-job/", - icon: "→" - }, - { - title: "Do — Tune the worker runtime", - body: "Recipe: pick the in-process / web-worker / subprocess runner, set WORKERS_CONCURRENCY, and choose a queue provider for your deployment.", - href: "/how-to/tune-worker-runtime/", - icon: "◆" - }, - { - title: "Understand — Polyglot tasks", - body: "Run non-TypeScript work (Python, .NET, shell) on the same durable queue and scheduler as a managed subprocess.", - href: "/capabilities/polyglot-tasks/", - icon: "◎" - } -] }) }} - -## Minimal example - -Add the workers plugin to a published workspace with the public package install flow: - -```bash -netscript plugin install @netscript/plugin-workers -``` - -For local-source contributor work inside this monorepo, use the maintainer binary when you need -first-party samples: - -```bash -deno run -A packages/cli/bin/netscript-dev.ts plugin install worker --name workers --samples -``` - -That local path lands real, compiling modules you can read and trigger immediately — including -`plugins/workers/jobs/health-check.ts` (a job handler) and -`plugins/workers/tasks/validate-payload.ts` (a polyglot task). The plugin's API service comes up -on **port 8091**. - -A job handler is an async callable over a `ctx` object that returns a `JobResult`. Parse the -payload with a Zod schema, do the work, and return `createSuccessResult(...)` or -`createFailureResult(...)`. The job's identity is attached with `Object.assign(handler, { id })`. - -```ts -// plugins/workers/jobs/process-payment.ts -import { - createFailureResult, - createSuccessResult, - defineJobHandler, -} from '@netscript/plugin-workers-core'; -import { z } from 'zod'; - -// Payload contract — parse ctx.payload before doing any work. -const ProcessPaymentPayloadSchema = z.object({ - orderId: z.string().min(1), - amountCents: z.number().int().positive(), -}); - -const handler = defineJobHandler(async (ctx) => { - const { orderId, amountCents } = ProcessPaymentPayloadSchema.parse(ctx.payload ?? {}); - - // Optional progress callback — the runtime wires reportProgress on real runs. - ctx.reportProgress?.(10, 'charging customer'); - - const charge = await chargeCustomer(orderId, amountCents); - if (!charge.ok) { - // A failure result is recorded and feeds the retry policy. - return createFailureResult(`charge declined: ${charge.reason}`); - } - - ctx.reportProgress?.(100, 'charged'); - // The success payload (data) is persisted on the execution record. - return createSuccessResult({ orderId, chargeId: charge.id, amountCents }); -}); - -// The id is how the runtime registers, lists, and triggers the job. -export default Object.assign(handler, { id: 'process-payment' }); -``` - -Once the workers API is up (Aspire first — see Production notes), enqueue a run by `id`: - -```bash -# Enqueue the process-payment job (workers API on port 8091). -curl -X POST http://localhost:8091/api/v1/workers/jobs/process-payment/trigger \ - -H 'content-type: application/json' \ - -d '{"orderId":"o_42","amountCents":4999}' - -# Watch it land in the KV-backed execution history. -curl http://localhost:8091/api/v1/workers/executions?limit=10 -``` - -## Key types first — `JobHandlerContext` & `JobResult` - -A handler is `(ctx: JobHandlerContext) => JobResult | Promise<…>`. These -two shapes are the contract you write against; read them before the option tables. - -{{ comp.apiTable({ - caption: "JobHandlerContext — the ctx passed to defineJobHandler", - rows: [ - { name: "id", type: "string", desc: "The execution id for this run. Stable per dispatched run." }, - { name: "job", type: "{ id: string } | undefined", desc: "The job definition reference (its id), when available." }, - { name: "payload", type: "TPayload", desc: "The enqueued payload. Parse it with your Zod schema before use." }, - { name: "correlationId", type: "string | undefined", desc: "Correlation id propagated across the dispatch for tracing." }, - { name: "traceparent", type: "string | undefined", desc: "W3C traceparent of the dispatching span; child spans nest under it." }, - { name: "tracestate", type: "string | undefined", desc: "W3C tracestate accompanying the traceparent." }, - { name: "reportProgress", type: "(percent, message?) => void | Promise", desc: "Optional progress callback wired by the runtime on real runs; emits job.progress events." } - ] -}) }} - -{{ comp.apiTable({ - caption: "JobResult — return createSuccessResult() or createFailureResult()", - rows: [ - { name: "success", type: "true | false", desc: "Discriminant. createSuccessResult sets true; createFailureResult sets false. Branch on this." }, - { name: "data", type: "TResult | undefined", desc: "The result payload, persisted on the execution record (present on success, optional on failure)." }, - { name: "error", type: "string", desc: "Failure message — required when success is false (the first arg to createFailureResult)." } - ] -}) }} - -## Worker runtime modes (`WORKER_RUNTIMES`) - -How a handler is isolated from the API process is a tunable: `WORKER_RUNTIMES` enumerates the -three runner modes the worker runtime supports. The scaffold default is **web-worker**, where -each worker is its own V8 isolate sized by the `WORKERS_CONCURRENCY` env var. Pick the mode -that matches your isolation, memory, and parallelism needs. - -{{ comp.apiTable({ - caption: "WORKER_RUNTIMES — runner isolation modes (WorkerRuntime type)", - rows: [ - { name: "in-process", type: "WorkerRuntime", desc: "Runs the handler in the same process via the in-process runner (registry-first). Lowest overhead, no isolation — best for tests, compiled binaries, and single-tenant local composition." }, - { name: "web-worker", type: "WorkerRuntime", desc: "Runs each worker in its own Web Worker / V8 isolate (~20-40 MB each). The scaffold default; WORKERS_CONCURRENCY sets the process pool size for parallel job execution. Keep it low to bound memory." }, - { name: "subprocess", type: "WorkerRuntime", desc: "Runs the handler in a spawned subprocess. Strongest process isolation; only Deno tasks get permission sandboxing through .permissions(). Python, .NET, shell, PowerShell, and cmd inherit the worker process's OS permissions." } - ] -}) }} - -{{ comp.apiTable({ - caption: "Deployment & scaling knobs (workers config)", - rows: [ - { name: "WORKERS_CONCURRENCY", type: "env (number)", desc: "Runtime worker process pool size. The entrypoint reads this plural variable; current Aspire metadata also emits WORKER_CONCURRENCY, but the runtime does not consume it." }, - { name: "concurrency", type: "number", desc: "Per-topic max concurrent workers (WorkersConfigData.concurrency / per-group scaling)." }, - { name: "mode", type: "'combined' | 'distributed'", desc: "Per-topic deployment mode: one combined runner vs. distributed runners. Defaults to 'combined'." }, - { name: "queueProvider", type: "'auto' | 'deno-kv' | 'redis' | 'postgres' | 'amqp'", desc: "Queue backend. 'auto' resolves a provider; see Choose a queue provider." }, - { name: "jobsDir / tasksDir", type: "string", desc: "Directories scanned for default-exported job and task modules." } - ] -}) }} - -{{ comp callout { type: "note", title: "Tune it without touching code" } }} -The runner mode and pool size are deployment settings, not handler concerns — the same -process-payment handler runs unchanged under any -WORKER_RUNTIMES mode. Start on the -web-worker default with a small WORKERS_CONCURRENCY, move to -subprocess when you need hard isolation, and drop to in-process for -tests and compiled single-binary deployments. Resolution precedence (schema default → config -file → env → override) is covered in -runtime configuration. -{{ /comp }} - -## Enqueue from a trigger - -The HTTP `…/trigger` endpoint is one way in; the other is **declaratively, from a trigger -handler**, which returns an `enqueueJob(...)` action. `enqueueJob(job, options)` comes from -`@netscript/plugin-triggers-core` and binds an imported job definition to a payload, so an -inbound webhook or a scheduled trigger drops work onto this same runtime. - -```ts -// plugins/triggers/triggers/payment-webhook.ts -import { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core'; -// Import the job definition you want to dispatch (its default export carries the id). -import processPayment from '../../workers/jobs/process-payment.ts'; - -const handler = defineWebhook( - // The handler returns trigger actions; enqueueJob is the most common one. - (event) => [ - enqueueJob(processPayment, { - payload: { orderId: event.payload.orderId, amountCents: event.payload.amountCents }, - idempotencyKey: event.payload.orderId, // optional: collapse duplicate deliveries - }), - ], - { id: 'payment-webhook', path: '/webhooks/payment', verifier: 'hmac-sha256', secretEnv: 'WEBHOOK_SECRET' }, -); - -export default handler; -``` - -{{ comp.apiTable({ - caption: "enqueueJob(job, options) — EnqueueJobOptions (from @netscript/plugin-triggers-core)", - rows: [ - { name: "payload", type: "TPayload", desc: "The payload handed to the job's ctx.payload. Parse it with the job's Zod schema." }, - { name: "idempotencyKey", type: "string", desc: "Collapses duplicate deliveries — at-most-once effect per key." }, - { name: "concurrencyKey", type: "string", desc: "Serializes runs that share a key (e.g. per-order) so they do not overlap." }, - { name: "priority", type: "number", desc: "Dispatch priority for this enqueue action." } - ] -}) }} - -## Trigger a job from a typed client - -The `…/trigger` curl above is the OpenAPI/REST way in. Because the workers API is a **plugin -service**, the same trigger is reachable with a generated typed client over the RPC route -`/api/rpc/*` — no OpenAPI-only fallback and no transport `404`. A first-party plugin API mounts -its router under a named segment, so the client takes a `routerName` alongside `serviceName`; -Aspire injects the URL under `services__workers-api__http__0`. - -```ts -// Reach the workers plugin API with a generated typed client. -import { createServiceClient } from '@netscript/sdk/client'; -import { workersContract } from '@netscript/plugin-workers/contracts'; - -const workers = createServiceClient({ - contract: workersContract, - serviceName: 'workers-api', - routerName: 'workers', -}); - -// triggerJob is served over /api/rpc/* and returns { jobId, triggered }. -// Over REST the {id} path segment carries the target job; over RPC (no path -// segment) it travels in this input object. Either way it resolves to -// input.id — with no id at all the call fails with a 422 VALIDATION_ERROR. -const { jobId, triggered } = await workers.triggerJob({ - id: 'process-payment', - payload: { orderId: 'o_42', amountCents: 4999 }, -}); - -// triggerTask is the polyglot-task twin, returning { taskId, triggered }. -const task = await workers.triggerTask({ id: 'validate-payload', payload: {} }); -``` - -{{ comp.apiTable({ - caption: "Trigger procedures — served over /api/rpc/* (typed client) and /api/v1/... (OpenAPI/REST)", - rows: [ - { name: "triggerJob({ id, payload?, priority?, delay?, correlationId? })", type: "{ jobId, triggered }", desc: "POST /jobs/{id}/trigger — enqueue a run of the job named by the {id} path segment; resolves the jobId and a triggered flag." }, - { name: "triggerTask({ id, payload?, priority?, delay?, correlationId? })", type: "{ taskId, triggered }", desc: "POST /tasks/{id}/trigger — enqueue a run of the polyglot task named by the {id} path segment; resolves the taskId and a triggered flag." } - ] -}) }} - -{{ comp callout { type: "note", title: "Declare the plugin dependency with pluginReferences" } }} -When a Service depends on a plugin API — the workers plugin, say — declare it in that Service's -config section with pluginReferences?: string[], a list of the plugin resource names it -consumes (alongside the existing dependsOn). The Aspire helper generation wires those -references so the Service can discover and load the user jobs the plugin API exposes at runtime, and -so the typed client above resolves the plugin's injected URL -(services__workers-api__http__0). It is the config seam behind Service→plugin-API job -discovery. -{{ /comp }} - -{{ comp callout { type: "caution", title: "The {id} path segment is authoritative" } }} -triggerJob / triggerTask resolve the target id from -input.id. On the REST route (POST /jobs/{id}/trigger) oRPC merges the -{id} URL path segment into input.id, so the path is authoritative and a -body id is a redundant fallback that can never disagree with it; on the RPC transport -there is no path segment, so the id you pass in the input object is used directly. If -neither resolves an id, the handler short-circuits to a typed VALIDATION_ERROR -(HTTP 422) through the centralized validationFailed contract helper before any -KV write — it never persists an undefined-keyed run. Handle it as the contract's -typed error on the client, not a generic 500. -{{ /comp }} - -## Graceful shutdown - -Background runners must drain in flight work before they exit, or a redeploy loses jobs -mid-run. The `@netscript/plugin-workers-core/shutdown` subpath provides a `ShutdownManager` -that registers stoppable resources and stops them in priority order, with a timeout, when a -shutdown is requested. - -```ts -// plugins/workers/bin/with-shutdown.ts -import { ShutdownManager } from '@netscript/plugin-workers-core/shutdown'; -import { startWorkers } from '@netscript/plugin-workers-core'; - -const runtime = await startWorkers({ autoStart: true }); -const shutdown = new ShutdownManager({ timeoutMs: 10_000 }); - -// Register the runtime so a drain stops it gracefully (lower priority stops first). -shutdown.register({ id: 'workers-runtime', priority: 100, stop: (reason) => runtime.stop(reason) }); - -// Tie OS signals to the drain, then report what stopped / failed / timed out. -Deno.addSignalListener('SIGTERM', async () => { - const report = await shutdown.shutdown('SIGTERM'); - console.info('shutdown', report.state, { stopped: report.stopped, timedOut: report.timedOut }); - Deno.exit(0); -}); -``` - -{{ comp.apiTable({ - caption: "ShutdownManager — @netscript/plugin-workers-core/shutdown", - rows: [ - { name: "register(resource)", type: "void", desc: "Register a ShutdownResource ({ id, stop(reason?), priority? }) to be stopped on drain." }, - { name: "unregister(id)", type: "void", desc: "Remove a resource from the drain set." }, - { name: "shutdown(reason?, { timeoutMs })", type: "Promise", desc: "Stop registered resources in priority order; returns { state, stopped, failed, timedOut }." }, - { name: "waitForShutdown()", type: "Promise", desc: "Resolves once shutdown has started — await it in long-running loops." }, - { name: "createAbortController()", type: "AbortController", desc: "An AbortController that aborts when shutdown begins; pass its signal into in-flight async work." }, - { name: "state", type: "'running' | 'shutting-down' | 'stopped'", desc: "Current lifecycle state of the manager." } - ] -}) }} - -## Workers API & where jobs come from - -Once Aspire is up and the schema is wired, the workers API on **`:8091`** registers your job -by `id` and exposes HTTP endpoints to seed, trigger, and inspect runs. These are the -endpoints the CLI E2E suite validates live. - -{{ comp.apiTable({ - caption: "Workers API — port 8091 (full generated surface in the workers reference)", - rows: [ - { name: "GET /health", type: "liveness", desc: "Health probe for the workers API service." }, - { name: "GET /api/v1/workers/jobs", type: "list", desc: "All registered job definitions (id, name, topic) discovered from the jobs directories." }, - { name: "POST /api/v1/workers/jobs/{id}/trigger", type: "enqueue", desc: "Enqueue a run of the job with this id, passing a JSON payload body." }, - { name: "GET /api/v1/workers/executions?limit=10", type: "history", desc: "Recent executions and outcomes (KV-backed execution state)." }, - { name: "GET /api/v1/workers/tasks", type: "list", desc: "Task registry view (polyglot defineTask entries)." }, - { name: "POST /api/v1/workers/seed", type: "seed", desc: "Seed the workers store with the registered jobs." }, - { name: "GET /api/v1/workers/subscribe", type: "SSE", desc: "Server-sent-events stream of execution updates (KV-watch)." } - ] -}) }} - -{{ comp callout { type: "note", title: "Where jobs come from" } }} -Each handler file under your jobs directory (workers/jobs by default, -*.ts) is discovered and compiled into one generated registry at -.netscript/generated/plugin-workers/job-registry.ts, keyed by the source filename — -so workers/jobs/process-payment.ts registers as process-payment. Both -the :8091 API service and the background runner load that single registry at -startup: the API service registers the generated user job definitions before it serves, -so your jobs — not just the built-in workers-plugin-health-check — appear in -GET /api/v1/workers/jobs and resolve on trigger. Background execution runs from -plugins/workers/bin/combined.ts, a separate process from the API service — -the API enqueues, the runner executes. A missing generated registry is tolerated as an empty set, -so a fresh workspace boots before you author any job. Set WORKERS_CONCURRENCY on the -worker background process when you need a specific process pool size. Current Aspire metadata also -emits WORKER_CONCURRENCY, but the runtime entrypoint does not consume it; use -Tune the worker runtime for the mismatch details. -{{ /comp }} - -## Observability: real job traces out of the box - -Job-level observability is **not** a stub. The workers runtime instruments the -scheduler → queue → worker → subprocess path with real OpenTelemetry spans, and those traces -appear in the [Aspire dashboard](/explanation/aspire/) automatically once Aspire is up. - -{{ comp.apiTable({ - caption: "What the worker runtime traces automatically (framework layer — real today)", - rows: [ - { name: "Job dispatch + execution", type: "span", desc: "Each enqueued run gets a span with attributes, duration, status, and job.started / job.completed / job.failed / job.exception events." }, - { name: "Step + progress events", type: "event", desc: "job.step.* and job.progress (current / total / percentage) events are emitted on real job runs." }, - { name: "Subprocess trace continuation", type: "context", desc: "W3C traceparent / tracestate is propagated into the subprocess runner, so the child trace links back to the dispatching span." }, - { name: "Scheduler + cron spans", type: "span", desc: "Scheduler-start, schedule-job, dispatch, and cron-run spans cover the timer/cron path that fires scheduled jobs." } - ] -}) }} - -For spans you author *inside* a handler, import directly from **`@netscript/telemetry`** -(e.g. `@netscript/telemetry/instrumentation` for `withChildSpan`). These nest correctly under -the automatic dispatch span. See [Observability](/explanation/observability/) for the model and -[Add OpenTelemetry](/how-to/add-opentelemetry/) for the recipe. - -## Production notes - -{{ comp callout { type: "important", title: "Aspire first, then jobs" } }} -The workers plugin persists job definitions to Postgres and uses Deno KV for execution state, -so bring orchestration up before you exercise it. Step 1 is the database service; -step 2 is Aspire: cd aspire && aspire start provisions Postgres and -Redis, then netscript db init --name init / netscript db generate -wire the schema. Only after Aspire is up will :8091 resolve jobs and record -executions. See Database & migration. -{{ /comp }} - -{{ comp callout { type: "warning", title: "Drain on redeploy, or lose in-flight runs" } }} -A runner that exits without draining abandons jobs mid-execution. Wire a -ShutdownManager to SIGTERM (above), bound it with a -timeoutMs, and check report.timedOut in your logs. Pair the drain -with an idempotencyKey on enqueue so a retried delivery after a forced exit does -not double-charge. For the full recipe see -Tune the worker runtime. -{{ /comp }} - -{{ comp.badge({ status: "partial" }) }} - -{{ comp callout { type: "note", title: "Deferred: the scaffold createJobTools helper is a no-op" } }} -The scaffold's createJobTools(ctx) helper (in -plugins/workers/jobs/job-tools.ts) exposes log, -progress, and trace shims — but in the generated copy its -progress and trace methods are no-op stubs (only log writes to -the console), and it is not a published @netscript/plugin-workers-core -export. This is a known, tracked limitation with a fix planned, not a permanent design choice, -and it is only in that scaffold-facing helper: job dispatch and execution themselves -emit real spans automatically (above). For custom handler spans today, call -@netscript/telemetry helpers directly. Do not say "worker tracing is a no-op" — -that is false for the framework layer. - -{{ /comp }} - -## Reference → - -This hub is intentionally thin — the full generated API for `@netscript/plugin-workers` -(`defineJobHandler`, the job/task builders, the runtime, the shutdown manager, and every -exported type and subpath) lives in the reference. - -{{ comp.xref({ key: "ref:workers" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Learn — ERP Sync, lesson 02", - body: "Author an import job and trigger it over :8091 as part of the continuous ERP-sync tutorial narrative.", - href: "/tutorials/erp-sync/02-import-job/", - icon: "→" - }, - { - title: "Do — Tune the worker runtime", - body: "Pick the runner mode, set WORKERS_CONCURRENCY, choose a queue provider, and wire graceful shutdown.", - href: "/how-to/tune-worker-runtime/", - icon: "◆" - }, - { - title: "Look up — workers reference", - body: "The full generated deno doc API for @netscript/plugin-workers — every exported symbol, type, and subpath.", - href: "/reference/workers/", - icon: "≡" - }, - { - title: "Understand — Durability model", - body: "How jobs compose with sagas: a job publishes a message a saga consumes. The why behind the choreography.", - href: "/explanation/durability-model/", - icon: "◎" - } -] }) }} - -{{ comp.nextPrev({ - prev: { label: "Services & contracts", href: "/capabilities/services/" }, - next: { label: "Durable sagas", href: "/capabilities/durable-sagas/" } -}) }} diff --git a/docs/site/capabilities/database.md b/docs/site/capabilities/database.md index 6ff610d58..44ad28e09 100644 --- a/docs/site/capabilities/database.md +++ b/docs/site/capabilities/database.md @@ -1,392 +1,4 @@ --- -layout: layouts/base.vto -title: Database & Prisma -templateEngine: [vento, md] -prev: { label: "Durable streams", href: "/capabilities/streams/" } -next: { label: "KV, queues & cron", href: "/capabilities/kv-queues-cron/" } +layout: layouts/redirect.vto +redirectTo: /data-persistence/database/ --- - -# Database & Prisma - -NetScript's persistence layer is **Prisma 7 over a driver adapter** (Postgres is the recommended engine, -with MSSQL and MySQL adapters shipped alongside), generated for the Deno runtime and -provisioned for you by **Aspire**. Every plugin contributes its own `.prisma` models, which -are aggregated into a single generated client over one primary datasource. - -{{ comp.diagram({ - src: "/assets/diagrams/database-schema-aggregation.svg", - alt: "Per-plugin .prisma schema files (root ExampleRecord, workers, sagas, triggers, auth) under database/postgres/schema/plugins are aggregated by netscript db generate into a single Deno-runtime Prisma client and matching zod schemas; the client talks to one datasource through a selected driver adapter — postgres, mssql, or mysql.", - caption: "One generated client, many contributors. Each plugin's .prisma file is aggregated into the same datasource; an adapter (postgres / mssql / mysql) selects the driver underneath." -}) }} - -{{ comp callout { type: "important", title: "Aspire is step 2 — before any db command" } }} -Every netscript db command talks to Postgres through Aspire. -In the default layout Postgres is a container that Aspire provisions, so -cd aspire && aspire start must be up before you run -netscript db init. Run a db command against a stopped AppHost and -it fails with aspire start failed: … Project file does not exist. That is the -dependency, not a bug — bring orchestration up first, then run the db workflow from the -workspace root. See Orchestration with Aspire for why. -{{ /comp }} - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for the database layer when you need durable, relational state — -records that survive restarts, are queried with a typed client, and are shared across your -service and its plugins. The engine is polyglot: pick it at scaffold time with ---db — Postgres (the recommended engine; or mysql / mssql / -sqlite). For ephemeral or high-throughput state (counters, locks, -work queues, scheduled triggers) reach for KV, -queues & cron instead; the scaffold uses both — Postgres for records, Redis/KV for -execution state. Postgres can also back the work queue itself — see the Postgres queue backend below. -{{ /comp }} - -## Learn → / Do → - -- **Learn** — the [Team Workspace tutorial, step 03](/tutorials/workspace/03-workspace-data/) - wires workspace data through the database from scratch. -- **Do** — the [Use a second database](/how-to/use-a-second-database/) recipe adds a second - adapter-backed datasource (MSSQL or MySQL) beside the primary Postgres. - -## How persistence is wired - -The scaffold engine is chosen with the `--db` flag — -`netscript init my-app --db postgres|mysql|mssql|sqlite`. Postgres is the recommended -engine (and what every tutorial uses); `mysql`, `mssql`, and `sqlite` are first-class -alternatives. `postgres` / `mysql` / `mssql` provision an Aspire container; `sqlite` is -file-backed with **no** Aspire container resource. The `--db postgres` scaffold -lays down a `database/postgres/` workspace (a different engine lays down -`database//`). A few facts are worth internalizing before you run anything, because -they differ from a typical single-file Prisma setup. - -```text -my-app/ -├── appsettings.json # NetScript.Databases.postgres — Engine/Mode/DatabaseName (the real config) -├── netscript.config.ts # databases.config is intentionally EMPTY here -└── database/postgres/ - ├── schema/ - │ ├── schema.prisma # root: generator client (runtime="deno"), generator zod, datasource db - │ └── plugins/ # per-plugin .prisma models aggregated here - │ ├── workers/workers.prisma - │ ├── sagas/sagas.prisma - │ ├── triggers/triggers.prisma - │ └── auth/auth.prisma # appears once the auth plugin is added (better-auth backend) - ├── prisma.config.ts # schema dir + migrations path + datasource url - ├── scripts/ # migrate.ts, seed.ts, generate-zod.ts, fix-zod-imports.ts, … - └── schema/.generated/ # appears after `db generate`: Prisma client + zod schemas -``` - -- **The datasource is driven by `appsettings.json`, not `netscript.config.ts`.** In the - scaffold, `netscript.config.ts` has `databases: { config: [] }` (empty). The connection - details live in `appsettings.json` under `NetScript.Databases.postgres` — - `Engine: "Postgres"`, `Mode: "Container"`, a generated `DatabaseName`, and a persistent - `DataPath` of `.data/postgres`, with `PrimaryDatabase: "postgres"`. Aspire reads that - block to provision the container. -- **One datasource, many contributors.** There is a single primary Postgres datasource. - Each first-party plugin ships its own `database/.prisma`, and those models are - aggregated under `database/postgres/schema/plugins//`. After adding `workers`, - `sagas`, and `triggers`, their tables (for example `model JobDefinition` from workers) - join the root `ExampleRecord` model in the same database — one `db init` / `db generate` - cycle covers the app and its plugins together. -- **The client is generated for Deno.** `schema.prisma` sets `generator client` with - `runtime = "deno"` and `output = "./.generated"`, plus a `generator zod` - (prisma-zod-generator) emitting matching zod schemas to `./.generated/zod`. You get a - type-safe client *and* runtime validation schemas from one generate. - -## Headline API: a model and a query - -Define models in Prisma schema files, then query them through the generated Deno client. -The root `schema.prisma` ships a single sample model, `ExampleRecord`; plugin schema files -add their own. The tabs below show a model alongside the typed query you'd write against -the generated client. - -{{ comp.tabbedCode({ tabs: [ - { - label: "schema.prisma", - lang: "prisma", - code: "// database/postgres/schema/schema.prisma\ngenerator client {\n provider = \"prisma-client\"\n output = \"./.generated\"\n runtime = \"deno\"\n}\n\ngenerator zod {\n provider = \"prisma-zod-generator\"\n output = \"./.generated/zod\"\n}\n\ndatasource db {\n provider = \"postgresql\"\n // The datasource URL is resolved by prisma.config.ts (POSTGRES_URI / DATABASE_URL),\n // not declared here.\n}\n\nmodel ExampleRecord {\n id String @id @default(uuid())\n name String\n createdAt DateTime @default(now())\n updatedAt DateTime @updatedAt\n}" - }, - { - label: "query.ts", - lang: "ts", - code: "// Import the Deno-runtime client generated by `netscript db generate`.\nimport { PrismaClient } from '../database/postgres/schema/.generated/client.server.ts';\n\nconst prisma = new PrismaClient();\n\n// Insert a record, then read it back — fully typed off the model above.\nconst created = await prisma.exampleRecord.create({\n data: { name: 'first' },\n});\n\nconst recent = await prisma.exampleRecord.findMany({\n orderBy: { createdAt: 'desc' },\n take: 10,\n});\n\nconsole.log(created.id, recent.length);" - } -] }) }} - -{{ comp callout { type: "note", title: "Generate after every schema change" } }} -The generated client under database/postgres/schema/.generated/ is a build -artifact. Edit a .prisma file and your code is still typed against the -old shape until you re-run netscript db generate. Generate -after every schema change, and re-run netscript db migrate to create and apply -the matching migration. -{{ /comp }} - -## The database workflow - -With `aspire start` up (Postgres reachable), run the public `netscript db` commands from the -workspace root in this order. The first three are the create-and-fill cycle; `status` -confirms it landed and `migrate` evolves the schema later. - -{{ comp.tabbedCode({ tabs: [ - { - label: "1 · init", - lang: "bash", - code: "# Create the first migration from the current Prisma schema and apply it.\n# --name labels the migration directory (here: a migration called \"init\").\nnetscript db init --name init" - }, - { - label: "2 · generate", - lang: "bash", - code: "# Generate the Deno-runtime Prisma client (and the zod schemas) into\n# database/postgres/schema/.generated so your code can import typed models.\nnetscript db generate" - }, - { - label: "3 · seed", - lang: "bash", - code: "# Run the workspace seed scripts (database/postgres/scripts/seed.ts)\n# to populate baseline rows.\nnetscript db seed" - }, - { - label: "4 · status", - lang: "bash", - code: "# Show migration / tooling status: which migrations are applied and\n# whether the database is in sync with the schema.\nnetscript db status" - }, - { - label: "5 · migrate", - lang: "bash", - code: "# Later, when the schema evolves: create + apply the next migration.\n# Edit a .prisma file, then:\nnetscript db migrate --name add_orders\nnetscript db generate # re-generate the client against the new shape" - } -] }) }} - -## Plugins contribute models - -Adding a first-party plugin installs its Prisma models to the *same* datasource — there is no -second database. The aggregated tables land under -`database/postgres/schema/plugins//` and migrate in the same `db init`/`migrate` -cycle as your app. - -{{ comp.apiTable({ - caption: "Plugin-contributed Prisma models (same primary datasource)", - rows: [ - { name: "workers", type: "@netscript/plugin-workers", desc: "Job and schedule definitions (for example JobDefinition) backing the durable background-job runtime." }, - { name: "sagas", type: "@netscript/plugin-sagas", desc: "Durable saga state — saga_runtime_state / saga_runtime_transition / saga_runtime_correlation — when the saga store backend is prisma (vs kv). See Durable sagas." }, - { name: "triggers", type: "@netscript/plugin-triggers", desc: "Trigger definitions and delivery bookkeeping for the inbound webhook/ingest runtime." }, - { name: "auth", type: "@netscript/plugin-auth", desc: "auth.prisma: auth_users / auth_sessions / auth_accounts / auth_verifications — used by the better-auth backend. (kv-oauth stores sessions in KV; WorkOS is stateless.)" } - ] -}) }} - -{{ comp callout { type: "note", title: "auth.prisma is backend-specific" } }} -The auth plugin contributes -auth.prisma with auth_users, auth_sessions, -auth_accounts, and auth_verifications. Those tables are the -persistence shape for the better-auth backend. The default -kv-oauth backend keeps sessions in KV and -WorkOS is stateless (sealed cookie), so they do not depend on these tables. -Run the auth migration only when you select the better-auth backend via -NETSCRIPT_AUTH_BACKEND. -{{ /comp }} - -{{ comp callout { type: "tip", title: "Postgres can also back the work queue" } }} -Persistence is not the only place Postgres shows up. The queue runtime now ships a -PostgreSQL backend alongside Deno KV, Redis, and RabbitMQ — a durable, -row-claimed (FOR UPDATE SKIP LOCKED) queue table in the same Postgres you -already provision. It is selectable only via an explicit -provider: 'postgres' (auto-discovery never picks it). See -KV, queues & cron for -connection.postgres.{url, tableName} and the full backend matrix. -{{ /comp }} - -## Adapter selection — postgres, mssql, mysql - -Prisma 7 talks to the database through a **driver adapter**. NetScript wraps each driver in -a small `DatabaseAdapter` that gives every backend the same lifecycle surface -(`getDriverAdapter` → `setClient` → `connect`/`disconnect`/`healthCheck`/`getStatus`). The -recommended scaffold is Postgres, but MSSQL and MySQL ship as sub-exports so you can target — or -add — a second engine without changing how your code consumes the client. - -The shared options shape comes from `DatabaseConnectionOptions` -(`@netscript/database/ports`): every adapter accepts either a `connectionString` or the -structured `host` / `port` / `database` / `username` / `password` / `ssl` / `poolSize` / -`timeout` parts, plus a few engine-specific extras. - -{{ comp.apiTable({ - caption: "Driver adapters (each implements DatabaseAdapter)", - rows: [ - { name: "adapters/postgres", type: "createPostgresAdapter(opts)", desc: "PostgreSQL via @prisma/adapter-pg. Re-exported from the @netscript/database/adapters barrel. PostgresConnectionOptions adds schema and applicationName. Builds a connection string under the hood." }, - { name: "adapters/mssql", type: "createMssqlAdapter(opts)", desc: "SQL Server via @prisma/adapter-mssql. Import from @netscript/database/adapters/mssql (sub-export — NOT in the barrel). MssqlConnectionOptions adds instanceName, trustServerCertificate, encrypt, integratedSecurity, applicationName, connectTimeout, requestTimeout. Accepts ADO.NET-style connection strings." }, - { name: "adapters/mysql", type: "createMysqlAdapter(opts)", desc: "MySQL 8.x / MariaDB via @netscript/prisma-adapter-mysql (a native-Deno driver). Import from @netscript/database/adapters/mysql (sub-export). MysqlConnectionOptions adds charset, timezone, connectionLimit, multipleStatements." } - ] -}) }} - -{{ comp.apiTable({ - caption: "DatabaseAdapter surface (every adapter)", - rows: [ - { name: "provider", type: "'postgres' | 'mssql' | 'mysql'", desc: "Readonly provider identity used in status reporting." }, - { name: "getDriverAdapter()", type: "→ driver adapter", desc: "Returns the Prisma driver adapter to pass as `new PrismaClient({ adapter })`. Lazily constructs it from the options." }, - { name: "setClient(client)", type: "(client) => void", desc: "Hand the constructed PrismaClient back to the adapter so connect / health / raw calls have a client to use." }, - { name: "getClient()", type: "→ PrismaClient", desc: "Return the configured Prisma client. Throws if setClient() has not been called — see the callout below." }, - { name: "connect() / disconnect()", type: "→ Promise", desc: "Explicit $connect / $disconnect. Prisma auto-connects on first query, so these are optional." }, - { name: "healthCheck()", type: "→ Promise", desc: "Lightweight `SELECT 1` probe; true when the connection answers." }, - { name: "getStatus()", type: "→ Promise", desc: "Snapshot: { connected, provider, database, host, lastConnected, error }." }, - { name: "executeRaw() / executeRawUnsafe()", type: "(query, ...params) => Promise", desc: "Raw query escape hatches routed through the configured client." } - ] -}) }} - -{{ comp callout { type: "important", title: "getClient() needs setClient() first" } }} -The adapter does not own the PrismaClient — you construct it. The flow is -always: build the adapter, call getDriverAdapter(), pass that to -new PrismaClient({ adapter }), then call adapter.setClient(client). -Only then does adapter.getClient() (and connect / -healthCheck / executeRaw) work — call getClient() -before setClient() and it throws -"… client not initialized. Call setClient() …". -{{ /comp }} - -## Tracing — Prisma OpenTelemetry spans - -Prisma query spans are wired through a lightweight tracing helper exported from -`@netscript/database/tracing`. It is a drop-in for `@prisma/instrumentation` that avoids the -CJS-heavy `@opentelemetry/instrumentation` dependency (which breaks Deno bundle/compile). -Call `enablePrismaTracing()` **once, before you construct any Prisma client**, and engine -spans dispatch into your OpenTelemetry tracer with the standard W3C `traceparent`. - -{{ comp.apiTable({ - caption: "@netscript/database/tracing — PrismaTracingConfig + hooks", - rows: [ - { name: "enablePrismaTracing(config?)", type: "(PrismaTracingConfig) => void", desc: "Register the tracing helper. Call once, before creating any PrismaClient." }, - { name: "config.tracerProvider", type: "PrismaTracingProvider", desc: "Tracer provider to use. Defaults to the globally registered OpenTelemetry provider." }, - { name: "config.ignoreSpanTypes", type: "(string | RegExp)[]", desc: "Span-name patterns to drop (string match or regex). Defaults to []." }, - { name: "disablePrismaTracing()", type: "() => void", desc: "Clear the global tracing helper." }, - { name: "isPrismaTracingEnabled()", type: "() => boolean", desc: "True when a tracing helper is currently registered." }, - { name: "enableInstrumentation()", type: "() => boolean (@netscript/database)", desc: "Convenience toggle from the package root: enables Prisma OTEL instrumentation only when OTEL_DENO=true; returns whether it was enabled." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Enable tracing", - lang: "ts", - code: "// services/orders/src/tracing.ts\nimport { enablePrismaTracing } from '@netscript/database/tracing';\n\n// Call ONCE, before constructing any PrismaClient. Engine spans then dispatch\n// into the globally registered OpenTelemetry tracer with a W3C traceparent.\nenablePrismaTracing({\n // tracerProvider defaults to the global provider; override only if you manage\n // your own. ignoreSpanTypes drops noisy internal spans by name or regex.\n ignoreSpanTypes: ['prisma:client:serialize', /detect_platform/],\n});\n\n// ... now construct the client (see the adapter example below).\nimport { PrismaClient } from '../database/postgres/schema/.generated/client.server.ts';\nconst prisma = new PrismaClient();" - }, - { - label: "Root toggle (OTEL_DENO)", - lang: "ts", - code: "// Alternatively, gate instrumentation on the OTEL_DENO flag from the package root.\nimport { enableInstrumentation } from '@netscript/database';\n\n// Returns true only when OTEL_DENO=true and wiring succeeded.\nconst tracing = enableInstrumentation();\nconsole.log('prisma tracing on:', tracing);" - } -] }) }} - -## Runnable example: wiring a second database (MySQL) - -The adapters let you stand up a **second** datasource beside the primary Postgres — for -example a reporting MySQL instance — without leaving the typed-client model. The pattern is -the same for every engine: create the adapter, pass `getDriverAdapter()` into a -`PrismaClient`, then `setClient()` so the adapter can manage lifecycle and health. Swap the -import to `adapters/mssql` and `createMssqlAdapter` for SQL Server — the surface is -identical. The MySQL adapter is a sub-export (not in the `adapters` barrel), so import it -from its own path. - -{{ comp.tabbedCode({ tabs: [ - { - label: "MySQL — createMysqlAdapter", - lang: "ts", - code: "// services/reporting/src/db.ts\n// MySQL ships as a sub-export — import from /adapters/mysql, not the barrel.\nimport { createMysqlAdapter } from '@netscript/database/adapters/mysql';\nimport { PrismaClient } from '../database/mysql/schema/.generated/client.server.ts';\n\n// 1) Build the adapter from structured parts (or pass { connectionString }).\nconst adapter = createMysqlAdapter({\n host: Deno.env.get('MYSQL_HOST') ?? 'localhost',\n port: 3306,\n database: 'reporting',\n username: Deno.env.get('MYSQL_USER') ?? 'root',\n password: Deno.env.get('MYSQL_PASSWORD'),\n ssl: false,\n});\n\n// 2) Pass the driver adapter into Prisma, then hand the client back.\nexport const reporting = new PrismaClient({ adapter: adapter.getDriverAdapter() });\nadapter.setClient(reporting);\n\n// 3) Now adapter lifecycle + health work off the same client.\nawait adapter.connect();\nconst ok = await adapter.healthCheck(); // SELECT 1\nconsole.log('mysql healthy:', ok, await adapter.getStatus());" - }, - { - label: "MSSQL — createMssqlAdapter", - lang: "ts", - code: "// services/reporting/src/db.ts — SQL Server variant (same shape)\nimport { createMssqlAdapter } from '@netscript/database/adapters/mssql';\nimport { PrismaClient } from '../database/mssql/schema/.generated/client.server.ts';\n\nconst adapter = createMssqlAdapter({\n host: Deno.env.get('MSSQL_SERVER') ?? 'localhost',\n port: 1433,\n database: 'reporting',\n username: 'sa',\n password: Deno.env.get('MSSQL_PASSWORD'),\n // local-dev TLS knobs; tighten for production\n encrypt: true,\n trustServerCertificate: true,\n});\n\nexport const reporting = new PrismaClient({ adapter: adapter.getDriverAdapter() });\nadapter.setClient(reporting);\nawait adapter.connect();" - }, - { - label: "Env config helper", - lang: "ts", - code: "// Prefer env-driven config? Each engine ships a getter that reads structured\n// vars (MYSQL_HOST/…) and falls back to a connection-string env var.\nimport { createMysqlAdapter } from '@netscript/database/adapters/mysql';\nimport { getMysqlConfig } from '@netscript/database/adapters/mysql';\n\n// getMysqlConfig() → MysqlAdapterConfig from MYSQL_* or MYSQLDB_URI / DATABASE_URL.\nconst cfg = getMysqlConfig();\nconst adapter = createMysqlAdapter({\n host: cfg.hostname,\n port: cfg.port,\n database: cfg.db,\n username: cfg.username,\n password: cfg.password,\n});\n// (getMssqlConfig / getPostgres equivalents exist per adapter.)" - } -] }) }} - -{{ comp callout { type: "note", title: "A second datasource is a second schema dir" } }} -A second engine is a separate Prisma schema workspace (for example -database/mysql/schema/) with its own generate output and its own -migrations — it does not merge into the Postgres aggregation above. Generate and -migrate each datasource independently. The full step-by-step lives in the -Use a second database recipe. -{{ /comp }} - -## Endpoints & ports - -The database itself isn't an HTTP service you call — it's a Postgres container Aspire -provisions and the generated Prisma client connects to. The relevant surfaces are the -Aspire resources and the connection-string env vars the workspace resolves. - -{{ comp.apiTable({ - caption: "Database surface (provisioned by Aspire)", - rows: [ - { name: "postgres", type: "Aspire resource", desc: "The Postgres container Aspire provisions from appsettings.json NetScript.Databases.postgres. Watch it go green in the dashboard." }, - { name: "redis", type: "Aspire resource", desc: "Redis cache — the default `--cache-backend`; Redis-compatible — backing KV/queues. A separate concern from Postgres; see KV, queues & cron. (`garnet` and `deno-kv` are alternative backends.)" }, - { name: "https://localhost:18888", type: "dashboard", desc: "Aspire dashboard (token printed by `aspire start`) — confirm the postgres resource is healthy." }, - { name: "POSTGRES_URI / DATABASE_URL", type: "env", desc: "Connection string resolved by prisma.config.ts and normalized to a URL. Set these yourself under --no-aspire." } - ] -}) }} - -{{ comp callout { type: "warning", title: "Production pitfalls" } }} -
    -
  • Forgetting Aspire. The most common failure: running netscript db init with no aspire start up. Start orchestration first, from the aspire/ folder.
    • -
  • Wrong directory. Run aspire start from aspire/, but run the netscript db commands from the workspace root (or pass --project-root).
    • -
  • Stale client after a schema change. Editing a .prisma file without re-running netscript db generate leaves your code typed against the old model.
    • -
  • Calling getClient() before setClient(). A second-database adapter throws until you hand it the constructed PrismaClient via setClient().
    • -
  • Tracing wired too late. enablePrismaTracing() must run before the first PrismaClient is constructed, or early query spans are lost.
    • -
  • Docker not running. Aspire provisions Postgres and Redis as containers; if Docker/Podman is down the postgres resource never goes green.
    • -
-{{ /comp }} - -{{ comp callout { type: "note", title: "MSSQL & MySQL are sub-exports" } }} -Postgres is the recommended engine and lives in the @netscript/database/adapters barrel. -The SQL Server and MySQL adapters are imported from their -own sub-paths — @netscript/database/adapters/mssql and -@netscript/database/adapters/mysql — so a Postgres-only app never pulls in the -heavier mssql / native-Deno MySQL drivers. The MySQL driver itself lives in the -standalone @netscript/prisma-adapter-mysql -package. -{{ /comp }} - -## Where to go next - -This hub is intentionally thin — the full generated API lives in the reference. Pick the -lane that matches what you're doing. - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Workspace data (Track B 03)", - body: "Guided tutorial: wire workspace data through Prisma, end to end, in the Team Workspace track.", - href: "/tutorials/workspace/03-workspace-data/", - icon: "→" - }, - { - title: "Do — Use a second database", - body: "Task recipe: add an MSSQL or MySQL datasource beside Postgres with a driver adapter.", - href: "/how-to/use-a-second-database/", - icon: "◆" - }, - { - title: "Do — Database & migration", - body: "Task recipe: bring up Postgres with Aspire, then run db init → generate → seed → status → migrate.", - href: "/how-to/database-migration/", - icon: "◆" - }, - { - title: "Look up — @netscript/database reference", - body: "The full generated API: ports, the postgres/mssql/mysql adapters, extensions, and tracing.", - href: "/reference/database/", - icon: "≡" - }, - { - title: "Understand — Orchestration with Aspire", - body: "Why the AppHost (aspire/apphost.mts) provisions Postgres and Redis, and how the resource graph fits together.", - href: "/explanation/aspire/", - icon: "◎" - }, - { - title: "MySQL — prisma-adapter-mysql", - body: "The native-Deno MySQL Prisma driver adapter surface, generated.", - href: "/reference/prisma-adapter-mysql/", - icon: "→" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Durable streams", href: "/capabilities/streams/" }, next: { label: "KV, queues & cron", href: "/capabilities/kv-queues-cron/" } }) }} diff --git a/docs/site/capabilities/durable-sagas.md b/docs/site/capabilities/durable-sagas.md index 0d3ec1a1b..06a57668f 100644 --- a/docs/site/capabilities/durable-sagas.md +++ b/docs/site/capabilities/durable-sagas.md @@ -1,508 +1,4 @@ --- -layout: layouts/base.vto -title: Durable sagas -templateEngine: [vento, md] -prev: { label: "Background jobs", href: "/capabilities/background-jobs/" } -next: { label: "Triggers & ingress", href: "/capabilities/triggers/" } +layout: layouts/redirect.vto +redirectTo: /durable-workflows/sagas/ --- - -# Durable sagas - -A saga is the answer to the question every retry loop dodges: *what happens between -step three and step four when the process dies?* In NetScript a saga is an explicit, -message-driven state machine — you declare the state it carries, the messages it -reacts to, and the effects each handler emits. It is authored with a single fluent -builder, persisted to a durable store, and served by the **sagas plugin** on port -`:8092`. The model is closer to Temporal than to a job queue, but it lives in plain -TypeScript inside your workspace — no separate cluster to operate. - -{{ comp.diagram({ - src: "/assets/diagrams/saga-state-machine.svg", - alt: "Saga state machine: an event advances the saga through typed states, a compensation branch unwinds a failed step, and every transition checkpoints to a kv or prisma store.", - caption: "A saga is a state machine. Each inbound message runs a handler that mutates typed state and returns an effect ledger (advance, complete, fail, compensate); the runtime checkpoints state between messages to the durable store (kv or prisma) so an instance survives a crash." -}) }} - -This is the third capability in the continuous-app thread. The -[background jobs](/capabilities/background-jobs/) capability ended with a job, -`create-user-settings`, that publishes a `UserSettingsCreated` message. Here that -message stops being fire-and-forget: a saga **consumes it**, advances its own state, -and emits a `sagaComplete(...)` effect that the runtime records as a first-class -outcome — and persists durably so the instance survives a process restart. - -{{ comp callout { type: "note", title: "Where it lives" } }} -The plugin is installed at plugins/sagas/ and referenced by -netscript.config.ts as ./plugins/sagas/mod.ts. The fluent -builder and effect helpers come from @netscript/plugin-sagas-core; the -config-time companion (defineSagaConfig) comes from -@netscript/plugin-sagas-core/config; the durable runtime factory -(createDurableSagaRuntime) and the HTTP publisher -(createSagaPublisher) come from the -@netscript/plugin-sagas/runtime subpath. Add it to a workspace with -the public package install flow (netscript plugin install @netscript/plugin-sagas). -For local-source contributor samples inside this monorepo, run -deno run -A packages/cli/bin/netscript-dev.ts plugin install saga --name sagas --samples. -{{ /comp }} - -## What it is - -A saga is a **durable, message-driven state machine**: a typed state object plus a set -of handlers, one per message type, that advance that state and return an effect ledger. -The runtime persists state between every message, so an instance that is mid-flight when -the process restarts resumes exactly where it left off. Unlike a background job — a -single idempotent unit — a saga correlates many messages into one long-running instance -and gives compensation (the *undo* of an already-applied step) a first-class place in -the model. Read the conceptual companion, the -{{ comp.xref({ key: "explain:durable-workflows" }) }}, for why state must outlive the -process and how effect-based outcomes differ from a retry loop. - -## Learn → · Do → - -{{ comp.featureGrid({ - columns: 2, - items: [ - { icon: "◆", title: "Learn — Build the checkout saga", body: "The storefront tutorial rung that adds the sagas plugin, correlates the order events, and compensates a failed payment step.", href: "/tutorials/storefront/04-checkout-saga/" }, - { icon: "→", title: "Do — Wire a durable workflow", body: "Add the sagas plugin to a running app, consume a cross-plugin message, and emit sagaComplete end to end.", href: "/tutorials/storefront/04-checkout-saga/" } - ] -}) }} - -{{ comp.featureGrid({ - columns: 3, - items: [ - { icon: "◆", title: "Fluent builder", body: "defineSaga(id).durability().state().on().compensate().build() — id, durability tier, typed state, message handlers, compensations, then build(). One chain, fully type-checked." }, - { icon: "▣", title: "Durable store backend", body: "Runtime state persists to kv or prisma, chosen by NETSCRIPT_SAGA_STORE / appsettings. createDurableSagaRuntime({ backend, prisma }) owns the resources." }, - { icon: "≋", title: "Effect-based outcomes", body: "Every handler returns an array of effects — sagaComplete, sagaFail, sagaCompensate, send, schedule, spawn are named outcomes returned from handlers, never a fall-through." }, - { icon: "⊡", title: "Served on :8092", body: "An oRPC API lists registered sagas, inspects running instances, publishes messages, and streams activity over SSE." }, - { icon: "⇄", title: "Cross-plugin choreography", body: "The workers create-user-settings job publishes UserSettingsCreated; this saga consumes it — one message crossing the plugin boundary, type-checked on both sides." }, - { icon: "◷", title: "Crash-survivable", body: "State checkpoints between messages, so an instance picks up exactly where it left off after a restart. That survival is the entire point of a saga." } - ] -}) }} - -## Minimal example — an order saga with compensation - -The runnable shape you grow into: a real state type, multiple `.on(...)` handlers, and a -`.compensate(...)` branch that unwinds an already-charged payment when fulfillment fails. -The first handler reserves stock and emits a `send(...)` effect to charge payment; the -failure path returns `sagaCompensate(...)` so the engine routes the matching compensation -handler, which refunds and then fails the instance. - -```ts -// plugins/sagas/order-saga.ts -import { - defineSaga, - sagaComplete, - sagaCompensate, - sagaFail, - send, -} from '@netscript/plugin-sagas-core'; - -// The state is the source of truth that survives across messages and crashes. -type OrderState = Readonly<{ - status: 'awaiting-payment' | 'awaiting-fulfillment' | 'completed' | 'refunded'; - orderId?: string; - paymentId?: string; -}>; - -type OrderPlaced = Readonly<{ orderId: string }>; -type PaymentCaptured = Readonly<{ orderId: string; paymentId: string }>; -type FulfillmentFailed = Readonly<{ orderId: string; reason: string }>; - -export const orderSaga = defineSaga('order-saga') - // Durability tier: where the runtime checkpoints state between messages. - .durability('t1') - // Typed initial state, seeded once at instance creation. - .state({ status: 'awaiting-payment' }) - // Handler 1: an order was placed — record it and ask the payment service to charge. - .on<'OrderPlaced', OrderPlaced>('OrderPlaced', (saga, event) => { - saga.state = { ...saga.state, orderId: event.payload.orderId }; - // `send` is an effect: dispatch a command to another target, not a direct call. - return [send({ kind: 'service', id: 'payments' }, { orderId: event.payload.orderId }, {})]; - }) - // Handler 2: payment captured — advance toward fulfillment. - .on<'PaymentCaptured', PaymentCaptured>('PaymentCaptured', (saga, event) => { - saga.state = { - ...saga.state, - status: 'awaiting-fulfillment', - paymentId: event.payload.paymentId, - }; - return []; - }) - // Handler 3: fulfillment failed AFTER we charged — request the compensation branch. - .on<'FulfillmentFailed', FulfillmentFailed>('FulfillmentFailed', (saga, event) => { - return [sagaCompensate({ type: 'FulfillmentFailed', payload: event.payload }, event.payload.reason)]; - }) - // Compensation: undo the already-applied payment, then fail the instance. - .compensate<'FulfillmentFailed', FulfillmentFailed>('FulfillmentFailed', (saga, event) => { - saga.state = { ...saga.state, status: 'refunded' }; - return [ - send({ kind: 'service', id: 'payments' }, { refund: saga.state.paymentId }, {}), - sagaFail(`order ${event.payload.orderId} unfulfilled: ${event.payload.reason}`), - ]; - }) - .build(); - -export default orderSaga; -``` - -{{ comp callout { type: "important", title: "Compensation: a .compensate() branch OR an effect" } }} -There are two ways to compensate, and they work together. The builder -exposes a first-class .compensate(eventType, handler) that registers an undo -handler keyed by message type. A running handler routes into it by returning the -sagaCompensate(message, reason?) effect. So a saga's whole lifecycle — -advance, sagaComplete, sagaFail, sagaCompensate, -send, schedule, spawn — is named outcomes -returned from handlers, never a fall-through or an unhandled throw. The undo logic -itself lives in the .compensate() handler, not inline in the forward path. -{{ /comp }} - -## Key types first - -Before the options, the primary interfaces the DSL works in. `SagaState` is the base -shape your typed state must satisfy; `SagaContext` is the read-only context passed to -every handler; `SagaDefinition` is the frozen object `build()` returns and the runtime -registers. - -{{ comp.apiTable({ - caption: "Primary saga types (@netscript/plugin-sagas-core)", - rows: [ - { name: "SagaState", type: "Readonly>", desc: "Base state shape every saga's typed state must extend. Your State type is intersected with this." }, - { name: "SagaMessage", type: "{ type, payload, correlationKey?, idempotencyKey?, concurrencyKey?, occurredAt?, traceparent? }", desc: "The event or command delivered to a handler. `type` discriminates; `payload` is your typed body." }, - { name: "SagaContext", type: "{ sagaId, instanceId, correlationKey, state, message, attempt, now, traceparent? }", desc: "Read-only handler context. `now` is the injected clock; `attempt` is the retry count; trace fields carry W3C context." }, - { name: "SagaHandler", type: "(saga, event, context) => readonly CascadedMessage[]", desc: "A synchronous handler: it mutates saga.state and RETURNS an effect ledger. No async, no direct I/O." }, - { name: "SagaDefinition", type: "Readonly<{ id, durability, initialState, handlers, compensations, correlations, retry?, concurrency?, schedule? }>", desc: "The frozen definition build() produces. Registered into the runtime and into KV under ['saga','registry', id]." }, - { name: "SAGA_DURABILITY_TIERS", type: "readonly ['t1','t2','t3']", desc: "The durability tiers a definition may declare via .durability(tier). 't1' is the scaffolded default." } - ] -}) }} - -## The builder API - -A saga is built with `defineSaga(id)` and a typestate chain: `.state()` must come before -any handler, and `.build()` requires at least one handler. These are the methods on the -returned `SagaBuilder`. - -{{ comp.apiTable({ - caption: "defineSaga(id) — SagaBuilder methods", - rows: [ - { name: ".durability(tier)", type: "SagaDurabilityTier ('t1' | 't2' | 't3')", desc: "Set the persistence tier the runtime checkpoints state to. Defaults to t1. Distinct from the kv/prisma store backend (see below)." }, - { name: ".state(initial)", type: "S extends SagaState", desc: "Declare the typed initial state, seeded once at instance creation. MUST be called before any handler (typestate-enforced)." }, - { name: ".on(type, handler)", type: "SagaHandler", desc: "Register a forward handler for a message type. Receives (saga, event, context), mutates saga.state, returns an effect ledger." }, - { name: ".compensate(type, handler)", type: "SagaHandler", desc: "Register a compensation (undo) handler keyed by message type. Routed when a handler returns sagaCompensate(...)." }, - { name: ".correlate(rule)", type: "SagaCorrelation", desc: "Extract a correlation key from an incoming message so it routes to the right running instance." }, - { name: ".concurrency(opts)", type: "{ limit: number; key?: (m) => string }", desc: "Bound how many messages run at once, optionally per derived key. Overlapping publishes for one key are rejected." }, - { name: ".schedule(cron)", type: "string", desc: "Attach a cron expression that ticks the saga definition on a schedule." }, - { name: ".onSignal(signal, handler)", type: "SignalDefinition, handler", desc: "Register a reserved signal handler (defineSignal). Runtime dispatch is deferred in the alpha." }, - { name: ".onQuery(query, handler)", type: "QueryDefinition, handler", desc: "Register a reserved synchronous read-only query handler (defineQuery). Promises are rejected at type level." }, - { name: ".build()", type: "=> SagaDefinition", desc: "Freeze the chain into a SagaDefinition. Requires at least one .on(...) handler (typestate-enforced)." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — minimal saga", - lang: "ts", - code: "// plugins/sagas/.ts\nimport { defineSaga, sagaComplete } from '@netscript/plugin-sagas-core';\n\ntype State = Readonly<{ status: string; processedAt?: string }>;\n\n// Build a saga: id -> durability tier -> typed state -> message handlers -> build().\nexport const userSettingsSaga = defineSaga('user-settings-saga')\n .durability('t1')\n .state({ status: 'pending' })\n .on<'UserSettingsCreated', { userId: string }>(\n 'UserSettingsCreated',\n (saga, event, context) => {\n // Advance the saga's own state, then emit completion as a recorded outcome.\n saga.state = {\n ...saga.state,\n status: 'completed',\n processedAt: context.now.toISOString(),\n };\n return [sagaComplete({\n userId: event.payload.userId,\n processedAt: context.now.toISOString(),\n })];\n },\n )\n .build();\n\nexport default userSettingsSaga;" - }, - { - label: "Config-time companion", - lang: "ts", - code: "// netscript.config.ts (excerpt)\nimport { defineSagaConfig } from '@netscript/plugin-sagas-core/config';\n\n// The config-time entry the scaffolder + CLI read. SEPARATE from the runtime\n// definition that defineSaga(...).build() produces.\nexport const orderSagaEntry = defineSagaConfig('order-saga', './plugins/sagas/order-saga.ts')\n .name('Order saga')\n .description('Reserve, charge, fulfill — with a compensation branch.')\n .topic('orders')\n .tags(['orders', 'checkout'])\n .build();" - } -] }) }} - -## Effect helpers - -A handler's only side effect is the array of **cascaded messages** it returns. These -helpers (`@netscript/plugin-sagas-core`) construct the named effects. Every kind in -`CASCADED_MESSAGE_KINDS` (`send | scheduled | spawn | complete | fail | compensate`) has -a constructor. - -{{ comp.apiTable({ - caption: "Effect helpers — saga handler outcomes", - rows: [ - { name: "sagaComplete(result?)", type: "=> CascadedMessage<'complete'>", desc: "Terminal success. Marks the instance finished and records the optional result payload." }, - { name: "sagaFail(reason)", type: "string | Error => CascadedMessage<'fail'>", desc: "Terminal failure. Records the reason; no further messages are applied to the instance." }, - { name: "sagaCompensate(message, reason?)", type: "=> CascadedMessage<'compensate'>", desc: "Route into the matching .compensate(type, ...) handler to undo an already-applied step." }, - { name: "send(target, payload, options)", type: "=> CascadedMessage<'send'>", desc: "Dispatch a command to a target (job, saga, or runtime adapter). options carry idempotencyKey / concurrencyKey / retry / queue." }, - { name: "schedule(message, delay)", type: "delay: Date | number | '5m' => CascadedMessage<'scheduled'>", desc: "Deliver a wrapped message after a delay (a Date, ms, or a '30s'/'5m'/'2h'/'1d' string)." }, - { name: "spawn(child, input, options)", type: "=> CascadedMessage<'spawn'>", desc: "Start a child saga from a definition or id, passing typed input. options take idempotencyKey / concurrencyKey." } - ] -}) }} - -## createParallelQueue — fan-out and concurrent processing - -Saga handlers stay synchronous and pure; the *transport* layer that carries cascaded -messages and feeds fan-out work is where you tune concurrency. `createParallelQueue` (from -{{ comp.xref({ key: "ref:queue", text: "@netscript/queue" }) }}) is the primitive for -that: it wraps a base queue so a single listener processes several messages at once. It is -the right tool when a saga spawns many independent children or pushes I/O-bound side work -(API calls, DB writes) that should run in parallel rather than one at a time. The -`concurrency` option is the whole story — `1` is plain sequential, anything higher enables -parallel processing. - -{{ comp.apiTable({ - caption: "createParallelQueue(name, options) — ParallelQueueOptions (extends QueueOptions)", - rows: [ - { name: "concurrency", type: "number (default 1)", desc: "Number of concurrent processors. Must be >= 1; values > 1 wrap the queue for parallel listening. Use for I/O-bound work; for CPU-bound work prefer web-worker tasks." }, - { name: "provider", type: "QueueProvider ('deno-kv' | 'redis' | 'rabbitmq' | 'postgres')", desc: "Backing queue provider. Omit to auto-discover from the Aspire environment." }, - { name: "autoDiscover", type: "boolean (default true)", desc: "Discover a queue service from Aspire. Priority RabbitMQ > Redis > Deno KV." }, - { name: "retryAttempts", type: "number (default 3)", desc: "Max retry attempts for failed messages, when the backend lacks native retry." }, - { name: "retryDelay", type: "number ms (default 1000)", desc: "Delay between retries, when the backend lacks native retry." }, - { name: "connection", type: "QueueConnectionOptions", desc: "Provider-specific connection options (denoKv / redis / rabbitmq / postgres)." }, - { name: "deadLetterStore", type: "DeadLetterStorePort", desc: "Where terminal message failures land. Omit to use the provider's durable default." }, - { name: "disableAutoTracing", type: "boolean (default false)", desc: "Skip the automatic TracedQueue wrapper when you trace manually." } - ] -}) }} - -```ts -// plugins/sagas/fan-out.ts — process saga side-effects concurrently -import { createParallelQueue, QueueProvider } from '@netscript/queue'; - -type NotifyMessage = Readonly<{ orderId: string; channel: 'email' | 'sms' }>; - -// Four messages processed at a time on a single listener — for I/O-bound fan-out -// (notifications, webhook calls) a saga emits as it advances. -const notifications = createParallelQueue('order-notifications', { - concurrency: 4, - provider: QueueProvider.Redis, -}); - -await notifications.listen(async (message) => { - // These run up to 4 at a time, not serially. - await deliverNotification(message.orderId, message.channel); -}); -``` - -{{ comp callout { type: "note", title: "Concurrency lives in two places" } }} -createParallelQueue({ concurrency }) tunes how many queue messages a -listener processes at once. defineSaga().concurrency({ limit, key }) bounds -how many messages run against one saga instance (or per derived key) — overlapping -publishes for the same key are rejected. Use the queue knob for fan-out throughput; use the -builder knob to serialize work on a single business entity (one order, one tenant). -{{ /comp }} - -## Choosing a durable store backend - -Authoring a saga decides *what* it does. The **durable store backend** decides *where -its runtime state lives between messages and across crashes*. NetScript ships two -backends, and the choice is **explicit and mandatory** — the runtime refuses to start -without one. - -- **`kv`** — durable saga state in Deno KV (the orchestration store stood up by - Aspire). Zero extra schema; the natural default for a single-service app and for - local development. -- **`prisma`** — durable saga state in your scaffolded relational database via Prisma - (Postgres in the recommended setup; equally `mysql` / `mssql` / `sqlite`, since the store writes through - your project's Prisma client — it is not Postgres-specific). The `PrismaSagaStore` - writes the dedicated runtime tables `saga_runtime_state`, `saga_runtime_transition`, - and `saga_runtime_correlation`. Reach for this when you want the saga's own write - path in your relational database alongside the rest of your data, with SQL-level - inspection of in-flight state and transition history. - -You select the backend with the `NETSCRIPT_SAGA_STORE` environment variable (`kv` | `prisma`) or the appsettings key `sagas.store.backend`. The plugin service resolves this on startup via `resolveSagaStoreBackend(...)`, which **throws** if neither is set — there is no silent default in the resolver, by design, so a deployment can never guess wrong about where durable state lands. (Calling `createDurableSagaRuntime(...)` directly without a `backend` falls back to the KV store; the mandatory-selection guarantee comes from the resolver the service runs at startup.) - -{{ comp.apiTable({ - caption: "Durable saga store backends — trait matrix", - rows: [ - { name: "kv", type: "Deno KV", desc: "Default for local/single-service. No extra schema. Provisioned by Aspire (Redis/KV). Resolved via NETSCRIPT_SAGA_STORE=kv or sagas.store.backend=kv." }, - { name: "prisma", type: "Relational / Prisma", desc: "Writes saga_runtime_state, saga_runtime_transition, saga_runtime_correlation. Requires a Prisma client passed to createDurableSagaRuntime — so it follows whatever engine you scaffolded (Postgres in the recommended setup; mysql / mssql / sqlite all work). SQL-inspectable in-flight state. Resolved via NETSCRIPT_SAGA_STORE=prisma or sagas.store.backend=prisma." }, - { name: "selection", type: "mandatory", desc: "No implicit default. resolveSagaStoreBackend(...) throws when neither NETSCRIPT_SAGA_STORE nor sagas.store.backend is set." }, - { name: "client requirement", type: "prisma only", desc: "backend: 'prisma' (or passing prisma) without a Prisma client throws 'Prisma saga store backend requires a Prisma client.'" } - ] -}) }} - -The factory that owns these resources is `createDurableSagaRuntime(...)` from the -`@netscript/plugin-sagas/runtime` subpath. It resolves a `SagaStorePort`, builds the -native runtime over it, and hands you back a `dispose()` that closes the store (and the -KV handle it opened). - -{{ comp.tabbedCode({ tabs: [ - { - label: "kv backend", - lang: "ts", - code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\n\n// Deno KV durable store — the default for local/single-service apps.\n// Selected at deploy time by NETSCRIPT_SAGA_STORE=kv (or appsettings sagas.store.backend=kv).\nconst { runtime, store, dispose } = await createDurableSagaRuntime({\n backend: 'kv',\n // kv is opened for you if you don't inject one (openSagaRuntimeKv()).\n});\n\n// ... register saga definitions on `runtime`, process messages ...\n\nawait dispose(); // closes the KV-backed store + handle" - }, - { - label: "prisma backend", - lang: "ts", - code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\nimport { PrismaClient } from './generated/prisma/client.ts';\n\n// Postgres/Prisma durable store — writes saga_runtime_* tables.\n// Selected at deploy time by NETSCRIPT_SAGA_STORE=prisma.\nconst prisma = new PrismaClient();\nconst { runtime, store, dispose } = await createDurableSagaRuntime({\n backend: 'prisma',\n prisma, // REQUIRED for prisma — omitting it throws.\n});\n\n// ... register saga definitions, process messages — transitions land in\n// saga_runtime_state / saga_runtime_transition / saga_runtime_correlation ...\n\nawait dispose();" - }, - { - label: "resolve from env / appsettings", - lang: "ts", - code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\nimport { resolveSagaStoreBackend } from '@netscript/plugin-sagas-core/stores';\n\n// Read the backend from the environment (or appsettings) — throws if unset.\nconst backend = resolveSagaStoreBackend({\n env: Deno.env.toObject(),\n // appsettings: loadedAppsettings, // sagas.store.backend\n});\n\nconst runtime = await createDurableSagaRuntime({\n backend,\n prisma: backend === 'prisma' ? prismaClient : undefined,\n});" - } -] }) }} - -{{ comp callout { type: "important", title: "Two durability concepts — don't conflate them" } }} -.durability('t1') on the saga definition selects a -durability tier (SAGA_DURABILITY_TIERS) — a property of the saga -itself. The store backend (kv | prisma) is a -runtime/deployment choice about which database holds durable state, -resolved by NETSCRIPT_SAGA_STORE / sagas.store.backend. A -saga keeps its declared tier regardless of which backend you deploy against. -{{ /comp }} - -{{ comp callout { type: "note", title: "Read model vs. durable write path" } }} -The Prisma backend's saga_runtime_* tables are the durable -write path the engine checkpoints to. They are distinct from -saga_instances (and saga_execution_history) — a -read-model projection into Postgres for API queries, analytics, and -debugging. The runtime persists to the former; the API surface lists from the latter. -{{ /comp }} - -## The HTTP publisher — createSagaPublisher - -A message reaches a saga through a **publisher**. `createSagaPublisher` (from -`@netscript/plugin-sagas/runtime`) returns a `SagaPublisherPort` whose `publish(...)` POSTs -to the sagas API publish endpoint, discovering the service URL from the Aspire environment -by default. The workers `create-user-settings` job uses exactly this to emit -`UserSettingsCreated` across the plugin boundary. - -{{ comp.apiTable({ - caption: "createSagaPublisher(options) — HttpSagaPublisherOptions", - rows: [ - { name: "serviceName", type: "string", desc: "Aspire service name to resolve a base URL from. Defaults to the sagas service discovery name." }, - { name: "baseUrl", type: "string", desc: "Explicit base URL override; skips discovery when set." }, - { name: "publishPath", type: "string (default '/api/v1/sagas/publish')", desc: "Path the publisher POSTs each message to." }, - { name: "headers", type: "Record", desc: "Extra headers sent with every publish (auth, tenant routing)." }, - { name: "retryableStatusCodes", type: "readonly number[] (default 408,409,425,429,5xx)", desc: "HTTP statuses treated as retryable so the receipt is marked retryable." }, - { name: "id", type: "string (default 'http-saga-publisher')", desc: "Stable publisher id surfaced in diagnostics." }, - { name: "fetcher / readEnv", type: "boundary fns", desc: "Test/injection seams for the fetch implementation and the env reader used for discovery." } - ] -}) }} - -The port returns a typed **receipt** rather than throwing: `publish(...)` resolves a -`SagaPublisherResult` — either `{ published: true, ... }` (a `SagaPublisherReceipt`) or -`{ published: false, reason, retryable }` (a `SagaPublisherRejected`). `publishMany(...)` -takes a `mode: 'sequential' | 'parallel'` so a batch can fan out. - -{{ comp.tabbedCode({ tabs: [ - { - label: "The job that publishes (workers)", - lang: "ts", - code: "// plugins/workers/jobs/create-user-settings.ts (core, verbatim from the scaffold)\nimport { createSagaPublisher } from '@netscript/plugin-sagas/runtime';\nimport { createSuccessResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { z } from 'zod';\n\nconst PayloadSchema = z.object({ userId: z.string().min(1) });\nconst sagaPublisher = createSagaPublisher();\n\nconst handler = defineJobHandler(async (ctx) => {\n const { userId } = PayloadSchema.parse(ctx.payload ?? {});\n // This is the message the saga below consumes — a typed receipt comes back.\n await sagaPublisher.publish({ type: 'UserSettingsCreated', payload: { userId } });\n return createSuccessResult({ userId, settingsCreated: true });\n});\n\nexport default Object.assign(handler, { id: 'create-user-settings' });" - }, - { - label: "The saga that consumes (sagas)", - lang: "ts", - code: "// plugins/sagas/user-settings-saga.ts\nimport { defineSaga, sagaComplete } from '@netscript/plugin-sagas-core';\n\ntype State = Readonly<{ status: string; processedAt?: string }>;\n\nexport const userSettingsSaga = defineSaga('user-settings-saga')\n .durability('t1')\n .state({ status: 'pending' })\n .on<'UserSettingsCreated', { userId: string }>(\n 'UserSettingsCreated',\n (saga, event, context) => {\n saga.state = { ...saga.state, status: 'completed', processedAt: context.now.toISOString() };\n return [sagaComplete({ userId: event.payload.userId, processedAt: context.now.toISOString() })];\n },\n )\n .build();" - } -] }) }} - -## Extension points - -The current scaffold uses the curated defaults, but the core package exposes the seams the -plugin is composed from. Each is a subpath of `@netscript/plugin-sagas-core` — cite the -sub-path in prose and look it up under {{ comp.xref({ key: "ref:sagas" }) }} (e.g. -`reference/sagas/presets`, `reference/sagas/transports`). - -{{ comp.apiTable({ - caption: "Saga extension seams (@netscript/plugin-sagas-core subpaths)", - rows: [ - { name: "/presets", type: "startSagas(), startSagaHandlers()", desc: "Composition helpers that build a runtime from explicit definitions and return a { runtime, bus, sagaCount, shutdown } bundle. startSagaHandlers is the distributed-handler alias." }, - { name: "/middleware", type: "createSagaMiddleware(), createSSEEventsMiddleware()", desc: "Hono middleware that injects saga helpers into request context, plus SSE event emission with optional durable history (SagaHistoryWriter)." }, - { name: "/transports", type: "createGarnetListTransport(), createNetScriptRedisTransport()", desc: "At-least-once delivery adapters: a Garnet/Redis LIST transport and a Redis Streams transport, both with immediate + delayed publish and ack/nack." }, - { name: "/agent", type: "agent surface", desc: "Agent integration seam for the plugin's agent-facing surface (alpha)." }, - { name: "/integration/publisher", type: "SagaPublisherPort", desc: "The publisher boundary createSagaPublisher implements — publish() / publishMany() with typed receipts." }, - { name: "/integration/workers", type: "workers bridge", desc: "The seam that lets a workers job emit and consume saga messages across the plugin boundary." } - ] -}) }} - -{{ comp callout { type: "tip", title: "Default transport vs. Redis/Garnet" } }} -The scaffolded sagas service runs over the in-process native runtime and Deno KV. The -/transports adapters (createGarnetListTransport, -createNetScriptRedisTransport) are for multi-process or distributed -delivery — when saga handlers run as separate processes and need at-least-once delivery with -consumer groups, delayed messages, and dead-letter handling. Reach for them only when a single -process is no longer enough; the KV path covers local and single-service apps. -{{ /comp }} - -## Endpoints and ports - -The sagas plugin runs an oRPC API service on `:8092`. It lists registered sagas and -inspects running instances; the registry is backed by Deno KV. These are the routes -the live scaffold serves — see {{ comp.xref({ key: "ref:sagas" }) }} for the full -generated surface. - -{{ comp.apiTable({ - caption: "Sagas plugin — runtime endpoints (port :8092)", - rows: [ - { name: "GET /health/live", type: "liveness", desc: "Liveness probe for the sagas API service." }, - { name: "GET /api/v1/sagas/sagas", type: "registry", desc: "List the saga definitions registered into KV (id, name, topic, handled message types, enabled)." }, - { name: "GET /api/v1/sagas/instances", type: "instances", desc: "List running and completed saga instances. Inspect one with /instances/{sagaName}/{correlationId}." }, - { name: "POST /api/v1/sagas/publish", type: "publish", desc: "Publish a message to the saga bus — the same path createSagaPublisher POSTs to and the workers create-user-settings job uses." }, - { name: "GET /api/v1/sagas/subscribe", type: "stream (SSE)", desc: "Server-sent-events stream of saga activity (saga:started / state_changed / completed / failed / compensating), KV-watch backed." } - ] -}) }} - -## The continuous-app choreography - -The thread that ties the capabilities together is real and it compiles. The workers -plugin's `create-user-settings` job calls a saga publisher and emits -`UserSettingsCreated`; this saga's `.on('UserSettingsCreated', ...)` handler consumes -it and emits `sagaComplete(...)`. One message crosses the plugin boundary, and both -halves are type-checked against the same message type. - -After both plugins are running under Aspire, trigger the workers job -(`POST :8091/api/v1/workers/jobs/create-user-settings/trigger`) and watch the saga -appear at `GET :8092/api/v1/sagas/instances` — the message crossed the boundary and a -durable instance recorded its completion. Whether that durable instance lives in Deno -KV or in your `saga_runtime_*` Postgres tables is exactly the `NETSCRIPT_SAGA_STORE` -choice above. - -## Production notes - -{{ comp callout { type: "warning", title: "Aspire first, then anything stateful" } }} -The sagas service needs Postgres and Redis up before it can persist and list -instances. Bring orchestration up first — cd aspire && aspire start -(dashboard at https://localhost:18888) — -before any netscript db command or before you expect -/api/v1/sagas/instances to return durable state. This holds for both -backends: kv needs Redis/KV up, and prisma needs Postgres up -with the saga_runtime_* tables migrated. DB commands require Aspire running -first. -{{ /comp }} - -{{ comp callout { type: "important", title: "Handlers are synchronous and pure" } }} -A SagaHandler returns readonly CascadedMessage[] — it is -synchronous and does no direct I/O. All side effects (charging payment, -sending mail, spawning a child) are expressed as effects the runtime applies, so -the handler is replayable and crash-safe. Do not await a network call inside a -handler; emit a send(...) effect and let the target do the work. Compensation -follows the same rule — its undo runs in the .compensate() handler as more -effects, not as inline cleanup. -{{ /comp }} - -## Why a saga, and why not - -{{ comp callout { type: "tip", title: "Reach for a saga when…" } }} -You have a multi-step process where steps can fail independently and the -state between steps must survive a crash — onboarding, checkout, -provisioning. The correlation, persistence, and named completion/failure/compensation -outcomes are exactly what a hand-rolled retry loop lacks. Pick prisma as the -store backend when you also want that in-flight state queryable in your relational database. -{{ /comp }} - -{{ comp callout { type: "warning", title: "Don't reach for a saga when…" } }} -The work is a single idempotent unit with no inter-step state — that is a -background job, not a saga. And remember -the alpha reality: durability and the instance store depend on the orchestration stack -being up via Aspire (Redis/KV for kv, Postgres for prisma), -so a saga is not a substitute for a database transaction within one handler. -{{ /comp }} - -## Reference → - -{{ comp.featureGrid({ - columns: 2, - items: [ - { icon: "≡", title: "Look up — sagas reference", body: "The full generated @netscript/plugin-sagas API: the defineSaga builder, durability tiers, createDurableSagaRuntime, the kv/prisma store backends, effect helpers, presets/middleware/transports seams, and every :8092 route.", href: "/reference/sagas/" }, - { icon: "✲", title: "Understand — the durability model", body: "Why a saga's state must outlive the process, how effect-based outcomes differ from a retry loop, and where the kv vs prisma backends fit.", href: "/explanation/durability-model/" }, - { icon: "◆", title: "Learn — Storefront checkout saga", body: "Build the order/checkout saga with a compensation branch as part of the storefront tutorial track.", href: "/tutorials/storefront/04-checkout-saga/" }, - { icon: "▣", title: "Look up — queue reference", body: "createParallelQueue, createQueue, providers, and connection options for the fan-out concurrency primitive.", href: "/reference/queue/" } - ] -}) }} - -{{ comp.nextPrev({ - prev: { label: "Background jobs", href: "/capabilities/background-jobs/" }, - next: { label: "Triggers & ingress", href: "/capabilities/triggers/" } -}) }} diff --git a/docs/site/capabilities/fresh-framework.md b/docs/site/capabilities/fresh-framework.md index eac17ae57..ea3e24d74 100644 --- a/docs/site/capabilities/fresh-framework.md +++ b/docs/site/capabilities/fresh-framework.md @@ -1,312 +1,4 @@ --- -layout: layouts/base.vto -title: Fresh meta-framework -templateEngine: [vento, md] -prev: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" } -next: { label: "Typed SDK & client", href: "/capabilities/sdk/" } +layout: layouts/redirect.vto +redirectTo: /web-layer/ --- - -# Fresh meta-framework - -The NetScript **Fresh meta-framework** is the application layer that turns a route into a -typed, server-rendered page. You author a page with `definePage()` — binding a typed -**route contract**, server **resources** and **layers** (each with its own loader, cache -window, and partial-refresh endpoint), and **forms** — then build it into Fresh route -wiring. The same contract object a page loader uses to call a service is the one the -[typed SDK client](/capabilities/sdk/) imports and the [oRPC service](/capabilities/services/) -implements, so the page → contract → SDK → service request model **cannot drift**. - -{{ comp.badge({ status: "alpha" }) }} - -{{ comp.diagram({ - src: "/assets/diagrams/fresh-page-model.svg", - alt: "Request flow: browser hits a Fresh route built by definePage; the server handler runs resource and layer loaders that call the typed SDK client, which calls an oRPC service backed by the database; the rendered HTML ships to the browser where an island hydrates against the same query key.", - caption: "The Fresh page model: definePage binds a route, runs server loaders through the typed SDK to a service, renders HTML, and hydrates islands against a shared query cache — one contract end to end." -}) }} - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for the Fresh meta-framework when you need a server-rendered, typed page -— a route with loaders that pull from your services, layered slots that refresh independently, -forms that validate and mutate server-side, and islands that hydrate just the interactive parts. -For components, theming, and copy-source primitives this page does NOT cover those — -see Fresh UI & design. For the data layer the -loaders call use the typed SDK; for the -contract those services share see Contracts. -{{ /comp }} - -## What it is - -`@netscript/fresh` is a **DSL / builder** package over [Fresh 2](https://fresh.deno.dev/). Its -public product is a small set of builders and typed contracts — `definePage()`, -`defineRouteContract()`, `definePartial()`, `defineFreshApp()`, plus form, query, and defer -factories — that translate page and route intent into Fresh runtime wiring. A page is composed -from three cooperating pieces: a **server layer** (data resolved by a loader), a **partial -route** that re-renders just that layer on demand, and a **hydrated island** that shares the -same query cache on the client. Rendering is **cache-first**: a fresh cached payload renders -immediately, a stale one renders then reloads in the background (SWR), and a miss falls back to -the layer's partial. The contract type-flow that makes this safe end-to-end is explained in -[Contracts](/explanation/contracts/). - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Build a page (Track D)", - body: "The guided tutorial track: route contract → definePage with a cache-first layer → its partial → a hydrated island, built from scratch.", - href: "/tutorials/live-dashboard/", - icon: "→" - }, - { - title: "Do — Customize the Fresh UI", - body: "Recolor the design system, swap primitives, and extend the generated app shell — the hands-on Fresh UI customization recipe.", - href: "/how-to/customize-fresh-ui/", - icon: "◆" - }, - { - title: "Do — TanStack Query in islands", - body: "Wire an island to the same cache the page loader filled, so hydration is instant and invalidation is shared.", - href: "/tutorials/live-dashboard/04-definePage-QueryIsland/", - icon: "◆" - }, - { - title: "Look up — Components & theming", - body: "The design-system layer — primitives, interactive components, and tokens — lives in the Fresh UI hub, not here.", - href: "/capabilities/fresh-ui/", - icon: "≡" - } -] }) }} - -## Minimal example — an orders page - -A page starts with a **typed route contract**, then `definePage()` binds it, attaches a -metadata resolver, and builds the Fresh route wiring. The search schema below is parsed and -type-checked, so `ctx.useSearch()` inside a loader is fully typed. - -```tsx -// routes/orders/index.tsx -import { definePage } from '@netscript/fresh/builders'; -import { defineRouteContract, paginationSearchSchema } from '@netscript/fresh/route'; -import { OrdersPanel } from '../../islands/OrdersPanel.tsx'; - -// 1. The route contract: typed, parsed search params (the single source of truth for this URL). -const ordersRoute = defineRouteContract({ - searchSchema: paginationSearchSchema({ - defaultLimit: 20, - defaultSort: 'createdAt', - defaultOrder: 'desc', - }), -}); - -// 2. The page: bind the route, load a server resource, render a cache-first layer. -export const ordersPage = definePage() - .withRoute(ordersRoute) - .withResource('orders', async (ctx) => { - const { limit, sort, order } = ctx.useSearch(); - return await ordersClient.list.query({ limit, sort, order }); - }) - .withLayer('list', OrdersPanel, { - loader: (ctx) => ({ orders: ctx.useResource('orders') }), - partialName: 'orders-list', - staleTime: 30_000, - staleReloadMode: 'background', - }) - .withMeta(() => ({ title: 'Orders', description: 'Browse the current order queue.' })) - .build(); -``` - -The matching **partial route** re-renders only the `orders-list` layer when the island asks -for fresh data — same loader, same query key, no full-page reload: - -```tsx -// routes/orders/_partials/orders-list.tsx -import { definePartial } from '@netscript/fresh/builders'; -import { OrdersPanel } from '../../../islands/OrdersPanel.tsx'; - -export const ordersListPartial = definePartial({ - name: 'orders-list', - loader: async (ctx) => ({ orders: await ordersClient.list.query(ctx.search) }), - component: OrdersPanel, -}); -``` - -## Key types first — the page builder chain - -`definePage()` returns a fluent builder. Each `with*` step is typed against the accumulated -state, so a search-param key, a resource name, or a layer's props are all inferred forward into -the loaders and components downstream. Call `.build()` last; with a bound route it returns a -routed definition exposing `route`, `nav`, and `hooks`. - -{{ comp.apiTable({ - caption: "definePage() builder methods (PageBuilder)", - rows: [ - { name: "withRoute(route)", type: "method", desc: "Bind the page to a typed route contract (from defineRouteContract). Makes path/search params type-safe in every loader and component." }, - { name: "withResource(key, factory)", type: "method", desc: "Resolve one named server resource (e.g. an SDK query). Read it later with ctx.useResource(key). withResources(map) adds several at once." }, - { name: "withParams / withPathParams / withSearchParams", type: "method", desc: "Attach path and/or search schemas directly (alternative to withRoute) when you do not need a shared route contract." }, - { name: "withPolicy(policy)", type: "method", desc: "Set the page-wide defer policy — a named profile or an override object. See the policy table below." }, - { name: "withLayer(id, component, config)", type: "method", desc: "Register a render slot with its own loader, cache window, and partial-refresh endpoint. config is a PageLayerConfig (or a bare loader). See the layer-config table." }, - { name: "withForm(id, component, config)", type: "method", desc: "Register a route-bound, server-validated form as a typed layer. config is a PageFormConfig. See the form-config table." }, - { name: "withLayout(layout)", type: "method", desc: "Compose the registered layer slots into the page shell: (slots) =>
{slots.list()}
." }, - { name: "withMeta(resolver)", type: "method", desc: "Resolve metadata (title, description) per request." }, - { name: "withTelemetry(config)", type: "method", desc: "Attach telemetry metadata (span naming) for the page's traces." }, - { name: "withHandler(method, handler)", type: "method", desc: "Register a raw method handler. Do not combine GET here with withHeader()/withStatus()." }, - { name: "build(options?)", type: "method", desc: "Finalize. build() / build('/path') / build({ routePattern }) — with a bound route the result exposes route, nav, and hooks." } - ] -}) }} - -## `withLayer` options — the cache-first slot - -A **layer** is the unit of independent loading and refresh. Its `loader` produces props, its -`partial`/`partialName` names the route that re-renders it in isolation, and the cache/policy -keys decide when a stale slot reloads. These are the real keys on `PageLayerConfig`: - -{{ comp.apiTable({ - caption: "PageLayerConfig — withLayer(id, component, config)", - rows: [ - { name: "loader", type: "(ctx) => Props | Promise", desc: "Async loader providing the layer component's props. Reads resources via ctx.useResource(key)." }, - { name: "partial", type: "string | (ctx) => string", desc: "Partial endpoint (or resolver) used to refresh this layer without a full-page reload." }, - { name: "partialName", type: "string | (ctx) => string", desc: "Stable Fresh partial name rendered into the response — the handshake key the island uses to ask for fresh data." }, - { name: "fallback", type: "unknown", desc: "Content shown while a deferred layer is still pending." }, - { name: "policy", type: "PageDeferPolicyInput | profile", desc: "Per-layer defer policy override (see the policy table)." }, - { name: "layerDeps", type: "(ctx) => unknown", desc: "Dependency projection (over path + search) deciding when the layer should reload." }, - { name: "staleTime", type: "number (ms)", desc: "Freshness window for the cached layer payload before it is considered stale." }, - { name: "gcTime", type: "number (ms)", desc: "Cache retention window before the payload is evicted." }, - { name: "staleReloadMode", type: "'blocking' | 'background'", desc: "When stale, reload before rendering (blocking) or render now and revalidate after (background SWR)." }, - { name: "shouldReload", type: "boolean | (ctx) => boolean", desc: "Explicit reload guard, overriding the freshness heuristics." }, - { name: "delivery", type: "PageLayerDelivery", desc: "Delivery mode for the layer (e.g. streamed)." } - ] -}) }} - -## `withPolicy` — the defer policy enum - -`withPolicy()` (and a layer's `policy`) accepts a **named profile** or an override object. The -full set of named profiles is: - -{{ comp.apiTable({ - caption: "PageDeferPolicyProfile + PageDeferPolicyInput overrides", - rows: [ - { name: "'balanced'", type: "profile (default)", desc: "Cache-first with sensible SWR — the default trade-off between first paint and freshness." }, - { name: "'aggressive-first-paint'", type: "profile", desc: "Render cached/fallback content as early as possible, revalidate after." }, - { name: "'background-refresh'", type: "profile", desc: "Prefer serving cache and always refresh in the background." }, - { name: "'low-bandwidth'", type: "profile", desc: "Minimize prewarm and client refresh traffic for constrained clients." }, - { name: "{ profile, staleTimeMs }", type: "override", desc: "Start from a named profile, then override the freshness window in ms." }, - { name: "{ prewarmOnMiss, prewarmOnStale }", type: "override", desc: "Prewarm the partial when the cache is missing and/or stale." }, - { name: "{ clientRefreshOnFreshCache, skipClientWhenServerPrewarm }", type: "override", desc: "Fine-tune whether the client refreshes when the server cache is already fresh or prewarming." } - ] -}) }} - -## `withForm` — server-validated forms as a layer - -`withForm(id, component, config)` registers a form as a typed layer: it wires the method -handler, CSRF headers, validation, and form metadata in one step. `TOutput` is inferred from -`mutate`'s return type, so `redirectTo`/`onSuccess` are typed against your result. - -{{ comp.apiTable({ - caption: "PageFormConfig — withForm(id, component, config)", - rows: [ - { name: "schema", type: "Schema (required)", desc: "Validation + constraint schema. The inference site for the form's input type." }, - { name: "mutate", type: "(input, ctx) => TOutput (required)", desc: "Runs the mutation with validated input. Its return type is the sole inference site for TOutput." }, - { name: "initial", type: "(ctx) => Partial", desc: "Resolves initial values on GET, merged with schema defaults." }, - { name: "onIntent", type: "(intent, values, ctx) => FormIntentResult", desc: "Handles non-submit intents (validate, reset) — short-circuits before validation." }, - { name: "redirectTo", type: "(output, ctx) => string | Response", desc: "Redirect target after a successful mutation. Takes precedence over onSuccess." }, - { name: "onSuccess", type: "(output, ctx) => { message?, nextValues? }", desc: "Success metadata when staying on the same page." }, - { name: "invalidate", type: "(output, ctx) => void", desc: "Cache invalidation after mutation, before the response is sent." }, - { name: "csrf", type: "boolean (default true)", desc: "CSRF protection toggle." }, - { name: "method", type: "'POST' | 'PUT' | 'PATCH' (default POST)", desc: "HTTP method for the form submission." }, - { name: "spanName", type: "string (default form.{id})", desc: "Telemetry span prefix for the form handler." } - ] -}) }} - -## Route contracts, partials, defer & islands - -The route contract is the typed boundary of a URL. `defineRouteContract({ pathSchema, -searchSchema })` produces a contract whose parsed params flow into the page; the -`@netscript/fresh/route` subpath ships the schema builders and link helpers, and -`@netscript/fresh/defer` + `@netscript/fresh/query` cover deferral and island hydration. - -{{ comp.apiTable({ - caption: "Surrounding surface (confirmed exports)", - rows: [ - { name: "defineRouteContract({ pathSchema, searchSchema })", type: "@netscript/fresh/route", desc: "Typed route contract. Parsed path/search params become type-safe page inputs." }, - { name: "paginationSearchSchema(opts)", type: "@netscript/fresh/route", desc: "Ready-made search schema for limit/sort/order pagination (defaultLimit/defaultSort/defaultOrder)." }, - { name: "enumPathParamSchema(name, values)", type: "@netscript/fresh/route", desc: "Typed enum path-param schema, e.g. a status segment constrained to a fixed set." }, - { name: "InferRouteContractSearch / InferRouteContractPath", type: "@netscript/fresh/route", desc: "Extract the parsed search/path types from a route contract for reuse." }, - { name: "createRouteReference / bindRoutePattern", type: "@netscript/fresh/route", desc: "Typed href and link-prop helpers bound to a route pattern." }, - { name: "definePartial({ name, loader, component })", type: "@netscript/fresh/builders", desc: "A framework-owned partial route — the isolated re-render target for a layer." }, - { name: "DeferPage / DeferComponent / Deferred", type: "@netscript/fresh/defer", desc: "Suspense-style deferred rendering: stream a fallback now, swap real content when the promise resolves." }, - { name: "QueryIsland / useIslandQuery / useIslandMutation / useLiveQuery", type: "@netscript/fresh/query", desc: "TanStack Query for islands, imported through one centralized subpath so the dependency never forks." }, - { name: "defineFreshApp(options)", type: "@netscript/fresh/server", desc: "Bootstrap the Fresh app runtime that serves the built pages." } - ] -}) }} - -{{ comp callout { type: "note", title: "Reading page state inside loaders & components" } }} -Inside a routed page's render tree, read the resolved state through the page context the loader -receives — ctx.useResource(key), ctx.useResources(), -ctx.useSearch(), ctx.useRoute(), and ctx.useSlots() — or -the value-level hooks usePageRoute(), usePagePath(), and -usePageSearch() for the canonical current page. These are context -accessors, not standalone exported hooks: a routed .build() also exposes -typed route.hooks.use*() bundles for inference inside islands. -{{ /comp }} - -## Production notes - -{{ comp callout { type: "warning", title: "Don't double-handle GET" } }} -withHeader() and withStatus() generate their own GET -handler that calls ctx.render(). Do not also register -withHandler('GET', …) on the same page — the two collide. Pick one: declarative -headers/status, or a hand-written GET handler. -{{ /comp }} - -{{ comp callout { type: "important", title: "Islands import TanStack Query through the subpath" } }} -Island code must import query hooks from @netscript/fresh/query (e.g. -useIslandQuery, QueryIsland), not from -@tanstack/preact-query directly. The subpath centralizes the dependency and the -hydration boundary, so the page loader's cache and the island's cache stay the same instance — -import the raw library and hydration silently breaks. -{{ /comp }} - -{{ comp callout { type: "important", title: "Runtime permissions & KV typing" } }} -Deferred prewarm and durable-stream helpers touch the platform at runtime: ---allow-net for partial prewarm and stream endpoints, --allow-env when -stream settings resolve from the environment, and --unstable-kv for the SSE helpers -that watch Deno.Kv. Type-check page entrypoints with --unstable-kv too — -the streaming server helpers expose KV-aware types. NetScript is alpha -({{ releaseVersion }}); pin the version and expect surface churn. -{{ /comp }} - -## Reference → - -This hub is intentionally thin — the full generated API for every subpath -(`server`, `builders`, `route`, `defer`, `form`, `query`, `interactive`, `streams`, `vite`) -lives in the reference. - -{{ comp.xref({ key: "ref:fresh" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — @netscript/fresh reference", - body: "The full generated API: definePage, defineRouteContract, the form/defer/query factories, and the server bootstrap.", - href: "/reference/fresh/", - icon: "≡" - }, - { - title: "Data — Typed SDK & client", - body: "The cache-first typed clients your page loaders call. Same contract as the service, no drift.", - href: "/capabilities/sdk/", - icon: "◆" - }, - { - title: "Design — Fresh UI & components", - body: "The component and theming layer this page renders: primitives, interactive components, and tokens.", - href: "/capabilities/fresh-ui/", - icon: "→" - }, - { - title: "Understand — Contracts", - body: "The contract → SDK client → page loader → island type flow that keeps caller and server in lockstep.", - href: "/explanation/contracts/", - icon: "◎" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" }, next: { label: "Typed SDK & client", href: "/capabilities/sdk/" } }) }} diff --git a/docs/site/capabilities/fresh-ui.md b/docs/site/capabilities/fresh-ui.md index f00772d26..589414d8b 100644 --- a/docs/site/capabilities/fresh-ui.md +++ b/docs/site/capabilities/fresh-ui.md @@ -1,264 +1,4 @@ --- -layout: layouts/base.vto -title: Fresh UI & design -templateEngine: [vento, md] -prev: { label: "Telemetry & logging", href: "/capabilities/telemetry/" } -next: { label: "Authentication", href: "/capabilities/auth/" } +layout: layouts/redirect.vto +redirectTo: /web-layer/fresh-ui/ --- - -# Fresh UI & design - -NetScript gives you a **front end with the same contracts-first rigor as the back end** — and it -does so in two distinct layers that are easy to conflate. There is **`@netscript/fresh`, a real -meta-framework** (a published package of Fresh runtime extensions, route/island builders, a -TanStack-Query bridge, forms, and a streams client), and there is the **scaffolded dashboard app** -(`apps/dashboard`) that the meta-framework *powers*. The package is the reusable engine; the app is -one fully-wired showcase built on top of it. Both ship in every workspace, but they live at -different addresses and you reach for them at different moments. - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for this layer when you need a typed, server-rendered web UI over your -services — an internal admin panel, an operations console, or a customer-facing app — that cannot -drift from your oRPC contracts. Use @netscript/fresh (the -meta-framework) to build pages/islands in any Fresh app; use apps/dashboard -(the scaffolded app) as the worked reference to copy patterns from. For the data boundary the UI -renders see Services & contracts; to add or regenerate UI -pieces see Customize the Fresh UI. -{{ /comp }} - -## Two layers, one design grammar - -{{ comp.apiTable({ - caption: "The meta-framework vs. the scaffolded app", - rows: [ - { name: "@netscript/fresh", type: "meta-framework (package)", desc: "A published Deno package: Fresh runtime extensions, the definePage/island builders, a TanStack-Query bridge, forms, defer primitives, a streams client, and route contracts. Reusable across any Fresh app you build. Reference: /reference/fresh/." }, - { name: "apps/dashboard", type: "scaffolded app", desc: "One concrete Fresh 2.x application generated into your repo, built ON @netscript/fresh and wired to your users service. The worked showcase you copy patterns from — not a finished product UI." }, - { name: "@netscript/fresh-ui", type: "component registry", desc: "The copy-source component library + design tokens installed into apps/dashboard/components/ui/ via netscript ui:init / ui:add. You own the copied files. Reference: /reference/fresh-ui/." } - ] -}) }} - -{{ comp callout { type: "important", title: "Don't conflate the package with the app" } }} -@netscript/fresh is a meta-framework you import — it exposes targeted -subpaths (/server, /builders, /route, /query, -/form, /defer, /streams, /interactive, -/vite, /testing, /error) and can power any Fresh app, -not just the scaffold. apps/dashboard is the app that consumes it. When -this page says "the dashboard," it means the scaffolded app; when it says -"@netscript/fresh," it means the reusable engine. -{{ /comp }} - -## The meta-framework: `@netscript/fresh` - -`@netscript/fresh` is a published package (`Fresh runtime extensions, builders, forms, defer -primitives, and route contracts for NetScript`) built on `@fresh/core` 2.x + Preact. You consume it -through **targeted subpath exports** rather than one fat barrel — import only the surface you need. -The full generated API for every subpath lives at the -[`@netscript/fresh` reference](/reference/fresh/); the table below is the map. - -{{ comp.apiTable({ - caption: "@netscript/fresh subpath exports (the meta-framework surface)", - rows: [ - { name: "@netscript/fresh/server", type: "bootstrap", desc: "defineFreshApp(...) — the baseline app bootstrap (static files, file-system routes) so every NetScript Fresh app boots identically." }, - { name: "@netscript/fresh/builders", type: "page builder", desc: "The definePage() / route-reference builders that bind a typed route, declare server-loaded layers, and wire telemetry." }, - { name: "@netscript/fresh/route", type: "route contracts", desc: "Route-reference primitives (createRouteReference) for typed, generated route patterns consumed via router.ts." }, - { name: "@netscript/fresh/query", type: "TanStack bridge", desc: "QueryIsland, useQuery, useMutation, useQueryClient — the TanStack-Query hydration bridge over your contract-derived query factories." }, - { name: "@netscript/fresh/form", type: "forms", desc: "Typed form primitives for server-validated, contract-aware form handling." }, - { name: "@netscript/fresh/defer", type: "streaming defer", desc: "Defer primitives for streaming/deferred server rendering of slower layers." }, - { name: "@netscript/fresh/streams", type: "streams client", desc: "The Fresh-side durable-stream client for consuming HTTP/SSE durable streams in the browser (the producer runtime lives in @netscript/plugin-streams-core)." }, - { name: "@netscript/fresh/interactive", type: "interactivity", desc: "Interactive island runtime helpers used by the hydrated client islands." }, - { name: "@netscript/fresh/vite", type: "build", desc: "The Vite integration the dev/build pipeline uses." }, - { name: "@netscript/fresh/testing", type: "testing", desc: "Test helpers for exercising pages, islands, and route contracts." }, - { name: "@netscript/fresh/error", type: "diagnostics", desc: "Typed error/diagnostics surface for the Fresh runtime." } - ] -}) }} - -{{ comp callout { type: "note", title: "Distinct from the back-end stream producer" } }} -The Fresh-side @netscript/fresh/streams client consumes durable streams over -HTTP/SSE in the browser. The producer runtime — the thing that writes stream state -— is real and lives server-side in @netscript/plugin-streams-core -(createDurableStream), served as an Aspire service on -:4437. See Streams for the -producer/consumer split. -{{ /comp }} - -## The scaffolded app: `apps/dashboard` - -Every NetScript workspace ships `apps/dashboard` — a [Fresh](https://fresh.deno.dev/) 2.x -application built with Preact, Tailwind CSS v4, and Vite. It is a workspace member registered in the -root `deno.json` and orchestrated by Aspire alongside your services. It boots through -`defineFreshApp` from `@netscript/fresh/server`, the meta-framework's baseline bootstrap, so every -NetScript Fresh app starts the same way. It is **not a placeholder**: the scaffold wires it directly -to your oRPC contracts so the dashboard renders typed, server-prefetched data from the same `users` -service the back end implements. - -{{ comp.apiTable({ - caption: "apps/dashboard layout (verbatim from a fresh scaffold)", - rows: [ - { name: "main.ts", type: "entry", desc: "App entry: export const app = defineFreshApp({ name: 'dashboard' }). Reads PORT (default 8010) and logs a startup banner." }, - { name: "router.ts", type: "routing", desc: "Stable route entrypoint. Re-exports generated routePatterns + routes and builds typed appRoutes via createRouteReference." }, - { name: "routes/", type: "pages", desc: "File-system routes: index.tsx, dashboard.tsx, health.tsx, examples/service/, examples/telemetry/, the (design) system pages, plus _app.tsx / _layout.tsx shells." }, - { name: "islands/", type: "interactivity", desc: "Client-hydrated Preact islands (e.g. ThemeToggle, SidebarToggle, Toast under islands/ui/)." }, - { name: "components/ui/", type: "design system", desc: "The copy-source component library you own (@netscript/fresh-ui): button, card, data-table, form-field, badge, and more (tsx + matching CSS in assets/ui/)." }, - { name: "lib/", type: "service wiring", desc: "example-service.ts builds a typed oRPC client + query factories from your contract; cn.ts, public-types.ts." }, - { name: "assets/", type: "styling", desc: "design.css, tokens.css/json, theme-bridge.css, and per-component CSS — the Tailwind v4 + design-token layer." }, - { name: ".generated/", type: "generated", desc: "manifest.ts + routes.ts produced by the Fresh route generator; consume via router.ts, never directly." } - ] -}) }} - -{{ comp callout { type: "note", title: "Two ports, one app" } }} -The dashboard's own dev server reads PORT and defaults to 8010 -(http://localhost:8010, health at /health). That is distinct from the -Aspire dashboard at https://localhost:18888, -which orchestrates the whole resource graph. Bring the platform up first with -cd aspire && aspire start (Postgres/Redis + every service), or run the app -standalone with deno task --cwd apps/dashboard dev. -{{ /comp }} - -## Contract-driven by default - -The dashboard does not hand-roll `fetch` calls. `apps/dashboard/lib/example-service.ts` imports your -`UsersContractV1` and turns it into a typed client and TanStack-Query factories through the NetScript -SDK — the same contract object the `users` service implements, so the UI cannot drift from the API. - -```ts -// apps/dashboard/lib/example-service.ts -import { createServiceClient } from '@netscript/sdk/client'; -import { createQueryFactories } from '@netscript/sdk/query'; -import { bridgeInvalidation } from '@netscript/sdk/query-client'; -import { UsersContractV1 } from '@plugin-smoke/contracts'; - -export const exampleServiceName = 'users'; -export const exampleServiceClient = createServiceClient({ - contract: UsersContractV1, - serviceName: exampleServiceName, - routerName: 'users', -}); - -// queryOptions / mutationOptions derived straight from the contract. -export const exampleServiceQueries = createQueryFactories({ - service: { contract: UsersContractV1, client: exampleServiceClient }, -}).service; -``` - -A **route** is declared with the meta-framework's `definePage()` builder (`@netscript/fresh/builders`) -— it binds a typed route reference, declares server-loaded layers, and wires telemetry — while an -**island** consumes the query factories above through the `@netscript/fresh/query` bridge for -hydration, optimistic mutations, and cache invalidation on the client. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Route — routes/examples/service/index.tsx", - lang: "tsx", - code: "// definePage() declares a typed page: route + server-loaded layers + telemetry.\nimport { appRoutes } from '@app/router.ts';\nimport { definePage } from '@app/utils.ts';\nimport { ServiceExampleLabPanel } from './(_components)/lab-panel.tsx';\nimport { loadServiceShowcaseData } from './(_shared)/service-showcase.ts';\n\nexport const serviceExamplePage = definePage()\n .withRoute(appRoutes.serviceExample)\n .withPolicy('balanced')\n .withTelemetry({ enabled: true, spanName: 'scaffold.examples.users' })\n .withMeta(() => ({ title: 'users example', description: 'Backed by the users service.' }))\n .withLayer('lab', ServiceExampleLabPanel, { loader: loadServiceShowcaseData })\n .build();\n\nexport const { default: page } = serviceExamplePage;\nexport { page as default };" - }, - { - label: "Island — (_islands)/ServiceShowcaseLab.tsx", - lang: "tsx", - code: "// Client-hydrated island: typed query + optimistic mutation over the contract.\nimport { QueryIsland, useMutation, useQuery, useQueryClient } from '@netscript/fresh/query';\nimport { exampleServiceQueries, exampleServiceListInvalidation } from '@app/lib/example-service.ts';\n\nconst Lab = (props: { input: { status?: string } }) => {\n const queryClient = useQueryClient();\n const { data, refetch, isRefetching } = useQuery({\n ...exampleServiceQueries.list.queryOptions(props.input),\n staleTime: 15_000,\n });\n const advance = useMutation({\n ...exampleServiceQueries.updateStatus.mutationOptions(),\n onSettled: () => queryClient.invalidateQueries(exampleServiceListInvalidation),\n });\n return ;\n};\n\nexport default (props: { input: { status?: string } }) => (\n \n);" - } -] }) }} - -{{ comp callout { type: "important", title: "Islands are the only client code" } }} -Fresh ships zero JavaScript by default. Anything under islands/ is the only -code hydrated in the browser; routes and components render on the server. The users -showcase loads its list on the server, dehydrates the query cache, and the -ServiceShowcaseLab island re-hydrates it for interactive refetch and optimistic status -updates — the contract-to-pixel proof end to end. -{{ /comp }} - -## The design system & owned components - -The scaffold installs the NetScript Fresh UI foundation (`@netscript/fresh-ui`) into -`apps/dashboard/components/ui/` and its CSS into `assets/ui/`, driven by design tokens in -`assets/tokens.json` / `tokens.css`. Because the components are copied into your repo, editing the -UI is editing your own files — there is no framework component you cannot open. The bundled -`(design)` route group renders a live token, component, and composition gallery so you can see every -primitive in your project. - -What ships falls into four groups, each installable by registry id (`ui:add `) or as a named -collection: - -{{ comp.apiTable({ - caption: "What the registry ships", - rows: [ - { name: "Interactive namespaces", type: "@netscript/fresh-ui/interactive", desc: "Stateful, accessible compound primitives: Accordion, Dialog, Drawer, Popover, Sheet, Tabs, Tooltip, and the headless Combobox seam (useCombobox) that powers the command palette and autocompletes." }, - { name: "L0 primitives", type: "@netscript/fresh-ui/primitives", desc: "Small platform-contract helpers: Show (wrapper-free conditional render), VisuallyHidden and its SrOnly alias, and Icon — a first-class stroke-SVG icon primitive also re-exported from the package root." }, - { name: "AI / workspace primitives", type: "ui:add", desc: "The agent-surface set: avatar, citation-chip, code-block, model-selector, tool-call-card, chart-block, donut, prompt-input (auto-grows to fit its content), message (renderInline + TypingIndicator), and dropzone (drag, clipboard-paste, and multi-file ingest) — plus the foundation set (button, card, badge, data-table, forms, feedback, dashboard blocks)." }, - { name: "Command & search utilities", type: "ui:add", desc: "command-palette (.ns-cmdk — the ⌘K modal that wraps the Combobox) and search (.ns-search — a compact nav affordance with a ⌘K hint that opens it)." } - ] -}) }} - -You manage that library with two CLI commands (run from the workspace root): - -{{ comp.apiTable({ - caption: "Fresh UI CLI commands", - rows: [ - { name: "netscript ui:init", type: "install foundation", desc: "Installs the Fresh UI foundation set into an app workspace. The scaffold runs the equivalent for you; run it once when adding UI to an app that lacks it." }, - { name: "netscript ui:add ", type: "add an item", desc: "Copies one registry item or a named collection into components/ui/, wires its CSS, and merges any required deno.json imports. Example: netscript ui:add data-table." } - ] -}) }} - -{{ comp callout { type: "warning", title: "Copy-source ownership tradeoff" } }} -Because ui:add copies code into your repo, you own it: customize freely, but you do -not get automatic upstream updates. Re-running ui:add … --force -re-copies the source — and overwrites your local edits — so treat foundation files as yours and keep -heavy customization in your own components. Full options live behind -netscript ui:init --help / netscript ui:add --help and in the -@netscript/fresh-ui reference. -{{ /comp }} - -## Scope - -The scaffolded dashboard is a working, contract-wired **showcase** — not a finished product UI. It -ships the `users` example (server-prefetched list + optimistic status mutation), a telemetry -example, a CRUD example, a health page, and the design gallery. Treat these as the canonical patterns -to copy from, not features to ship as-is; build your real screens by following the same -`definePage` + island + query-factory shape against `@netscript/fresh`. The meta-framework is the -durable surface — the app is one application of it. - -## Endpoints & ports - -{{ comp.apiTable({ - caption: "Fresh UI runtime surface", - rows: [ - { name: ":8010", type: "port", desc: "Dashboard dev server (Deno.env.get('PORT') || '8010'). Standalone: deno task --cwd apps/dashboard dev." }, - { name: "/health", type: "HTTP", desc: "App health route logged at startup (http://localhost:8010/health)." }, - { name: ":18888", type: "Aspire", desc: "Aspire dashboard that orchestrates the dashboard alongside services/plugins; token printed by aspire start." }, - { name: ":3001", type: "upstream", desc: "The users service the dashboard's typed client calls — same UsersContractV1, no drift." }, - { name: ":4437", type: "streams", desc: "The durable-streams Aspire service the @netscript/fresh/streams client consumes over HTTP/SSE (producer runtime in @netscript/plugin-streams-core)." } - ] -}) }} - -## Where to go next - -This hub is intentionally thin — the full generated APIs live in the reference. Pick the lane that -matches what you're doing. - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Build a service", - body: "Guided tutorial: contract → users service on :3001 → typed client → the island that renders it in the dashboard.", - href: "/tutorials/storefront/02-catalog-service/", - icon: "→" - }, - { - title: "Do — Customize the Fresh UI", - body: "Task recipe: ui:init / ui:add, the copy-source ownership model, and editing the design tokens.", - href: "/how-to/customize-fresh-ui/", - icon: "◆" - }, - { - title: "Look up — @netscript/fresh reference", - body: "The meta-framework's full generated API: defineFreshApp, definePage, the query/route/form/defer/streams builders, and the server runtime.", - href: "/reference/fresh/", - icon: "≡" - }, - { - title: "Look up — @netscript/fresh-ui reference", - body: "The component registry, foundation set, design tokens, and the ui:init / ui:add surface.", - href: "/reference/fresh-ui/", - icon: "≡" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Telemetry & logging", href: "/capabilities/telemetry/" }, next: { label: "Authentication", href: "/capabilities/auth/" } }) }} diff --git a/docs/site/capabilities/index.md b/docs/site/capabilities/index.md index de2c10a9c..cfee589f7 100644 --- a/docs/site/capabilities/index.md +++ b/docs/site/capabilities/index.md @@ -1,150 +1,4 @@ --- -layout: layouts/base.vto -title: Capabilities -templateEngine: [vento, md] -prev: null -next: { label: "Services & contracts", href: "/capabilities/services/" } +layout: layouts/redirect.vto +redirectTo: / --- - -# Capabilities - -A **capability** is a composable slice of the NetScript framework — a single, -self-contained concern (defining a service, running background jobs, ingesting a -webhook, authenticating a user, persisting state) that you add to a workspace -without rewiring the host. Each capability owns one mental model, one headline -authoring API, and — where it runs — one HTTP surface on a known port. You -compose them the way you compose small building blocks: add what the system -needs, leave out what it doesn't, and the contracts keep the seams type-safe. - -This zone is the **hub of hubs**. Every page below is a capability hub built on -the same shape — a one-screen concept, the headline API lifted verbatim from a -scaffolded project, the real endpoints and ports, and a *Learn / Do / Reference* -triplet that points you at the tutorial that teaches it, the how-to that ships -it, and the generated [API reference](/reference/) for the full surface. The -hubs stay deliberately thin so they orient rather than duplicate the reference. - -{{ comp callout { type: "note", title: "Five composable plugins + the platform underneath them" } }} -The capabilities you add are plugins. The public command installs a plugin package -(for example netscript plugin install @netscript/plugin-workers); local-source contributor -samples use deno run -A packages/cli/bin/netscript-dev.ts plugin install <kind> --name <name> --samples. -Public install emits dependency-backed glue and samples that your workspace owns; contributor installs -can materialize full local source. The remaining five are -platform capabilities: services, database, KV/queues/cron, telemetry, and the -Fresh UI come from the scaffold itself. Plugins register their contributions through their manifest; -the host application never changes. -{{ /comp }} - -## The five composable plugins - -These are the slices you bolt on with `netscript plugin install`. Each is an isolated -background processor or service that exposes its own HTTP API on a dedicated -port, wired into Aspire automatically. Bring Aspire up first — `cd aspire && -aspire start` provisions Postgres and a shared cache — `redis` by default, or `garnet` / `deno-kv` via `--cache-backend` — and starts the dashboard at -`https://localhost:18888` — **before** any `netscript db` command or plugin call, -so the plugin services and their dependencies exist when you reach for them. - -{{ comp.featureGrid({ items: [ - { title: "Background jobs", icon: "⚙️", body: "Workers run thread-isolated jobs with defineJobHandler and createSuccessResult. Trigger one over HTTP on :8091. Job dispatch, execution, and scheduler spans are real OTel automatically.", href: "/capabilities/background-jobs/" }, - { title: "Durable sagas", icon: "🔁", body: "Message-driven state machines authored with the fluent defineSaga(...).build() builder, correlated and listed on :8092. Durable store is kv or prisma.", href: "/capabilities/durable-sagas/" }, - { title: "Triggers & ingress", icon: "🪝", body: "defineWebhook turns an inbound POST into an enqueued job (enqueueJob). Typed v1 oRPC contract for trigger/event introspection + management on :8093; the webhook ingress endpoint alone stays a raw, HMAC-verifying route.", href: "/capabilities/triggers/" }, - { title: "Durable streams", icon: "🌊", body: "createDurableStream gives you a real producer runtime served on :4437 and wired into workers, auth, and sagas. The plugin-streams manifest helpers stay stubbed and fail loud.", href: "/capabilities/streams/" }, - { title: "Authentication", icon: "🔐", body: "auth-api oRPC service on :8094 with five endpoints. A pure-backend seam composing one active backend — kv-oauth (interactive), WorkOS, or better-auth.", href: "/capabilities/auth/" } -] }) }} - -{{ comp callout { type: "important", title: "Alpha status" } }} -A few seams are intentionally not-yet-live and the hubs say so plainly. The streams producer runtime is real via @netscript/plugin-streams-core (createDurableStream) — only the @netscript/plugin-streams manifest helpers (defineStreamProducer/defineStreamConsumer) fail loud but differently: defineStreamConsumer.subscribe() synchronously throws StreamUnsupportedOperationError; defineStreamProducer.publish() returns a rejected Promise with StreamUnsupportedOperationError. Worker job tracing is real — only the scaffold createJobTools(ctx) handler helpers (trace.addEvent, withChildSpan, progress) remain no-op stubs (a tracked limitation with a fix planned). Trigger enqueueJob is live; defer is defined-but-unsupported (throws and routes to the DLQ). Scaffolded JSR imports use exact {{ releaseSpecifier }} pins for the aligned alpha train. - - - -{{ /comp }} - -## The platform capabilities - -These ship with the scaffold — no `plugin install` required. Services and contracts -are the backbone the five plugins lean on; the rest are the persistence, -primitives, observability, and UI the whole workspace shares. - -{{ comp.featureGrid({ items: [ - { title: "Services & contracts", icon: "🔌", body: "defineService(router, {...}) for one-shot services, or the fluent createService(...).serve() builder. oRPC + zod contracts via implement(); RPC mounts at /api/rpc/*. Example users service on :3001.", href: "/capabilities/services/" }, - { title: "Database & Prisma", icon: "🗄️", body: "Prisma 7.8 with runtime=\"deno\" over Postgres (the recommended engine; or mysql / mssql / sqlite via --db at scaffold time). Per-plugin .prisma files aggregate under database/postgres/schema/plugins//. Requires Aspire up first.", href: "/capabilities/database/" }, - { title: "KV, queues & cron", icon: "⏱️", body: "Deno KV, queue, and cron primitives behind the workspace's unstable:[\"kv\"] flag. The queue has four backends including Postgres — the durable substrate jobs and sagas build on.", href: "/capabilities/kv-queues-cron/" }, - { title: "Telemetry & logging", icon: "🔭", body: "OpenTelemetry and structured logs are wired into handlers, RPC, job dispatch, and SSE from line one — observable by default, with traces visible in the Aspire dashboard.", href: "/capabilities/telemetry/" }, - { title: "Fresh UI & design", icon: "🎨", body: "apps/dashboard is a Fresh + Preact + Tailwind v4 + Vite frontend. Copy-source: the CLI copies components into your repo and the code is yours.", href: "/capabilities/fresh-ui/" } -] }) }} - -## The AI stack - -AI is neither one of the five composable plugins nor a scaffold platform -capability — it is a two-layer stack, and the layers have different -availability. The **durable chat UI ships today**: a durable chat route on -`@netscript/fresh/ai` plus the `fresh-ui` chat components give you a transcript, -streaming markdown, and tool-call cards that survive reload and reconnect. The -**provider-agnostic engine** — `@netscript/ai` with `@netscript/plugin-ai-core` -and `@netscript/plugin-ai` — arrives from `0.0.1-beta.2` and will replace -today's direct model wiring. - -{{ comp.featureGrid({ items: [ - { title: "AI stack", icon: "🤖", body: "Durable chat UI ships now on @netscript/fresh/ai + fresh-ui — a transcript, streaming markdown, and tool-call cards that survive reload and reconnect. The provider-agnostic engine (@netscript/ai) arrives from 0.0.1-beta.2 and replaces the direct model wiring you use today.", href: "/capabilities/ai/" } -] }) }} - -## The capability matrix - -At a glance: what each capability is for, how it runs, and where to read the full -generated API. Ports apply only to capabilities that expose an HTTP surface; the -primitive and UI capabilities run inside the workspace rather than on their own -port. Always read the generated [API reference](/reference/) for the exhaustive -surface — these hubs orient, the reference enumerates. - -{{ comp.apiTable({ - caption: "All eleven capabilities — kind, headline API, runtime surface, reference unit", - rows: [ - { name: "Services & contracts", type: "platform · :3001", desc: "defineService / createService().serve() + oRPC contracts via implement(); RPC at /api/rpc/*. Full API at /reference/service/ and /reference/contracts/." }, - { name: "Background jobs", type: "plugin · :8091", desc: "defineJobHandler + createSuccessResult; trigger via POST /api/v1/workers/jobs/{id}/trigger. Job/scheduler OTel spans are real. Full API at /reference/workers/." }, - { name: "Durable sagas", type: "plugin · :8092", desc: "defineSaga(id).durability('t1')...build(); durable store kv | prisma; list at /api/v1/sagas/sagas. Full API at /reference/sagas/." }, - { name: "Triggers & ingress", type: "plugin · :8093", desc: "defineWebhook → enqueueJob; typed v1 oRPC contract for trigger/event introspection + management; webhook ingress POST /api/v1/webhooks/:triggerId stays a raw HMAC-verifying route. defer throws + DLQs. Full API at /reference/triggers/." }, - { name: "Durable streams", type: "plugin · :4437", desc: "createDurableStream producer runtime is real; plugin-streams manifest helpers throw StreamUnsupportedOperationError. Full API at /reference/streams/." }, - { name: "Authentication", type: "plugin · :8094", desc: "auth-api oRPC; /api/v1/auth/{signin,callback,signout,session,me}; one active backend (kv-oauth | workos | better-auth). Full API at /capabilities/auth/." }, - { name: "Database & Prisma", type: "platform · Postgres", desc: "Prisma runtime=\"deno\", per-plugin schema aggregation; Postgres is the recommended engine, selected with --db postgres; mysql / mssql / sqlite are the other choices. Full API at /reference/database/." }, - { name: "KV, queues & cron", type: "platform · primitives", desc: "Deno KV / queue (four backends incl. postgres) / cron behind unstable:[\"kv\"]. Full API at /reference/kv/, /reference/queue/, /reference/cron/." }, - { name: "Telemetry & logging", type: "platform · OTel", desc: "@opentelemetry/api spans + structured logs in handlers, RPC, and job dispatch. Full API at /reference/telemetry/ and /reference/logger/." }, - { name: "Fresh UI & design", type: "platform · copy-source", desc: "Fresh + Preact + Tailwind v4 dashboard you own. Full API at /reference/fresh/ and /reference/fresh-ui/." }, - { name: "AI stack", type: "durable-chat runtime + engine", desc: "Durable chat plane runtime is @netscript/fresh/ai (ships in @netscript/fresh) with the fresh-ui chat components — available now. The provider-agnostic @netscript/ai engine arrives from 0.0.1-beta.2. Reference at /capabilities/ai/ and /reference/fresh/." } - ] -}) }} - -## How the seams fit together - -The plugins are not islands — they compose through the same contracts and durable -substrate. A **service** defines an oRPC contract and serves it at `/api/rpc/*`. -A **trigger** turns an inbound webhook into a queued job via `enqueueJob`. A -**worker** picks that job off the queue and runs it thread-isolated, emitting -real OTel spans as it goes. A **saga** drives a message-correlated state machine, -persisting its durable state to `kv` or `prisma`. A **stream** producer mirrors -execution state out of workers, auth, and sagas over the durable-streams service -on `:4437`. And **authentication** sits in front as a pure-backend seam: the core -defines `AuthBackendPort`, the three backends are pure adapters, and the plugin -composes exactly one active backend at a time. Underneath them all, the platform -capabilities — services, database, KV/queues/cron, telemetry, and the Fresh UI — -supply the contracts, persistence, primitives, and observability the whole -workspace shares. - -{{ comp callout { type: "note", title: "One active auth backend at a time" } }} -Authentication is a hard single-active-backend boundary in v1 — you select one of kv-oauth, workos, or better-auth via NETSCRIPT_AUTH_BACKEND (default kv-oauth). Only kv-oauth exposes the interactive sign-in flow; on WorkOS or better-auth the signin/callback endpoints return a typed AUTH_PROVIDER_ERROR by design. There is no multi-active routing, cross-backend linking, or global logout yet. See Authentication for the full backend capability matrix. -{{ /comp }} - -## Where to go next - -The hubs are concept-first; the rest of the site does the teaching and the -shipping. If you learn best by building, walk the [tutorials ladder](/tutorials/) -— it builds one continuous app where a service publishes a saga message, a saga -consumes it, and a webhook enqueues a job, exercising several of these -capabilities in sequence. If you have a task in hand, the -[how-to recipes](/how-to/) ship each capability directly — including -[adding authentication](/how-to/add-authentication/). For the *why* behind the -design, the [core concepts](/explanation/) zone explains the contract type flow, -the plugin system, the durability model, the [auth model](/explanation/auth-model/), -and the Aspire orchestration that ties it together. - -{{ comp callout { type: "tip", title: "Start with services" } }} -Every other capability assumes you can define a contract and serve it. Begin at Services & contracts — the backbone the five plugins compose around — then follow the matrix above into whichever slice your system needs. -{{ /comp }} diff --git a/docs/site/capabilities/kv-queues-cron.md b/docs/site/capabilities/kv-queues-cron.md index 67eb91a94..ef514c449 100644 --- a/docs/site/capabilities/kv-queues-cron.md +++ b/docs/site/capabilities/kv-queues-cron.md @@ -1,378 +1,4 @@ --- -layout: layouts/base.vto -title: KV, queues & cron -templateEngine: [vento, md] -prev: { label: "Database & Prisma", href: "/capabilities/database/" } -next: { label: "Telemetry & logging", href: "/capabilities/telemetry/" } +layout: layouts/redirect.vto +redirectTo: /data-persistence/kv-queues-cron/ --- - -# KV, queues & cron - -The integration trio. Most backends end up hand-assembling three boring-but-load-bearing -seams: a **key-value store** for cache and session state, a **message queue** for -fire-and-forget work, and a **cron scheduler** for time-driven jobs. NetScript ships all -three as provider-agnostic packages — `@netscript/kv`, `@netscript/queue`, and -`@netscript/cron` — each exposing one typed contract across several backends and -**auto-detecting** the best available adapter from the Aspire environment, with a zero-config -local fallback so the same code runs on a laptop and in production. - -{{ comp.diagram({ - src: "/assets/diagrams/queue-worker-scheduler.svg", - alt: "A cron schedule fires a tick that enqueues work onto a provider-agnostic queue; one or more worker listeners drain the queue in parallel while a KV store holds cursors, dedupe keys, and counters that keep the loop idempotent.", - caption: "The trio in one loop: cron tick → enqueue → parallel queue listeners → KV state holds the cursor, dedupe keys, and counters." -}) }} - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for KV when you need fast, typed read/write state — caches, sessions, -feature flags, counters — with optional TTL and reactive watches. Reach for a -queue when you have fire-and-forget or fan-out work that should not -block a request: emails, webhooks, image processing. Reach for cron when work -is time-driven: nightly reports, hourly cleanups, weekday digests. For -message-driven, long-running orchestration with state, step up to a -durable saga; for inbound HTTP that kicks off -work use a trigger. -{{ /comp }} - -## What it is - -Each package is built on the same NetScript opinion: a single typed contract, several adapters -behind it, and provider selection that resolves automatically from the environment so you never -branch on "is this local or Aspire?" in product code. KV unifies **Deno KV and Redis-compatible caches (`redis` by default, or `garnet` via `--cache-backend`), plus -in-memory**; the queue unifies **Deno KV, Redis, RabbitMQ (AMQP), and PostgreSQL** (plus a -KV-polling fallback for remote KV Connect endpoints); cron unifies **native `Deno.cron()` and an -in-memory** scheduler. None of the three requires you to operate Redis, RabbitMQ, or PostgreSQL -during development — the local adapters carry the same code until [Aspire](/explanation/aspire/) -provisions the real backends. When [telemetry](/capabilities/telemetry/) is on, queues are traced -for you with no manual instrumentation. - -{{ comp.featureGrid({ items: [ - { - title: "@netscript/kv", - body: "Reactive key-value storage. getKv() auto-detects Deno KV or Redis/Garnet; construct MemoryKvAdapter directly for tests. get/set/watch/atomic with per-key TTL.", - href: "/reference/kv/", - icon: "◆" - }, - { - title: "@netscript/queue", - body: "Provider-agnostic message queues. createQueue() wraps Deno KV, Redis, RabbitMQ (AMQP), and PostgreSQL behind one MessageQueue contract; createParallelQueue() adds concurrency.", - href: "/reference/queue/", - icon: "≡" - }, - { - title: "@netscript/cron", - body: "Runtime-agnostic scheduling. createScheduler() uses native Deno.cron() when available and an in-memory adapter for tests.", - href: "/reference/cron/", - icon: "◎" - } -] }) }} - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Do — Choose a queue provider", - body: "Recipe: pick between Deno KV, Redis, RabbitMQ, and PostgreSQL — and wire the Aspire-provisioned, local-fallback, and explicit-provider paths.", - href: "/how-to/choose-a-queue-provider/", - icon: "→" - }, - { - title: "Understand — Orchestration with Aspire", - body: "How the scaffold wires Redis (the default cache backend; or Garnet via --cache-backend) and message brokers as resources, and why auto-detection just works once aspire start is up.", - href: "/explanation/aspire/", - icon: "◎" - } -] }) }} - -## One API, many backends - -Each package keeps a single typed contract with several adapters behind it, and provider -selection resolves automatically from the environment. The matrix below is the full provider -surface — read across a capability to see which backends it supports and how each is chosen. Note -one deliberate exception: the **PostgreSQL queue backend is explicit-provider only** — it is real -and durable, but auto-detection never selects it (see the callout below). - -{{ comp.apiTable({ - caption: "Adapter × capability — supported backends and how they are selected", - rows: [ - { name: "Deno KV", type: "kv · queue · cron", desc: "Default zero-config fallback. KV stores locally; the queue uses native Deno KV queue ops; cron shares the same local-first philosophy via Deno.cron(). No external service required." }, - { name: "memory", type: "kv · cron", desc: "Process-local. MemoryKvAdapter for KV (must be constructed explicitly) and the in-memory cron adapter (provider: 'memory') — deterministic for tests. The KV default for local development without Redis is Deno KV (local file), not the in-memory adapter." }, - { name: "Redis / Garnet", type: "kv · queue", desc: "Production cache and queue backend. Selected when CACHE_PROVIDER=redis|garnet, REDIS_URI/GARNET_URI, or Aspire services__redis__*/services__garnet__* are present. Redis is the default cache Aspire provisions; Garnet is the Redis-compatible alternative via --cache-backend garnet." }, - { name: "RabbitMQ (AMQP)", type: "queue", desc: "Durable broker for high-throughput, multi-consumer queues. Chosen first by auto-detection when Aspire reports a rabbitmq service. Imported via @netscript/queue/adapters/amqp for direct access." }, - { name: "PostgreSQL", type: "queue", desc: "Durable SQL-backed queue (FOR UPDATE SKIP LOCKED row-claim, visibility timeout, ack/nack, DLQ). EXPLICIT-PROVIDER ONLY — set provider: QueueProvider.Postgres; never auto-detected. Configure connection.postgres.{url,tableName}." }, - { name: "KV-polling", type: "queue", desc: "KvPollingAdapter — used automatically when the Deno KV path is a remote HTTP/HTTPS endpoint (KV Connect), where native queue ops are unavailable. Tunable via connection.denoKv.{pollInterval,visibilityTimeout,maxRetries}." }, - { name: "Deno.cron()", type: "cron", desc: "Native runtime scheduler. Used by createScheduler() whenever the runtime exposes Deno.cron(); falls back to the in-memory adapter otherwise." } - ] -}) }} - -{{ comp callout { type: "warning", title: "PostgreSQL queue is explicit-provider only" } }} -The queue's auto-detection order is RabbitMQ → Redis → Deno KV — it never -selects PostgreSQL. The Postgres backend is a real, durable adapter -(FOR UPDATE SKIP LOCKED row-claiming, visibility timeout, ack/nack, dead-letter -store), but you must opt in explicitly with provider: QueueProvider.Postgres (or -the string 'postgres'). The url in -connection.postgres is optional — when omitted, the adapter falls back to the -Aspire-provisioned Postgres URI — and tableName defaults to -message_queue. Reach for it when you want queue state to live in the same -transactional database as your domain data. -{{ /comp }} - -{{ comp callout { type: "important", title: "Aspire first — then the production adapters appear" } }} -Auto-detection upgrades to Redis/Garnet and RabbitMQ only when those resources are -running. Bring orchestration up first: -cd aspire && aspire start provisions Postgres, Redis, and any message broker -(dashboard at https://localhost:18888) -before any netscript db command or service that expects those -backends. Without Aspire, KV falls back to local Deno KV, the queue falls back to Deno KV, and -cron falls back to the in-memory adapter — your code is identical either way. See -Orchestration with Aspire. -{{ /comp }} - -## KV — key types first - -`getKv(config?)` returns a `WatchableKv` bound to the auto-detected backend. Keys are typed -tuples, values are fully typed, and `KvSetOptions.expireIn` (milliseconds) gives per-key TTL on -every adapter. Pass a `SharedKvConfig` to override provider selection on first access. - -{{ comp.apiTable({ - caption: "SharedKvConfig — first-access overrides for getKv()", - rows: [ - { name: "provider", type: "KvProvider?", desc: "Force a specific provider. The default 'auto' inspects the environment and chooses the best available backend." }, - { name: "path", type: "string?", desc: "Deno KV path or URL." }, - { name: "redisUrl", type: "string?", desc: "Explicit Redis connection URL." }, - { name: "redisNamespace", type: "string?", desc: "Prefix used for Redis-backed keys." }, - { name: "skipServiceDiscovery", type: "boolean?", desc: "Skip environment-based provider detection." } - ] -}) }} - -{{ comp.apiTable({ - caption: "KvSetOptions — per-key write options", - rows: [ - { name: "expireIn", type: "number?", desc: "Time-to-live in milliseconds; the key is removed after the window elapses. Honoured by every adapter." } - ] -}) }} - -`getKv()` is the headline entrypoint; the same module also exposes `closeKv()`, `resetKv()` -(for tests), `getRawKv()` (the underlying `Deno.Kv` when the provider is Deno KV), -`getActiveProvider()`, `isKvInitialized()`, and the `isWatchable()` type guard. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — auto-detected getKv()", - lang: "ts", - code: "// main.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv(); // picks Redis/Garnet under Aspire, else local Deno KV\n\nawait kv.set(['users', 'alice'], { name: 'Alice', role: 'admin' });\n\nconst entry = await kv.get<{ name: string; role: string }>(['users', 'alice']);\nconsole.log(entry?.value.name); // \"Alice\"\n\n// Per-key TTL works on every adapter — expire a session after one hour.\nawait kv.set(['sessions', 'tok_abc'], { userId: 'u1' }, { expireIn: 3_600_000 });" - }, - { - label: "Reactive — watch a prefix", - lang: "ts", - code: "// watch-jobs.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv();\n\n// Stream every change under a prefix, including newly created keys.\nfor await (const event of kv.watchPrefix(['jobs', 'order-processor'])) {\n console.log(`${event.key.join('/')} -> ${event.type}`, event.value);\n}" - }, - { - label: "Atomic — compare-and-swap", - lang: "ts", - code: "// counter.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv();\n\nconst entry = await kv.get(['counters', 'visits']);\nconst result = await kv.atomic(\n [{ key: ['counters', 'visits'], versionstamp: entry?.versionstamp ?? null }],\n [{ type: 'set', key: ['counters', 'visits'], value: (entry?.value ?? 0) + 1 }],\n);\n\nif (!result.ok) {\n // A concurrent write landed since the read — retry the cycle.\n}" - }, - { - label: "Tests / explicit backend", - lang: "ts", - code: "// kv.test.ts — deterministic, process-local store for tests.\nimport { MemoryKvAdapter } from '@netscript/kv';\n\nconst kv = new MemoryKvAdapter();\nawait kv.set(['test', 'key'], 'value');\n\n// Or pin Redis explicitly via the subpath import (keeps ioredis out of other graphs).\nimport { RedisKvAdapter } from '@netscript/kv/redis';\n\nconst redisKv = new RedisKvAdapter({ url: 'redis://localhost:6379', namespace: 'myapp' });\nawait redisKv.set(['cache', 'featured'], items, { expireIn: 60_000 });" - } -] }) }} - -## Queues — key types first - -`createQueue(name, options?)` returns a `MessageQueue` over the auto-detected backend. The -factory stays synchronous; the heavy Redis, RabbitMQ (AMQP), and PostgreSQL adapters resolve -lazily on first use and never enter your module graph until then. Producers call -`enqueue(message, options?)`; consumers call `listen(handler, options?)`; `stop()` drains and -releases the connection. The `QueueProvider` enum names the four backends — -`DenoKv`, `Redis`, `RabbitMQ`, `Postgres`. - -{{ comp.apiTable({ - caption: "QueueOptions — passed to createQueue / createTypedQueue / createParallelQueue", - rows: [ - { name: "provider", type: "QueueProvider?", desc: "Pin a backend. Omit to auto-discover (RabbitMQ → Redis → Deno KV). Set QueueProvider.Postgres to opt into the explicit-only Postgres adapter." }, - { name: "autoDiscover", type: "boolean? = true", desc: "Enable Aspire-environment discovery. Priority: RabbitMQ > Redis > Deno KV." }, - { name: "retryAttempts", type: "number? = 3", desc: "Max retries for failed messages — only applies when the backend has no native retry." }, - { name: "retryDelay", type: "number? = 1000", desc: "Delay between retry attempts in milliseconds — only when the backend has no native retry." }, - { name: "connection", type: "QueueConnectionOptions?", desc: "Provider-specific connection: connection.{denoKv,redis,rabbitmq,postgres}. KV Connect tuning lives under connection.denoKv (pollInterval, visibilityTimeout, maxRetries)." }, - { name: "deadLetterStore", type: "DeadLetterStorePort?", desc: "Custom terminal-failure store. When omitted, adapters use their provider-specific durable default." }, - { name: "disableAutoTracing", type: "boolean? = false", desc: "Skip the automatic TracedQueue wrapper when you prefer to wire spans by hand." } - ] -}) }} - -`createTypedQueue(name, schema, options?)` wraps a Zod schema and validates on enqueue/dequeue; -its `TypedQueueOptions` adds `validateOnEnqueue` (default true), `validateOnDequeue` (default -true), and `onValidationError` (`'discard'` default, `'dlq'`, or `'throw'`). `createParallelQueue` -takes the same options plus a `concurrency` count. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Enqueue — producer", - lang: "ts", - code: "// enqueue-email.ts\nimport { createQueue } from '@netscript/queue';\n\n// Auto-detects: RabbitMQ (AMQP) under Aspire, else Redis, else Deno KV.\nconst emails = createQueue<{ to: string; body: string }>('emails');\n\nawait emails.enqueue({\n to: 'user@example.com',\n body: 'Welcome to NetScript.',\n});\n\n// Delay availability (e.g. a 5-minute reminder) via EnqueueOptions.\nawait emails.enqueue({ to: 'user@example.com', body: 'Still there?' }, { delay: 5 * 60 * 1000 });" - }, - { - label: "Consume — listener", - lang: "ts", - code: "// consume-email.ts\nimport { createQueue } from '@netscript/queue';\n\nconst emails = createQueue<{ to: string; body: string }>('emails');\n\n// The handler receives the message and a MessageContext (messageId, deliveryCount, ack/nack).\nawait emails.listen(async (message, context) => {\n await sendEmail(message.to, message.body);\n // Some backends ack/retry natively (queue.nativeRetrial === true).\n});\n\n// On shutdown, drain in-flight work and release the backend connection.\nawait emails.stop();" - }, - { - label: "Parallel — bounded concurrency", - lang: "ts", - code: "// parallel-jobs.ts\nimport { createParallelQueue } from '@netscript/queue';\n\n// Process up to 4 I/O-bound messages concurrently on a single listener.\nconst jobs = createParallelQueue<{ orderId: string }>('order-jobs', { concurrency: 4 });\n\nawait jobs.listen(async (message) => {\n await fulfilOrder(message.orderId); // these run in parallel, up to 4 at a time\n});\n\n// concurrency <= 1 behaves exactly like createQueue. For CPU-bound work prefer Web Workers." - }, - { - label: "Typed — runtime validation", - lang: "ts", - code: "// typed-notifications.ts\nimport { z } from 'zod';\nimport { createTypedQueue } from '@netscript/queue';\n\nconst NotificationSchema = z.object({\n type: z.enum(['email', 'sms']),\n to: z.string(),\n body: z.string(),\n});\n\n// Invalid payloads route to a dead-letter queue instead of being discarded.\nconst notifications = createTypedQueue('notifications', NotificationSchema, {\n onValidationError: 'dlq',\n});" - }, - { - label: "PostgreSQL — durable, explicit", - lang: "ts", - code: "// pg-queue.ts\nimport { createQueue, QueueProvider } from '@netscript/queue';\n\n// PostgreSQL is never auto-detected — opt in explicitly.\nconst jobs = createQueue<{ orderId: string }>('order-jobs', {\n provider: QueueProvider.Postgres, // or provider: 'postgres'\n connection: {\n postgres: {\n // url is optional — falls back to the Aspire-provisioned Postgres URI.\n url: 'postgres://app:secret@localhost:5432/app',\n tableName: 'message_queue', // default table name\n },\n },\n});\n\nawait jobs.enqueue({ orderId: 'ord_123' });" - }, - { - label: "Pin another provider", - lang: "ts", - code: "// pin-provider.ts\nimport { createQueue, QueueProvider } from '@netscript/queue';\n\n// Force Redis regardless of what auto-detection would pick.\nconst jobs = createQueue('jobs', { provider: QueueProvider.Redis });\n\n// Or tune the Deno KV / KV-polling adapter for a remote KV Connect endpoint.\nconst remote = createQueue('jobs', {\n connection: {\n denoKv: { path: 'https://kv.example.com', pollInterval: 500, visibilityTimeout: 60_000 },\n },\n});" - } -] }) }} - -{{ comp callout { type: "note", title: "Queues are traced for you" } }} -When telemetry is enabled, createQueue() auto-wraps the queue with -@netscript/telemetry, so enqueue and consume spans show up in the Aspire dashboard -alongside RPC and SSE work — no manual instrumentation. Opt out with -disableAutoTracing: true when you want to wire spans by hand. See -Telemetry & logging. -{{ /comp }} - -## Cron — key types first - -`createScheduler(options?)` returns a `CronScheduler` that uses native `Deno.cron()` when the -runtime exposes it and an in-memory adapter otherwise. -`schedule(id, expression, handler, options?)` registers a job from a standard cron expression or -a `CronPresets` constant and resolves to a `ScheduledJob`; `trigger(id)` fires it manually (handy -in tests); `unschedule(id)` removes one; `stop()` tears the scheduler down. For simple -single-scheduler apps, `getScheduler()` returns a shared singleton (`stopScheduler()` resets it). -The handler may be a bare `JobHandler` or a `ContextualJobHandler` that receives a `JobContext` -(`jobId`, `scheduledTime`, `actualTime`, `attempt`, `signal`). - -{{ comp.apiTable({ - caption: "CreateSchedulerOptions — passed to createScheduler / getScheduler", - rows: [ - { name: "provider", type: "CronProvider?", desc: "Pin a provider. Omit to auto-detect by runtime — Deno.cron() when available, else the in-memory adapter." }, - { name: "tickInterval", type: "number? = 60000", desc: "Poll interval in milliseconds for the memory adapter; only used when provider is 'memory'." } - ] -}) }} - -{{ comp.apiTable({ - caption: "ScheduleOptions — per-job scheduling options", - rows: [ - { name: "timezone", type: "string? = 'UTC'", desc: "IANA timezone the cron expression is evaluated against, e.g. 'America/New_York'." }, - { name: "runOnInit", type: "boolean?", desc: "Run the handler once immediately on registration, in addition to the schedule." }, - { name: "enabled", type: "boolean? = true", desc: "Whether the job starts enabled. Register a job disabled and turn it on later." }, - { name: "backoff", type: "object?", desc: "Retry backoff: { type: 'fixed' | 'exponential' | 'linear', initialDelay, maxDelay?, multiplier? }. Delays are milliseconds." }, - { name: "maxRetries", type: "number?", desc: "Maximum retries on handler failure before the run is recorded as failed." }, - { name: "metadata", type: "Record?", desc: "Arbitrary metadata stored with the job and surfaced on its ScheduledJob record." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — preset schedule", - lang: "ts", - code: "// daily-report.ts\nimport { createScheduler, CronPresets } from '@netscript/cron';\n\nconst scheduler = createScheduler(); // native Deno.cron() when available\n\nconst report = async () => {\n await generateDailyReport();\n};\n\nawait scheduler.schedule('daily-report', CronPresets.WEEKDAYS_9AM, report, {\n timezone: 'America/New_York',\n});\n\n// Fire once on demand — useful for verifying wiring.\nawait scheduler.trigger('daily-report');\nawait scheduler.stop();" - }, - { - label: "Raw expression + context", - lang: "ts", - code: "// hourly-cleanup.ts\nimport { createScheduler } from '@netscript/cron';\n\nconst scheduler = createScheduler({ provider: 'memory', tickInterval: 100 });\n\n// Hourly cleanup; the contextual handler receives scheduling metadata.\nconst cleanup = async (context) => {\n console.log(context.jobId, context.attempt, context.scheduledTime.toISOString());\n};\n\nawait scheduler.schedule('cleanup', '0 * * * *', cleanup);" - }, - { - label: "Lifecycle events", - lang: "ts", - code: "// observe-jobs.ts\nimport { createScheduler } from '@netscript/cron';\n\nconst scheduler = createScheduler({ provider: 'memory' });\n\n// Observe jobRun / jobError / jobScheduled / jobUnscheduled.\nscheduler.on('jobRun', (event) => {\n if (!event.result.success) {\n console.error(`Job ${event.jobId} failed`, event.result.error);\n }\n});" - }, - { - label: "Validate expressions", - lang: "ts", - code: "// validate-cron.ts\nimport { isValidCronExpression, parseCronExpression } from '@netscript/cron';\n\nif (isValidCronExpression('0 9 * * 1-5')) {\n const parsed = parseCronExpression('0 9 * * 1-5');\n console.log(parsed?.hour, parsed?.dayOfWeek);\n}" - } -] }) }} - -## How they compose - -The trio is strongest together. A common pattern: a **cron** job wakes on a schedule and -**enqueues** a batch of work; **queue** consumers process each message in parallel; and a **KV** -store holds the cursor, dedupe keys, or rate-limit counters that keep the whole thing idempotent -across restarts. None of these requires you to operate Redis, RabbitMQ, or PostgreSQL during -development — the local Deno KV / in-memory adapters carry the same code until Aspire provisions -the real backends (or until you opt in to the PostgreSQL queue explicitly). - -{{ comp.apiTable({ - caption: "Picking the right primitive", - rows: [ - { name: "Key-value state", type: "@netscript/kv", desc: "Synchronous-feeling read/write state: caches, sessions, flags, counters, cursors. Use TTL for ephemerality and watches for reactivity." }, - { name: "Fire-and-forget work", type: "@netscript/queue", desc: "Decouple slow work from the request path. Fan out to multiple consumers; let the backend handle retries where nativeRetrial is true. Four backends: Deno KV, Redis, AMQP, PostgreSQL." }, - { name: "Time-driven work", type: "@netscript/cron", desc: "Run handlers on a schedule. Pair with a queue to fan a scheduled tick out into many parallel jobs." }, - { name: "Stateful orchestration", type: "durable saga", desc: "When work spans steps with correlation and compensation, a queue is not enough — model it as a durable saga instead." } - ] -}) }} - -{{ comp callout { type: "important", title: "Production notes" } }} -Review these production constraints before deploying: (1) the PostgreSQL queue is -never auto-detected — if you forget provider: QueueProvider.Postgres you -silently get the Deno KV fallback. (2) retryAttempts / -retryDelay apply only when the backend lacks native retry; check -queue.nativeRetrial before assuming your config is in effect. (3) -KV per-key TTL is set via expireIn (milliseconds), not an absolute timestamp. -(4) cron timezone defaults to UTC — set it explicitly -for business-hours schedules. (5) always await scheduler.stop() and -await queue.stop() on shutdown so in-flight work drains and connections release. -{{ /comp }} - -## Reference → - -{{ comp.xref({ key: "ref:kv" }) }} · {{ comp.xref({ key: "ref:queue" }) }} · -{{ comp.xref({ key: "ref:cron" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Do — Choose a queue provider", - body: "Task recipe: stand up each primitive in an existing workspace, with the Aspire-provisioned, local-fallback, and explicit-PostgreSQL paths spelled out.", - href: "/how-to/choose-a-queue-provider/", - icon: "◆" - }, - { - title: "Look up — @netscript/kv reference", - body: "Full generated API: getKv, WatchableKv, KvSetOptions, SharedKvConfig, the Deno KV / memory / Redis adapters, and the auto-detection table.", - href: "/reference/kv/", - icon: "≡" - }, - { - title: "Look up — @netscript/queue reference", - body: "createQueue, createTypedQueue, createParallelQueue, QueueProvider, QueueOptions, MessageQueue, and the Deno KV / Redis / AMQP / PostgreSQL / KV-polling adapters.", - href: "/reference/queue/", - icon: "≡" - }, - { - title: "Look up — @netscript/cron reference", - body: "createScheduler, getScheduler, CronPresets, CronScheduler, ScheduleOptions, JobContext, and the Deno.cron() and memory adapters.", - href: "/reference/cron/", - icon: "≡" - }, - { - title: "Understand — Orchestration with Aspire", - body: "How the scaffold wires Redis (the default cache backend; or Garnet via --cache-backend) and message brokers as resources, and why auto-detection just works once aspire start is up.", - href: "/explanation/aspire/", - icon: "◎" - }, - { - title: "Understand — Telemetry & logging", - body: "Why queues are traced by default and how KV/queue/cron spans land in the Aspire dashboard.", - href: "/capabilities/telemetry/", - icon: "◎" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Database & Prisma", href: "/capabilities/database/" }, next: { label: "Telemetry & logging", href: "/capabilities/telemetry/" } }) }} diff --git a/docs/site/capabilities/polyglot-tasks.md b/docs/site/capabilities/polyglot-tasks.md index b39d969de..5d4d2a681 100644 --- a/docs/site/capabilities/polyglot-tasks.md +++ b/docs/site/capabilities/polyglot-tasks.md @@ -1,237 +1,4 @@ --- -layout: layouts/base.vto -title: Polyglot tasks -templateEngine: [vento, md] -prev: { label: "Typed SDK & client", href: "/capabilities/sdk/" } -next: { label: "Runtime configuration", href: "/capabilities/runtime-config/" } +layout: layouts/redirect.vto +redirectTo: /background-processing/polyglot-tasks/ --- - -# Polyglot tasks - -A **polyglot task** runs non-TypeScript work — a Python script, a .NET program, a -shell or PowerShell script, or any executable — as a **managed subprocess** spawned by the -worker runtime. NetScript hands the task its input as command-line arguments and -environment variables, captures every line of `stdout`/`stderr`, parses a final JSON line -into a structured result, and normalizes the exit code into a `TaskResult`. It is the -escape hatch for the moments your platform is otherwise all-TypeScript: an ML model in -Python, a legacy .NET DLL, a system `pwsh` script. {{ comp.badge({ status: "alpha" }) }} - -{{ comp.diagram({ src: "/assets/diagrams/polyglot-task-execution.svg", alt: "The worker runtime resolves a TaskDefinition to a runtime adapter, which builds an argv and spawns a python/node/dotnet subprocess; input flows in as args and env, the subprocess streams stdout/stderr back, and the last JSON line of stdout becomes the structured result returned to the queue and database.", caption: "A task is dispatched to a runtime adapter that spawns a subprocess. Input arrives as argv + env; the last JSON line of stdout is parsed into the result; the exit code, captured logs, and duration become a TaskResult." }) }} - -## What it is - -A **job** runs in-process TypeScript on a worker; a **task** runs a *subprocess* in -another runtime. Tasks share the worker plugin's queue, retry, and telemetry machinery -with [background jobs](/capabilities/background-jobs/) — the difference is purely the -execution surface. The `MultiRuntimeTaskExecutor` keeps a map of **runtime adapters** -(one per `TaskType`) and dispatches a `TaskDefinition` to the adapter that supports its -`type`. Each adapter builds an `argv` for its runtime (e.g. `python3 -u script.py …`, -`pwsh -File script.ps1 …`) and runs it through a Dax-backed process runner that streams -output and times the process out. This page covers that subprocess seam; for in-process -TS handlers, runtime modes, and the queue lifecycle, start at -[background jobs](/capabilities/background-jobs/). - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for a polyglot task when the work must run in another runtime — a -Python data/ML script, a .NET tool, an existing shell or PowerShell automation, or a -prebuilt binary — and you want it queued, retried, traced, and result-captured like any -other unit of work. If the work is plain TypeScript, use an in-process -background job instead — it avoids a process -spawn entirely. -{{ /comp }} - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Workers track", - body: "Tutorial Track C builds the worker plugin end to end; lesson 03 adds a polyglot task with explicit permissions.", - href: "/tutorials/erp-sync/", - icon: "→" - }, - { - title: "Do — Run a polyglot task", - body: "Task recipe: define a python (or shell) task, wire its entrypoint and permissions, and execute it through the runtime.", - href: "/how-to/run-a-polyglot-task/", - icon: "◆" - }, - { - title: "Understand — Background jobs", - body: "The queue, worker-runtime modes, retry, and shutdown machinery that polyglot tasks share with in-process jobs.", - href: "/capabilities/background-jobs/", - icon: "◎" - } -] }) }} - -## Minimal example - -A task is authored with the `defineTask` typestate builder, then run through the default -multi-runtime executor. The builder's default `runtime` is `'deno'`; call `.runtime(type)` -to target another language. Input reaches the script as **argv** (`.args(...)`) and -**environment variables** (`.env({...})`); the script returns a result by writing a single -JSON object as the **last line of `stdout`**. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Define + run a python task", - lang: "ts", - code: "// workers/tasks/score-batch.ts\nimport { defineTask } from '@netscript/plugin-workers-core/builders';\nimport { createDefaultTaskExecutor } from '@netscript/plugin-workers-core/executor';\n\n// Build a task definition: python runtime, script entrypoint, explicit inputs.\nconst scoreBatch = defineTask('score-batch')\n .runtime('python')\n .entrypoint('./scripts/score.py')\n .env({ MODEL_PATH: './models/scorer.pkl' })\n .args('--threshold', '0.8')\n .timeout(120_000) // ms; defaults to 300_000\n .build();\n\n// The executor resolves the python adapter and spawns: python3 -u ./scripts/score.py --threshold 0.8\nconst executor = createDefaultTaskExecutor();\nconst result = await executor.execute(scoreBatch, {\n onStdout: (line) => console.log('[score]', line),\n});\n\nif (result.success) {\n // result.result is the parsed JSON object from the LAST stdout line, or null.\n console.log('scored', result.result);\n} else {\n console.error('task failed', result.exitCode, result.error);\n}" - }, - { - label: "score.py (the subprocess)", - lang: "python", - code: "# scripts/score.py\nimport json, os, sys\n\n# Input arrives as argv + env (NOT stdin).\nthreshold = float(sys.argv[sys.argv.index('--threshold') + 1])\nmodel_path = os.environ['MODEL_PATH']\n\n# ... do the work ...\nscored = {'kept': 42, 'dropped': 3, 'threshold': threshold}\n\n# Any prior prints become captured logs. The result is the LAST line of stdout,\n# and must be a single JSON object (not an array) to populate result.result.\nprint('scoring complete', file=sys.stderr)\nprint(json.dumps(scored))" - } -] }) }} - -{{ comp callout { type: "important", title: "How input and output cross the process boundary" } }} -Input is passed as command-line arguments (args) and -environment variables (env) — there is no JSON-over-stdin -channel. The runtime merges Deno.env, the task's env, and the -call's options.env, and injects TRACEPARENT, -TRACESTATE, and CORRELATION_ID for trace propagation. Output is -the reverse: the subprocess returns a structured value by printing one JSON object -as the final line of stdout; the runtime parses that line into -result.result (a non-object or non-final-line value yields null). -All other lines are captured into stdout/stderr and streamed to -the onStdout/onStderr/onLog callbacks. -{{ /comp }} - -## Key types first — TaskResult - -Every task — whatever its runtime — resolves to one `TaskResult`. This is the shape your -calling code reads; check it before writing per-runtime branches. - -{{ comp.apiTable({ - caption: "TaskResult — returned by executor.execute()", - rows: [ - { name: "taskId", type: "string", desc: "The TaskDefinition.id this result belongs to." }, - { name: "status", type: "string", desc: "Lifecycle status: 'completed' | 'failed' | 'timeout' | 'cancelled' (running/pending used in-flight)." }, - { name: "success", type: "boolean", desc: "True only when status is 'completed' (exit code 0). Branch on this." }, - { name: "exitCode", type: "number", desc: "Subprocess exit code; -1 when the process never started (spawn error, cancelled, timeout)." }, - { name: "stdout", type: "string", desc: "All captured stdout lines, joined by newlines." }, - { name: "stderr", type: "string", desc: "All captured stderr lines (and the error message on a spawn failure)." }, - { name: "result", type: "Record | null", desc: "The JSON object parsed from the LAST line of stdout, or null if that line is not a JSON object." }, - { name: "error", type: "string | null", desc: "Human-readable failure message (includes exit code and first stderr line); null on success." }, - { name: "duration", type: "number", desc: "Wall-clock execution time in milliseconds." }, - { name: "startedAt / completedAt", type: "string", desc: "ISO-8601 timestamps bracketing the run." }, - { name: "attempt", type: "number", desc: "Attempt index for this execution (0 from the executor itself)." } - ] -}) }} - -## TaskDefinition & runtimes - -`defineTask(id)` returns a typestate builder; `.build()` is only callable once an -`.entrypoint(path)` (subprocess) or `.handler(fn)` (in-process) is set. The resulting -`TaskDefinition` carries the runtime `type`, entrypoint, args, env, timeout, and -permissions. The seven supported runtimes (`TaskType`) and how each is launched: - -{{ comp.apiTable({ - caption: "TaskType — built-in runtime adapters", - rows: [ - { name: "deno", type: "default", desc: "Spawns `deno run` with permission flags built from .permissions(...). The only runtime that is sandboxed by per-task Deno permissions." }, - { name: "python", type: "subprocess", desc: "Spawns `python3 -u ` (or `py` on Windows); honors metadata.pythonConfig.venvPath / pythonPath and NETSCRIPT_PYTHON_PATH." }, - { name: "dotnet", type: "subprocess", desc: "Runs a .cs via `dotnet run`, a project via `dotnet run --project`, or a built executable directly; metadata.dotnetConfig.runtimeArgs/useDotnetRun." }, - { name: "shell", type: "subprocess", desc: "Runs the entrypoint under bash (metadata.shellConfig.shell / loginShell); resolves Git Bash util paths on Windows." }, - { name: "powershell", type: "subprocess", desc: "Spawns `powershell` (Windows) or `pwsh` with -NoProfile -NonInteractive -ExecutionPolicy Bypass -File." }, - { name: "cmd", type: "subprocess", desc: "Spawns `cmd.exe /c ` (Windows)." }, - { name: "executable", type: "subprocess", desc: "Runs the entrypoint directly as a prebuilt binary with the task's args." } - ] -}) }} - -The per-task permission set passed to `.permissions(...)` mirrors the Deno permission -model. **It is enforced only for the `deno` runtime** — those keys are translated into -`--allow-*` flags on the `deno run` command line. For `python`, `shell`, and the other -external runtimes there is no Deno sandbox; the subprocess inherits the OS-level access of -the worker process. - -{{ comp.apiTable({ - caption: "permissions() options — applied to the deno runtime", - rows: [ - { name: "net", type: "boolean | string[]", desc: "--allow-net (true) or --allow-net=HOSTS (array). Network access." }, - { name: "read", type: "boolean | string[]", desc: "--allow-read or --allow-read=PATHS. Filesystem read." }, - { name: "write", type: "boolean | string[]", desc: "--allow-write or --allow-write=PATHS. Filesystem write." }, - { name: "env", type: "boolean | string[]", desc: "--allow-env or --allow-env=VARS. Environment-variable access." }, - { name: "run", type: "boolean | string[]", desc: "--allow-run or --allow-run=CMDS. Subprocess spawn." }, - { name: "ffi", type: "boolean", desc: "--allow-ffi. FFI access to native libraries." }, - { name: "import", type: "string[]", desc: "--allow-import=SPECIFIERS. Allowed dynamic-import sources." } - ] -}) }} - -{{ comp callout { type: "note", title: "Omitting permissions widens the Deno sandbox" } }} -For a deno task, calling .build() without -.permissions(...) produces an --allow-all command line. Always -pass an explicit, least-privilege permission set for untrusted or third-party Deno task -code. Non-Deno runtimes ignore these keys entirely — gate those at the OS level instead. -{{ /comp }} - -## Production notes - -{{ comp callout { type: "warning", title: "The result contract is the LAST stdout line" } }} -A subprocess returns structured data only by printing a single JSON -object as its final stdout line. A trailing log line, a pretty-print, -a JSON array, or a stray newline after the payload all cause -result.result to be null even though the task -succeeded. Print diagnostics to stderr, and emit the JSON result -last with no trailing output. Buffering can also reorder this — for Python the runtime runs -python3 -u (unbuffered) for exactly this reason; flush your own output in other -runtimes. -{{ /comp }} - -{{ comp callout { type: "important", title: "Runtimes must exist on the host; exit codes are diagnostic" } }} -Each non-Deno runtime requires its toolchain on the worker host: a Python -interpreter (or a configured venv), the .NET SDK, pwsh/powershell, -or bash. A missing interpreter surfaces as a failed task — exit code -127 is reported as command not found and 126 as -command not executable. cmd and powershell.exe are -Windows-only; shell uses bash and resolves Git Bash utility paths on Windows. -A task that exceeds its timeout resolves with status timeout and -exit code -1; passing an already-aborted signal yields -cancelled. -{{ /comp }} - -{{ comp callout { type: "note", title: "Tasks run external code — treat them as a trust boundary" } }} -Polyglot tasks spawn arbitrary processes with the worker's OS privileges. Pin entrypoints -to known scripts, prefer a pinned interpreter / venv (pythonConfig.venvPath) -over $PATH discovery, and avoid interpolating untrusted input into -args or the entrypoint path. Trace context -(TRACEPARENT/CORRELATION_ID) is injected into the subprocess -environment so cross-runtime spans stitch together in telemetry. -{{ /comp }} - -## Reference → - -The full generated API — `defineTask`, `MultiRuntimeTaskExecutor`, -`createDefaultTaskExecutor`, every runtime adapter, and the executor option types — lives -in the workers reference. - -{{ comp.xref({ key: "ref:workers" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — Workers reference", - body: "defineTask, the multi-runtime executor, runtime adapters, TaskDefinition / TaskResult / TaskExecutionOptions, and permission types.", - href: "/reference/workers/", - icon: "≡" - }, - { - title: "Do — Run a polyglot task", - body: "Step-by-step recipe for a python and a shell task with explicit permissions and result capture.", - href: "/how-to/run-a-polyglot-task/", - icon: "◆" - }, - { - title: "Understand — Background jobs", - body: "The queue, worker-runtime modes, retry, and graceful-shutdown drain shared with in-process jobs.", - href: "/capabilities/background-jobs/", - icon: "◎" - }, - { - title: "Next — Runtime configuration", - body: "How worker, service, and adapter settings resolve from schema defaults, config files, env vars, and overrides.", - href: "/capabilities/runtime-config/", - icon: "→" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Typed SDK & client", href: "/capabilities/sdk/" }, next: { label: "Runtime configuration", href: "/capabilities/runtime-config/" } }) }} diff --git a/docs/site/capabilities/runtime-config.md b/docs/site/capabilities/runtime-config.md index 935c44aa4..59bd64598 100644 --- a/docs/site/capabilities/runtime-config.md +++ b/docs/site/capabilities/runtime-config.md @@ -1,283 +1,4 @@ --- -layout: layouts/base.vto -title: Runtime configuration -templateEngine: [vento, md] -prev: { label: "Polyglot tasks", href: "/capabilities/polyglot-tasks/" } -next: { label: "Capabilities", href: "/capabilities/" } +layout: layouts/redirect.vto +redirectTo: /orchestration-runtime/runtime-config/ --- - -# Runtime configuration - -NetScript splits configuration into two layers: a **typed project config** you author -once in `netscript.config.ts` and load at startup (`@netscript/config`), and a -**hot-reloadable runtime override layer** that operators roll out without redeploying -(`@netscript/runtime-config`). Together they give you one validated, type-safe view of how -the app is wired — defaults baked into the schema, values pulled from a config file and the -environment, and last-mile operational overrides applied at runtime. - -{{ comp.diagram({ src: "/assets/diagrams/runtime-config-resolution.svg", alt: "Configuration resolution chain: schema defaults flow into the loaded config file, environment variables override typed values via resolveEnv, and hot-reloadable runtime override files apply last to produce a frozen resolved config.", caption: "Resolution chain — schema defaults → config file (defineConfig / loadConfig) → environment (resolveEnv / getEnv) → runtime override files (loadRuntimeConfig) → the validated config the framework reads." }) }} - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for project config (@netscript/config) when you need to -declare how the app is built and wired — services, ports, databases, plugins, -paths — once, with full TypeScript inference and Zod validation. Reach for -runtime overrides (@netscript/runtime-config) when an -operator needs to toggle a feature flag, disable a job/saga/trigger, or add a task -without a redeploy. For one-off typed reads of a single environment -variable, use getEnv. See also -architecture. -{{ /comp }} - -## What it is - -Project config is the **static, schema-validated topology** of a NetScript app. You author -it with `defineConfig` (or `defineConfigAsync` when the shape depends on mode/command), -NetScript validates it against a Zod schema, and `loadConfig` / `initConfig` resolve the -authored file into a fully typed `NetScriptConfig`. Environment variables feed into this -layer through `resolveEnv` / `getEnv`, which coerce and default typed values per call. - -Runtime config is a **separate, dynamic override layer**. It lives under a `runtime/` -directory of versioned JSON files (jobs, sagas, triggers, features, tasks) selected by a -`current` version pointer. `loadRuntimeConfig` reads it into a `RuntimeConfig` snapshot; -`watchRuntimeConfig` reloads on change. Missing files produce **empty defaults** so startup -never blocks on overrides that have not been rolled out yet. The two packages are -deliberately distinct: `@netscript/config` owns the build-time contract, and -`@netscript/runtime-config` owns the hot-reloadable operational layer. - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Architecture", - body: "How project config, the workspace layout, and the plugin system fit together at startup.", - href: "/explanation/architecture/", - icon: "◎" - }, - { - title: "Do — Choose a queue provider", - body: "A config-driven decision recipe: pick and wire a queue backend through project config.", - href: "/how-to/choose-a-queue-provider/", - icon: "◆" - } -] }) }} - -## Minimal example - -Author the project config once, then load and inspect it at startup. The `runtimeConfig` -section of the project config controls where the **generated** runtime schema/output lives; -the live overrides are read separately by `@netscript/runtime-config`. - -```ts -// netscript.config.ts -import { defineConfig } from '@netscript/config'; - -export default defineConfig({ - name: 'orders', - version: '1.0.0', - databases: { - active: 'postgres', - config: [{ provider: 'postgres', schema: 'database/postgres/schema' }], - }, - services: { - api: { port: 3000 }, - }, - plugins: ['@netscript/plugin-workers'], -}); -``` - -```ts -// src/bootstrap.ts — load project config + apply runtime overrides -import { initConfig, getConfig } from '@netscript/config'; -import { isFeatureEnabled, loadRuntimeConfig } from '@netscript/runtime-config'; - -// 1. Project config: validated once, then read synchronously anywhere via getConfig(). -await initConfig(); -const config = getConfig(); - -// 2. Runtime overrides: hot-reloadable JSON the operator controls (empty if unset). -const runtime = await loadRuntimeConfig(); -const rolloutOn = isFeatureEnabled(runtime, 'worker-rollout', false); - -console.log(config.name, config.services?.api?.port, rolloutOn); -``` - -## Key types first - -The project config you author is `NetScriptConfigInput`; what `loadConfig` / `initConfig` -return is the fully validated `NetScriptConfig`. These are the load-bearing fields. - -{{ comp.apiTable({ - caption: "NetScriptConfig — the validated project config (returned by loadConfig / initConfig / getConfig)", - rows: [ - { name: "name", type: "string (required)", desc: "Project name." }, - { name: "version", type: "string (required)", desc: "Project version." }, - { name: "paths", type: "PathsConfig (required)", desc: "Workspace path conventions (services, packages, apps, workers, sagas, triggers, plugins, contracts, database, tasks, deploy)." }, - { name: "databases", type: "DatabasesConfig (required)", desc: "{ active?: provider, config: DatabaseConfig[] } — configured databases and the active selector." }, - { name: "plugins", type: "string[] (required)", desc: "Enabled plugin package names or specifiers." }, - { name: "services", type: "Record?", desc: "Service configuration by service name." }, - { name: "apps", type: "Record?", desc: "Frontend application configuration by app name." }, - { name: "logging", type: "LoggingConfig?", desc: "{ level, format, timestamps, colors? } logging behavior." }, - { name: "aspire", type: "AspireConfig?", desc: "{ appHost, dashboardPort } Aspire orchestration settings." }, - { name: "sagas / triggers / gateway / sdk / deploy", type: "section objects?", desc: "Optional per-capability config sections." }, - { name: "runtimeConfig", type: "RuntimeConfigSection?", desc: "Runtime schema/config OUTPUT settings (where generated runtime schema is written) — not the live overrides." } - ] -}) }} - -The runtime override snapshot is a separate type — `RuntimeConfig` — returned by -`loadRuntimeConfig()`. Each field is an array loaded from one topic directory. - -{{ comp.apiTable({ - caption: "RuntimeConfig — the hot-reloadable override snapshot (returned by loadRuntimeConfig)", - rows: [ - { name: "jobs", type: "JobOverride[]", desc: "Job overrides from runtime/jobs/*.json (enabled, schedule, timeout, maxRetries, timezone, concurrency)." }, - { name: "sagas", type: "SagaOverride[]", desc: "Saga overrides from runtime/sagas/*.json (enabled, timeout, maxRetries, compensationTimeout)." }, - { name: "triggers", type: "TriggerOverride[]", desc: "Trigger overrides from runtime/triggers/*.json (enabled, paths)." }, - { name: "features", type: "FeatureFlag[]", desc: "Feature flags from runtime/features/*.json (id, enabled, description?, rolloutPercentage?)." }, - { name: "tasks", type: "RuntimeTask[]", desc: "Runtime task definitions from runtime/tasks/*.json (id, name, runtime, entrypoint, enabled?, timeout?, schedule?)." } - ] -}) }} - -## Resolution & precedence - -There is no single monolithic merge function — resolution happens across the two packages, -each with its own precedence. The order below is the conceptual chain the diagram depicts. - -{{ comp.apiTable({ - caption: "How a value is resolved (lowest precedence first)", - rows: [ - { name: "1. Schema defaults", type: "@netscript/config", desc: "Zod schema fills omitted fields when defineConfig validates the authored input into NetScriptConfig." }, - { name: "2. Config file", type: "loadConfig / initConfig", desc: "File search order: options.configFile → netscript.config.ts → .js → .mjs. cwd resolves from options.cwd → NETSCRIPT_PROJECT_ROOT → Deno.cwd()." }, - { name: "3. Environment", type: "resolveEnv / getEnv", desc: "Per-variable: read the env var (def.env overrides the name) and coerce by type (string|number|boolean|json); if absent, use default; if required and absent with no default, throw." }, - { name: "4. Runtime overrides", type: "loadRuntimeConfig", desc: "Hot-reloadable JSON under the runtime dir, keyed by the 'current' version pointer; applied per-id (job/saga/trigger/feature/task). Missing files → empty defaults." } - ] -}) }} - -{{ comp.apiTable({ - caption: "@netscript/config — primary functions", - rows: [ - { name: "defineConfig(input)", type: "(NetScriptConfigInput) => NetScriptConfig", desc: "Type-safe, validated config definition for netscript.config.ts. The 80% path." }, - { name: "defineConfigAsync(fn)", type: "((env: ConfigEnv) => …) => () => Promise", desc: "Async/env-aware config when the shape depends on { mode, command }." }, - { name: "loadConfig(options?)", type: "(LoadConfigOptions?) => Promise", desc: "Find, import, and validate the config file. Options: { cwd?, configFile? }." }, - { name: "initConfig(options?)", type: "(LoadConfigOptions?) => Promise", desc: "Load once and cache; call at startup. Then read synchronously via getConfig()." }, - { name: "getConfig()", type: "() => NetScriptConfig", desc: "Synchronous access to the cached config; throws if initConfig() has not run." }, - { name: "isConfigLoaded() / clearConfigCache()", type: "() => boolean / () => void", desc: "Inspect or reset the config cache (clearConfigCache is for tests/reload)." }, - { name: "resolveEnv(schema) / getEnv(name, opts?)", type: "typed env readers", desc: "Coerce env vars (string|number|boolean|json) with defaults; getEnv reads one variable." }, - { name: "getMode() / isDev() / isProd() / isTest() / hasEnv(name)", type: "env helpers", desc: "Current mode (DENO_ENV/NODE_ENV) and presence checks." }, - { name: "discoverWorkspace(root?) / findWorkspaceRoot(dir) / findMember(ws, name)", type: "workspace", desc: "Classify and locate Deno workspace members." }, - { name: "inspectConfig(target)", type: "(Partial | string) => InspectionReport", desc: "JSON-stable diagnostic report for CLI rendering." } - ] -}) }} - -{{ comp.apiTable({ - caption: "@netscript/runtime-config — primary functions", - rows: [ - { name: "loadRuntimeConfig()", type: "() => Promise", desc: "Load overrides from the runtime dir via the 'current' pointer; empty defaults when files are missing." }, - { name: "watchRuntimeConfig(onChange, opts?)", type: "(cb, { signal?, prefix? }) => void", desc: "Watch the runtime config files and invoke onChange after debounced reloads." }, - { name: "isFeatureEnabled(config, flagId, defaultValue)", type: "(RuntimeConfig, string, boolean) => boolean", desc: "Read a feature flag, falling back when the flag is absent." }, - { name: "getJobOverride / getSagaOverride / getTriggerOverride", type: "(RuntimeConfig, id) => …Override | undefined", desc: "Look up a single override by id." }, - { name: "getRuntimeTask(config, taskId)", type: "(RuntimeConfig, string) => RuntimeTask | undefined", desc: "Get a runtime task definition by id." }, - { name: "summarizeRuntimeConfig(config, prefix?)", type: "(RuntimeConfig, string?) => RuntimeConfigSummary", desc: "Structured summary of active overrides (disabled jobs/sagas/triggers/features, message lines) for caller-owned presentation." }, - { name: "RUNTIME_CONFIG_TOPICS", type: "('jobs'|'sagas'|'triggers'|'features'|'tasks')[]", desc: "The topic names backed by versioned JSON files." } - ] -}) }} - -## Runtime override layout - -The runtime override directory is resolved from the environment, then read through the -`current` pointer that names the active version of each topic file. - -{{ comp.fileTree({ tree: { - name: "runtime", - comment: "NETSCRIPT_RUNTIME_CONFIG_DIR (else NETSCRIPT_TASKS_DIR's parent, else ./runtime)", - children: [ - { name: "current", comment: "version pointer — JSON { version, jobs, sagas, … } or plain 'x.y.z'" }, - { name: "features", children: [ { name: "v1.0.0.json", comment: "FeatureFlag[]" } ] }, - { name: "jobs", children: [ { name: "v1.0.0.json", comment: "JobOverride[]" } ] }, - { name: "sagas", children: [ { name: "v1.0.0.json", comment: "SagaOverride[]" } ] }, - { name: "triggers", children: [ { name: "v1.0.0.json", comment: "TriggerOverride[]" } ] }, - { name: "tasks", children: [ { name: "v1.0.0.json", comment: "RuntimeTask[]" } ] } - ] -} }) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Read a feature flag at startup", - lang: "ts", - code: "// src/feature-gate.ts\nimport { isFeatureEnabled, loadRuntimeConfig } from '@netscript/runtime-config';\n\nconst runtime = await loadRuntimeConfig();\n\n// Third arg is the fallback when the flag file/entry is absent.\nif (isFeatureEnabled(runtime, 'worker-rollout', false)) {\n // run the gated path\n}" - }, - { - label: "Hot-reload on change", - lang: "ts", - code: "// src/runtime-watch.ts\nimport { summarizeRuntimeConfig, watchRuntimeConfig } from '@netscript/runtime-config';\n\nconst controller = new AbortController();\n\nwatchRuntimeConfig(async (config) => {\n const summary = summarizeRuntimeConfig(config);\n // re-apply overrides; summary.messages is presentation-ready\n console.log(summary.messages.join('\\n'));\n}, { signal: controller.signal });" - }, - { - label: "Typed env reads", - lang: "ts", - code: "// src/env.ts\nimport { getEnv, resolveEnv } from '@netscript/config';\n\nconst port = getEnv('PORT', { type: 'number', default: 3000 });\n\nconst env = resolveEnv({\n DATABASE_URL: { env: 'DB_URL', required: true },\n DEBUG: { type: 'boolean', default: false },\n});" - } -] }) }} - -## Production notes - -{{ comp callout { type: "warning", title: "Two packages, two precedence rules — not one merge" } }} -There is no single function that merges defaults, file, env, and runtime -overrides into one object. @netscript/config validates and loads the -project config; per-variable env coercion is explicit via resolveEnv / -getEnv; and @netscript/runtime-config is a separate -hot-reloadable layer your code reads with loadRuntimeConfig and applies per id. -Don't expect a runtime override file to silently change your NetScriptConfig — -you read and apply the override explicitly. -{{ /comp }} - -{{ comp callout { type: "important", title: "Runtime dir resolution & missing files" } }} -The runtime override directory is resolved as -NETSCRIPT_RUNTIME_CONFIG_DIR → else the parent of -NETSCRIPT_TASKS_DIR → else ./runtime. A missing directory, -pointer, or topic file is not an error — loadRuntimeConfig -returns empty defaults so startup proceeds while operators roll overrides out -independently. Make sure your deployment sets the env var (or the working directory) you -expect, or you may silently read empty overrides. -{{ /comp }} - -{{ comp callout { type: "note", title: "getConfig() must follow initConfig()" } }} -getConfig() returns the cached config and throws if -initConfig() has not run. Call initConfig() once at startup, then -read synchronously anywhere. In tests, use clearConfigCache() to force a -reload between cases. -{{ /comp }} - -## Reference → - -This hub is intentionally thin — the full generated API surface for both packages lives in -the reference. - -{{ comp.xref({ key: "ref:runtime-config" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — @netscript/runtime-config", - body: "loadRuntimeConfig, watchRuntimeConfig, the override types, and the topic helpers.", - href: "/reference/runtime-config/", - icon: "≡" - }, - { - title: "Look up — @netscript/config", - body: "defineConfig, loadConfig/initConfig, resolveEnv/getEnv, the NetScriptConfig schema, and workspace helpers.", - href: "/reference/config/", - icon: "≡" - }, - { - title: "Understand — Architecture", - body: "Where project config, the workspace, and the plugin system sit in the framework.", - href: "/explanation/architecture/", - icon: "◎" - }, - { - title: "Browse — Capabilities", - body: "All NetScript capabilities: services, jobs, sagas, triggers, database, and more.", - href: "/capabilities/", - icon: "◆" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Polyglot tasks", href: "/capabilities/polyglot-tasks/" }, next: { label: "Capabilities", href: "/capabilities/" } }) }} diff --git a/docs/site/capabilities/sdk.md b/docs/site/capabilities/sdk.md index a1a2ec859..7f5cf56b5 100644 --- a/docs/site/capabilities/sdk.md +++ b/docs/site/capabilities/sdk.md @@ -1,228 +1,4 @@ --- -layout: layouts/base.vto -title: Typed SDK & client -templateEngine: [vento, md] -prev: { label: "Fresh meta-framework", href: "/capabilities/fresh-framework/" } -next: { label: "Polyglot tasks", href: "/capabilities/polyglot-tasks/" } +layout: layouts/redirect.vto +redirectTo: /services-sdk/sdk/ --- - -# Typed SDK & client - -The `@netscript/sdk` is the **typed client and data layer** for NetScript: it turns an -[`@orpc/contract`](/explanation/contracts/) into a discovered, type-safe service client, -wraps each contract action in a **cache-first query factory** (KV-backed SWR), and bridges -those into [TanStack Query](https://tanstack.com/query) for islands. The contract object is -the single source of truth — the *same* object the [service](/capabilities/services/) -implements is the one the client imports, so caller and server **cannot drift**. -{{ comp.badge({ status: "alpha" }) }} - -{{ comp.diagram({ - src: "/assets/diagrams/sdk-data-flow.svg", - alt: "A contract object flows into a generated typed client, then a cache-first query factory backed by Deno KV, then into a Fresh island; the same contract object is imported by the oRPC service so the two ends cannot drift.", - caption: "One contract → typed client → KV cache-first query → island. The service imports the same contract, so a renamed field is a compile error on both sides." -}) }} - -## What it is - -The SDK is layered. **L1** is the typed client: `createServiceClient()` resolves a -service URL from [Aspire service discovery](/explanation/aspire/) and returns a callable -object whose method signatures are inferred from the contract. **L2** wraps each contract -action in a query factory (`createQueryFactories`) that runs through a shared KV-backed cache -provider and exposes `queryOptions`/`mutationOptions`/`clientKey`/`key`/`getCachedEntry` per -action. **L3** is the `defineServices()` preset that builds clients, server query factories, -and frontend query utils from one contract map. You consume the layer you need and can drop -down without rewiring. See [Contracts](/explanation/contracts/) for the type-flow theory. - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for the SDK when an app or a service needs to call a typed oRPC service -and render its data with cache-first, server-hydrated queries. Use the bare -createServiceClient for service-to-service -RPC; add createQueryFactories when you want KV-backed SWR and TanStack option -helpers; reach for Fresh when an island must -hydrate that same query key on the client. -{{ /comp }} - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Build a service + client", - body: "Track D walks contract → typed client → query factory → definePage layer → island, end to end.", - href: "/tutorials/storefront/02-catalog-service/", - icon: "→" - }, - { - title: "Do — Discover a service", - body: "Task recipe: resolve a service URL from Aspire env and stand up a typed client for it.", - href: "/how-to/discover-services/", - icon: "◆" - } -] }) }} - -## Minimal example - -A single `lib/api-clients.ts` is the spine of the data layer: build the typed client from a -contract, then derive a query factory from `{ contract, client }`. Every consumer — server -loader or island — imports from here. - -```ts -// apps/playground/lib/api-clients.ts -import { createServiceClient } from '@netscript/sdk/client'; -import { createQueryFactories } from '@netscript/sdk/query'; -import { ordersContract } from '@contracts'; - -// L1 — typed client. `serviceName` resolves a URL via Aspire discovery. -export const ordersClient = createServiceClient({ - contract: ordersContract, - serviceName: 'orders', -}); - -// L2 — cache-first query factory bound to that client (KV-backed SWR). -export const api = createQueryFactories({ - orders: { contract: ordersContract, client: ordersClient }, -}); - -// Direct typed call: `.list()` is fully inferred from the contract. -const recent = await ordersClient.list({ limit: 10 }); -``` - -{{ comp callout { type: "note", title: "Plugin services add a routerName" } }} -A first-party plugin API (workers, sagas, triggers) mounts its router under a named segment, so -its client needs routerName in addition to serviceName: -createServiceClient<typeof workersContract>({ contract, serviceName: 'workers-api', routerName: 'workers' }). -Plain workspace services omit it. -{{ /comp }} - -## Key types first - -The query factory is the heart of the data layer. Each contract action becomes an -**`ActionMethod`** carrying the full cache-first surface — keys, prefetch, cached reads, and -TanStack option helpers. These are the methods you call from a `definePage` loader and an -island; reading them first explains the rest of the page. - -{{ comp.apiTable({ - caption: "ActionMethod — per-action query helpers (api.orders.list.*)", - rows: [ - { name: "queryOptions(props, options?)", type: "(input, ActionQueryOptions?) => QueryOptions", desc: "TanStack queryOptions with a typed queryKey and queryFn derived from the contract. The primary helper a loader/island passes to useQuery." }, - { name: "mutationOptions(options?)", type: "(ActionMutationOptions?) => MutationOptions", desc: "TanStack mutationOptions with a typed mutationKey and mutationFn for writes (e.g. updateStatus)." }, - { name: "key(props)", type: "(input) => readonly [string, action, string]", desc: "Canonical SERVER cache key [resource, action, serializedInput] used by the KV cache." }, - { name: "clientKey(props?)", type: "(input?) => readonly unknown[]", desc: "CLIENT-side TanStack query key, prefix-matchable for invalidateQueries. clientKey() vs key() is the server↔client split." }, - { name: "getCachedEntry(props)", type: "(input) => Promise | null>", desc: "Read cached data WITH its cache timestamp (cachedAt) — the SWR primitive a layer loader uses to decide stale-reload." }, - { name: "getCachedData(props)", type: "(input) => Promise", desc: "Read cached data only, without fetching." }, - { name: "prefetch(props, options?)", type: "(input, QueryParams?) => void", desc: "Warm the cache in the background (fire-and-forget)." }, - { name: "invalidate()", type: "() => Promise", desc: "Invalidate all cached queries for this action." } - ] -}) }} - -## Constructing the client & factory - -The two L1/L2 constructors and their options. `createServiceClient` is the only call that -touches discovery; `createQueryFactories` is pure wiring over `{ contract, client }`. - -{{ comp.apiTable({ - caption: "createServiceClient(options) — CreateServiceClientOptions", - rows: [ - { name: "contract", type: "TContract — required", desc: "The @orpc/contract object. Drives both client typing AND HTTP method inference." }, - { name: "serviceName", type: "string — required", desc: "Service name registered in Aspire / NetScript config. Resolved to a URL via discovery (services__{name}__{protocol}__{index}; browser uses the VITE_ mirror)." }, - { name: "routerName", type: "string?", desc: "Router-name segment for URL path construction. Required for plugin API services, omitted for plain services." }, - { name: "protocol", type: "'http' | 'https'?", desc: "Resolved protocol for service discovery." }, - { name: "apiPath / apiVersion", type: "string?", desc: "Base RPC path and API version segment overrides." }, - { name: "propagateTraceContext", type: "boolean?", desc: "Auto-propagate W3C traceparent/tracestate headers on each call." } - ] -}) }} - -The L3 alternative builds all three layers from one map: -`defineServices({ orders: { contract, serviceName: 'orders' } })` returns -`{ clients, queryFactories, queryUtils }` — the same L2 values, just composed in one call. - -## Service discovery & query client - -`serviceName` is resolved at call time from Aspire-injected environment variables. The -`@netscript/sdk/discovery` subpath exposes the resolvers directly when you need a raw URL, and -`@netscript/sdk/query-client` provides the browser-side TanStack glue. - -{{ comp.apiTable({ - caption: "Discovery & query-client helpers", - rows: [ - { name: "getServiceUrl(name, protocol, index)", type: "@netscript/sdk/discovery", desc: "Resolve a service URL from Aspire browser-VITE or server services__* env. This is how serviceName resolves." }, - { name: "getServiceInfo(name) / isServiceAvailable(name) / getAllServices()", type: "@netscript/sdk/discovery", desc: "Inspect a service's endpoints, check availability, or list all server-side Aspire service names (topology)." }, - { name: "createNetScriptQueryClient(options)", type: "@netscript/sdk/query-client", desc: "A TanStack QueryClient with server-first defaults: staleTime 30s, gcTime 300s, refetchOnWindowFocus false, retry 1." }, - { name: "createServiceQueryUtils()", type: "@netscript/sdk/query-client", desc: "Bridge a typed SDK client into oRPC/TanStack frontend query utilities." }, - { name: "bridgeInvalidation(resource, action?)", type: "@netscript/sdk/query-client", desc: "Build a client-side invalidation filter ({ queryKey }) for queryClient.invalidateQueries()." }, - { name: "toClientKeyPrefix(resource, action?)", type: "@netscript/sdk/query-client", desc: "Map a server resource/action to a prefix-matchable client query key, e.g. ['orders','list']." }, - { name: "cacheQuery.setCachedData(key, data, ttl)", type: "@netscript/sdk/cache", desc: "Server-only: fire-and-forget pre-warm of an entity into the KV cache. Importing /cache auto-registers the shared provider." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Server loader (definePage layer)", - lang: "ts", - code: "// routes/(dashboard)/orders/(_loaders)/orders-list.ts\nimport { api } from '@/lib/api-clients.ts';\n\n// Cache-first: read the KV entry (with cachedAt) for SWR; fall back to a fetch.\nexport const loadOrders = async () => {\n const entry = await api.orders.list.getCachedEntry({ limit: 20 });\n if (entry) return entry; // serve cached; SDK reloads stale in the background\n return { data: await api.orders.list.queryOptions({ limit: 20 }).queryFn(), cachedAt: Date.now() };\n};" - }, - { - label: "Island (TanStack hydration)", - lang: "tsx", - code: "// orders/(_islands)/OrdersQueryIsland.tsx\nimport { useQuery, useQueryClient } from '@netscript/fresh/query';\nimport { api } from '@/lib/api-clients.ts';\n\n// Same contract action → same query key as the server loader, so the island\n// hydrates from server state instead of refetching on mount.\nconst OrdersList = () => {\n const qc = useQueryClient();\n const orders = useQuery(api.orders.list.queryOptions({ limit: 20 }));\n // invalidate by prefix after a write:\n const refresh = () => qc.invalidateQueries({ queryKey: api.orders.list.clientKey() });\n return null; // render orders.data\n};" - }, - { - label: "Safe error narrowing", - lang: "ts", - code: "// services/orders/src/routers/v1.ts — service-to-service call\nimport { safe, isDefinedError } from '@netscript/sdk/client';\nimport { usersClient } from '@/lib/api-clients.ts';\n\nconst [error, user, isDefined] = await safe(usersClient.getById({ id }));\nif (error) {\n // narrow to a typed, contract-declared error\n if (isDefinedError(error)) return { code: error.code, status: error.status };\n throw error;\n}" - } -] }) }} - -## Production notes - -{{ comp callout { type: "important", title: "/cache and /discovery are server-only" } }} -@netscript/sdk/cache touches Deno KV and @netscript/sdk/discovery -reads services__* env on the server — never import them into island/browser code. -Browsers resolve service URLs through the VITE_-mirrored keys -that Aspire injects, and the query factory's cache reads happen on the server loader; the -island receives hydrated state and a clientKey() for invalidation only. -{{ /comp }} - -{{ comp callout { type: "warning", title: "clientKey() vs key() — don't cross the streams" } }} -key(props) is the server KV cache key -([resource, action, serializedInput]); clientKey(props?) is the -client TanStack key (prefix-matchable). Invalidate on the client with -clientKey() / toClientKeyPrefix(...) — passing a server -key() to invalidateQueries will silently match nothing. NetScript is in -alpha; subpath barrels are stable but signatures may still move. -{{ /comp }} - -## Reference → - -This hub is intentionally thin — the full generated API (client, query, query-client, cache, -collections, discovery, streams, telemetry, ports) lives in the reference. - -{{ comp.xref({ key: "ref:sdk" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — @netscript/sdk reference", - body: "Generated API for every subpath: createServiceClient, createQueryFactories, defineServices, discovery, and the cache engine.", - href: "/reference/sdk/", - icon: "≡" - }, - { - title: "Understand — Contracts", - body: "The oRPC contract → implement → handler → typed client → query → island type flow that makes drift a compile error.", - href: "/explanation/contracts/", - icon: "◎" - }, - { - title: "Build — Services & contracts", - body: "The server side of the contract: defineService, the fluent builder, OpenAPI/Scalar, and the auth seam.", - href: "/capabilities/services/", - icon: "◆" - }, - { - title: "Render — Fresh meta-framework", - body: "How a definePage layer + island consume these query factories with cache-first hydration.", - href: "/capabilities/fresh-framework/", - icon: "→" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Fresh meta-framework", href: "/capabilities/fresh-framework/" }, next: { label: "Polyglot tasks", href: "/capabilities/polyglot-tasks/" } }) }} diff --git a/docs/site/capabilities/services.md b/docs/site/capabilities/services.md index a17df598e..d6e5bf832 100644 --- a/docs/site/capabilities/services.md +++ b/docs/site/capabilities/services.md @@ -1,452 +1,4 @@ --- -layout: layouts/base.vto -title: Services & contracts -templateEngine: [vento, md] -prev: { label: "Capabilities", href: "/capabilities/" } -next: { label: "Background jobs", href: "/capabilities/background-jobs/" } +layout: layouts/redirect.vto +redirectTo: /services-sdk/services/ --- - -# Services & contracts - -A NetScript **service** is a typed HTTP runtime that *implements* an -[`@orpc/contract`](/explanation/contracts/) definition. You author the contract once -(route + zod input/output), `implement()` it, bind `.handler()`s, and serve the resulting -router on a Hono + oRPC runtime. The contract object is the single source of truth: the -same object a typed client imports is the one the server implements, so caller and server -**cannot drift**. The example `users` service answers on port **3001** with both an -OpenAPI surface (`/api/v1/users/*`) and a typed oRPC endpoint (`/api/rpc/*`). - -{{ comp.diagram({ - src: "/assets/diagrams/request-lifecycle.svg", - alt: "A browser request flows into the service router, through middleware to the matched contract handler, optionally to the database, and back as a typed response.", - caption: "Request lifecycle: browser → service router → middleware → contract handler → database → typed response." -}) }} - -{{ comp callout { type: "tip", title: "Use this when" } }} -Reach for a service when you need a synchronous, request/response API with -a versioned, type-locked contract shared between front end and back end — a CRUD surface, -an internal RPC boundary, or a public REST/OpenAPI endpoint. For fire-and-forget -work use a background job; for -message-driven, long-running orchestration use a -durable saga; for inbound HTTP that kicks -off work use a trigger; for -signed-in identity and sessions compose the -auth plugin. -{{ /comp }} - -## What it is - -A service is the **Layer 3 preset** (`defineService`) over a three-layer package. Layer 1 is -small primitives — health, error, RPC, OpenAPI, and Scalar docs handlers you can mount in any -Hono app. Layer 2 is `createService()`, a fluent builder that materializes a mountable -`ServiceApp` (via `build()`) or starts a Deno listener (via `serve()`). Layer 3 is -`defineService()`, the curated one-call preset used by generated service entrypoints. Every -layer takes the oRPC router as its input, and the same contract the router implements is the -one a typed client imports — so a renamed field is a compile error on both sides. The full -type story is in [Contracts](/explanation/contracts/). - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Build a service", - body: "Track A 02: author a contract, stand up the users service on :3001, call it from a typed client.", - href: "/tutorials/storefront/02-catalog-service/", - icon: "→" - }, - { - title: "Do — Expose OpenAPI & Scalar", - body: "Recipe: turn on the generated OpenAPI spec and the Scalar docs UI for an existing service.", - href: "/how-to/expose-openapi-scalar/", - icon: "◆" - }, - { - title: "Do — Graceful shutdown", - body: "Recipe: drain in-flight requests and run teardown hooks on SIGINT/SIGTERM before the process exits.", - href: "/how-to/graceful-shutdown/", - icon: "◆" - } -] }) }} - -## The contract is the source of truth - -Before there is a service there is a contract. A contract is plain `@orpc/contract` -routes whose inputs and outputs are zod schemas, collected into an object and passed to -`implement()` from `@orpc/server`. `implement()` returns a `.handler()`-bindable object -the service router consumes. The example workspace versions its contracts under -`contracts/versions/v1/` and re-exports them as `@/contracts` — so a contract -bump is an explicit, reviewable version directory, never an accidental break. - -```ts -// contracts/versions/v1/users.contract.ts -import { z } from 'zod'; -import { oc } from '@orpc/contract'; -import { implement } from '@orpc/server'; - -export const UsersContractV1 = { - health: oc.route({ method: 'GET' }).input(z.object({}).optional()).output(UsersHealthSchemaV1), - list: oc.route({ method: 'POST' }).input(UsersListInputSchemaV1).output(UsersListResponseSchemaV1), - updateStatus: oc.route({ method: 'POST' }).input(UsersUpdateStatusInputSchemaV1).output(UsersUpdateStatusResponseSchemaV1), -}; - -// `implement()` produces the .handler()-bindable object the service router consumes. -export const UsersV1 = implement(UsersContractV1); -``` - -The chain end-to-end — contract → `implement()` → `.handler()` → typed client → query → -island — is laid out in [Contracts](/explanation/contracts/). The key property: the -client imports the *same* contract object, so a renamed field or a changed output shape is -a **compile error in both the handler and the caller** before it can ship. - -## Headline API: two ways to construct a service - -NetScript ships **two** service-construction APIs, and the example project uses both. -Workspace services use the one-call `defineService(router, options)` — the right default -for the 80% case. Plugin API services (workers, sagas, auth) use the fluent -`createService(router, options).with*().serve()` builder when they need to add CORS, -OpenAPI, a database client, auth middleware, or custom context step by step. Both stand up -the same Hono + oRPC runtime; `defineService` is a curated preset over the same builder. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — defineService (one-shot)", - lang: "ts", - code: "// services/users/src/main.ts\nimport { defineService } from '@netscript/service';\nimport { router } from './router.ts';\n\n// One call wires CORS, request logging, OpenAPI, RPC, and health endpoints.\nawait defineService(router, {\n name: 'users',\n version: '1.0.0',\n port: parseInt(Deno.env.get('PORT') || '3001'),\n openapi: { title: 'Users API', description: 'users service' },\n debug: true,\n});" - }, - { - label: "Advanced — createService().serve() (fluent)", - lang: "ts", - code: "// plugins/workers/services/src/main.ts — step-by-step composition\nimport { createService } from '@netscript/service';\nimport { router } from './router.ts';\n\nawait createService(router, { name: 'workers', version: '1.0.0', port: 8091 })\n .withCors()\n .withLogger()\n .withOpenAPI({ title: 'Workers API' })\n .withDocs()\n .withDatabase(dbClient)\n .withContext(() => ({ workers: runtime }))\n .withRPC({ traceContext: true })\n .withHealth()\n .withServiceInfo()\n .onStartup(async () => {/* seed, warm caches */})\n .serve();" - } -] }) }} - -### `defineService(router, options)` — the preset options - -`DefineServiceOptions` extends the base `ServiceConfig` (`name`, `version`, `port`) and adds -the preset-only keys below. These are the **complete** option keys confirmed against the -package surface — nothing is omitted. - -{{ comp.apiTable({ - caption: "DefineServiceOptions (extends ServiceConfig)", - rows: [ - { name: "name", type: "string (required)", desc: "Service name used for logging, telemetry, and health-check labels." }, - { name: "version", type: "string?", desc: "Service version (e.g. '1.0.0'); surfaced on /health and the OpenAPI spec." }, - { name: "port", type: "number?", desc: "Default listener port if serve() is not passed an explicit port. The generated entrypoint reads Deno.env.get('PORT') || '3001'." }, - { name: "db", type: "DbContext?", desc: "Database context injected as context.db. Accepts a single Prisma client (with $queryRaw) or a multi-db record like { netscript, mdb, prosco, prev }; the first value exposing $queryRaw is auto-wired as the /health and /health/ready probe client." }, - { name: "openapi", type: "{ title?; description? }?", desc: "Turns on the generated OpenAPI spec endpoint and the Scalar docs UI with this title/description." }, - { name: "debug", type: "boolean?", desc: "Enables verbose oRPC logging. Defaults to the NETSCRIPT_DEBUG env var." }, - { name: "auth", type: "{ authn: AuthnOptions; authz?: AuthzOptions }?", desc: "Installs the authentication (and optional authorization) gate on guarded paths — the preset form of .withAuthn()/.withAuthz()." }, - { name: "tls", type: "ServiceTlsOptions?", desc: "Opt-in TLS: { cert, key } as PEM strings. When set, the listener serves HTTPS and negotiates HTTP/2 via ALPN automatically. Forwarded to serve() as .serve({ tls }). See TLS & HTTP/2 below." } - ] -}) }} - -{{ comp callout { type: "note", title: "Handlers bind to the contract" } }} -A router aggregates versioned handlers — export const router = { v1: { users: { ...UsersV1, health } } } — -and each handler is bound from the implemented contract: -v1.users.list.handler(async ({ input }) => { /* return records */ }). In the -scaffold's Step 5 the users handlers return seeded -in-memory records (no database yet) — the front-end ↔ contract proof. Wiring -Prisma comes later; see Database. -{{ /comp }} - -## Endpoints & ports - -A `defineService` runtime exposes its OpenAPI routes, a typed oRPC endpoint mounted under -`/api/rpc/*`, and a health check. The example `users` service is reachable once -`aspire start` is up (Aspire provisions Postgres/Redis first) or when you run it directly -with `deno task --cwd services/users dev`. - -{{ comp.apiTable({ - caption: "Users service surface (port 3001)", - rows: [ - { name: "/api/v1/users/*", type: "HTTP/OpenAPI", desc: "REST surface generated from the contract (list, updateStatus, health)." }, - { name: "/api/rpc/*", type: "oRPC", desc: "Typed RPC endpoint a generated client calls — same contract object, no drift. This is the served default path (not /rpc)." }, - { name: "/health", type: "HTTP", desc: "Liveness/readiness check; anonymous by default (excluded from authn)." }, - { name: ":3001", type: "port", desc: "Default service port; read from Deno.env.get('PORT') || '3001'." } - ] -}) }} - -{{ comp callout { type: "important", title: "Aspire first, then hit the endpoint" } }} -Services that touch the database need orchestration up first: -cd aspire && aspire start brings up Postgres and Redis (dashboard at -https://localhost:18888) before any -netscript db command. The seeded-records example service runs without the DB, -but the real workflow expects Aspire running. See Aspire. -{{ /comp }} - -## OpenAPI spec & Scalar docs - -Passing `openapi` to `defineService` (or calling `.withOpenAPI().withDocs()` on the builder) -turns on a REST surface generated from the same contract, a machine-readable OpenAPI JSON -spec, and the **Scalar** interactive docs UI. Under the hood three Layer-1 primitives do the -work, and you can mount them directly in any host Hono app when you need finer control: -`createOpenAPISpec` serves the spec JSON, `createScalarDocs` serves the Scalar HTML page, and -`createOpenAPIHandler` serves the REST routes themselves (it adds the `ZodSmartCoercionPlugin` -so query-string values coerce to their schema types automatically). - -{{ comp.apiTable({ - caption: "OpenAPI / Scalar primitives (@netscript/service)", - rows: [ - { name: "createOpenAPISpec(router, config)", type: "→ ServiceHandler", desc: "Serves the OpenAPI JSON spec. config is OpenAPIConfig (see below). Mount at e.g. /api/openapi.json." }, - { name: "createScalarDocs(options)", type: "→ ServiceHandler", desc: "Serves the Scalar docs UI HTML that loads the spec. options is ScalarDocsOptions (specUrl, title?, theme?)." }, - { name: "createScalarJs()", type: "→ ServiceHandler", desc: "Serves the bundled Scalar JS so the docs UI works offline without CDN access." }, - { name: "createOpenAPIHandler(router, config?)", type: "→ FetchHandler", desc: "The OpenAPI REST request handler (with ZodSmartCoercionPlugin). config is RPCHandlerConfig; mount under /api/*." } - ] -}) }} - -{{ comp.apiTable({ - caption: "OpenAPIConfig (createOpenAPISpec)", - rows: [ - { name: "title", type: "string (required)", desc: "API title shown in the spec and docs." }, - { name: "version", type: "string (required)", desc: "API version, e.g. '1.0.0'." }, - { name: "description", type: "string?", desc: "Longer API description." }, - { name: "servers", type: "Array<{ url; description? }>?", desc: "Server URLs advertised in the spec (e.g. staging vs. production base URLs)." } - ] -}) }} - -{{ comp.apiTable({ - caption: "ScalarDocsOptions (createScalarDocs)", - rows: [ - { name: "specUrl", type: "string (required)", desc: "URL the docs UI fetches the OpenAPI spec from, e.g. '/api/openapi.json'." }, - { name: "title", type: "string?", desc: "Docs page title." }, - { name: "theme", type: "'default' | 'kepler' | 'moon' | 'purple' | 'saturn'", desc: "Scalar UI theme." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Preset — defineService({ openapi })", - lang: "ts", - code: "// services/users/src/main.ts\nimport { defineService } from '@netscript/service';\nimport { router } from './router.ts';\n\n// Turns on the OpenAPI spec + Scalar docs UI in one option.\nawait defineService(router, {\n name: 'users',\n version: '1.0.0',\n port: 3001,\n openapi: {\n title: 'Users API',\n description: 'User management service',\n },\n});" - }, - { - label: "Primitives — mount in a host app", - lang: "ts", - code: "// host/openapi-routes.ts — wire the primitives onto an existing Hono app\nimport {\n createOpenAPISpec,\n createScalarDocs,\n createScalarJs,\n} from '@netscript/service';\nimport { router } from './router.ts';\n\napp.get('/api/openapi.json', createOpenAPISpec(router, {\n title: 'Users API',\n version: '1.0.0',\n description: 'User management service',\n}));\n\napp.get('/api/docs', createScalarDocs({\n specUrl: '/api/openapi.json',\n title: 'Users API',\n theme: 'kepler',\n}));\n\n// Serve Scalar offline (no CDN dependency)\napp.get('/api/docs/scalar.js', createScalarJs());" - } -] }) }} - -## Health, readiness & liveness - -The preset wires `/health` automatically; the underlying primitives let you compose probes -by hand or mount them in a host app. `createHealthHandler` runs every registered -`HealthCheck` in parallel and returns an aggregate `HealthResponse` -(`status` ∈ `healthy | degraded | unhealthy`). `createReadinessHandler` takes an array of -async boolean checks and is the one to point an orchestrator's *readiness* probe at — it -fails until dependencies (DB, caches) are reachable. `createLivenessHandler` is a bare -"is the process up" probe that returns 200 with no dependency checks, for an orchestrator's -*liveness* probe. The `healthChecks` namespace ships pre-built checks for the common -dependencies so you don't hand-roll them. - -{{ comp.apiTable({ - caption: "Health primitives & checks (@netscript/service)", - rows: [ - { name: "createHealthHandler(options?)", type: "→ ServiceHandler", desc: "Runs all checks in parallel, returns an aggregate HealthResponse. options: HealthHandlerOptions { checks?, version?, includeDetails? (default true) }." }, - { name: "createReadinessHandler(checks)", type: "→ ServiceHandler", desc: "Readiness probe: checks is Array<() => Promise>; reports not-ready until every check resolves true. Mount at /health/ready." }, - { name: "createLivenessHandler()", type: "→ ServiceHandler", desc: "Liveness probe: 200 OK while the process runs, no dependency checks. Mount at /health/live." }, - { name: "healthChecks.database(db)", type: "→ HealthCheck", desc: "Pre-built DB probe; runs a $queryRaw `SELECT 1`-style ping against the Prisma client." }, - { name: "healthChecks.kv()", type: "→ HealthCheck", desc: "Pre-built check for the KV store." }, - { name: "healthChecks.service(name, baseUrl)", type: "→ HealthCheck", desc: "Probes another service's health endpoint by name + base URL." }, - { name: "healthChecks.custom(name, fn)", type: "→ HealthCheck", desc: "Wraps your own async () => Promise as a named check." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Compose probes by hand", - lang: "ts", - code: "// host/health-routes.ts — readiness + liveness + aggregate health\nimport {\n createHealthHandler,\n createLivenessHandler,\n createReadinessHandler,\n healthChecks,\n} from '@netscript/service';\nimport { db } from '@database';\n\n// Aggregate health (parallel checks + details)\napp.get('/health', createHealthHandler({\n version: '1.0.0',\n checks: [healthChecks.database(db)],\n}));\n\n// Liveness: is the process up?\napp.get('/health/live', createLivenessHandler());\n\n// Readiness: are dependencies reachable?\napp.get('/health/ready', createReadinessHandler([\n async () => { await db.$queryRaw`SELECT 1`; return true; },\n]));" - } -] }) }} - -{{ comp callout { type: "note", title: "Liveness vs. readiness" } }} -Point an orchestrator's liveness probe at /health/live (restart -the container only when the process is wedged) and its readiness probe at -/health/ready (stop routing traffic until the DB and caches answer). Wiring both -to the same dependency-checking endpoint is the classic footgun: a transient DB blip then -triggers a restart instead of a brief traffic pause. -{{ /comp }} - -## Graceful shutdown - -`serve()` installs SIGINT/SIGTERM (or SIGBREAK on Windows) handlers and drains in-flight -requests before exiting. Register teardown work with `.onShutdown(hook)` on the builder — a -`ShutdownHook` receives a `ShutdownContext` (`reason`, optional `signal`) and runs during the -drain. The drain is bounded by `drainTimeoutMs` (default `30_000`); when it elapses the -service stops anyway and the resulting `ShutdownReport` records `timedOut: true` plus a -per-hook `ShutdownHookOutcome` for each registered hook. Calling `running.stop()` triggers -the same drain manually (reason `'manual'`). - -{{ comp.apiTable({ - caption: "Shutdown types (@netscript/service)", - rows: [ - { name: "ShutdownHook", type: "(context: ShutdownContext) => Promise | void", desc: "Async teardown callback registered via .onShutdown(). Run in registration order during the drain." }, - { name: "ShutdownContext", type: "{ reason: ShutdownReason; signal?: Deno.Signal }", desc: "Passed to each hook. signal is set only when reason is 'signal'." }, - { name: "ShutdownReason", type: "'signal' | 'manual' | 'startup-failure'", desc: "Why the drain started: an OS signal, a manual stop() call, or a failed startup hook." }, - { name: "ShutdownHookOutcome", type: "{ ok: boolean; error?: string }", desc: "Per-hook result; error holds a normalized message when a hook throws or rejects." }, - { name: "ShutdownReport", type: "{ reason; timedOut: boolean; hooks: readonly ShutdownHookOutcome[] }", desc: "Final result of a completed shutdown — the reason, whether the timeout elapsed, and every hook outcome in execution order." } - ] -}) }} - -{{ comp.apiTable({ - caption: "ServeOptions (serve()) — shutdown-relevant keys", - rows: [ - { name: "port", type: "number?", desc: "Preferred listener port; use 0 for an ephemeral port." }, - { name: "signal", type: "AbortSignal?", desc: "External signal that stops the listener when aborted (drives the drain)." }, - { name: "drainTimeoutMs", type: "number?", desc: "Max time to wait for in-flight requests and shutdown hooks before forcing exit. Defaults to 30_000." }, - { name: "handleSignals", type: "boolean?", desc: "Install SIGINT/SIGTERM (or SIGBREAK) handlers. Defaults to true." }, - { name: "tls", type: "ServiceTlsOptions?", desc: "Opt-in TLS ({ cert, key } PEM strings). Present → HTTPS + HTTP/2 via ALPN; absent → plain HTTP/1.1. See TLS & HTTP/2 below." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "onShutdown teardown", - lang: "ts", - code: "// services/users/src/main.ts — drain + teardown on signal or manual stop\nimport { createService } from '@netscript/service';\nimport { router } from './router.ts';\nimport { db } from '@database';\n\nconst running = await createService(router, { name: 'users', version: '1.0.0' })\n .withRPC()\n .withHealth()\n .onShutdown(async ({ reason, signal }) => {\n // reason: 'signal' | 'manual' | 'startup-failure'\n audit.record({ event: 'shutdown', reason, signal });\n await db.$disconnect();\n })\n .serve({\n port: 3001,\n drainTimeoutMs: 10_000, // wait up to 10s for in-flight work\n handleSignals: true, // SIGINT/SIGTERM/SIGBREAK\n });\n\n// Later, in tests or a supervisor: trigger the same drain manually.\nawait running.stop();" - } -] }) }} - -## TLS & HTTP/2 (opt-in) - -By default a service listens over plain HTTP. Pass a `tls` option and the *same* listener serves -HTTPS and negotiates **HTTP/2 automatically via ALPN** — HTTP/1.1 stays available as a fallback, and -there is no separate `http2` flag. TLS is purely additive: a service that omits `tls` is unchanged. - -`tls` is a `ServiceTlsOptions` object whose `cert` and `key` are **PEM-encoded strings** — the -certificate chain and the private key as *contents*, not file paths. It is accepted by both -`defineService(router, { …, tls })` and the fluent builder's `.serve({ tls })`. - -```ts -// services/users/src/main.ts — serve HTTPS + HTTP/2 with an inline cert/key -import { defineService } from '@netscript/service'; -import { router } from './router.ts'; - -await defineService(router, { - name: 'users', - port: 3000, - tls: { - cert: await Deno.readTextFile('cert.pem'), // PEM contents - key: await Deno.readTextFile('key.pem'), // PEM contents - }, -}); -// Fluent equivalent: createService(router, { name }).withHealth().serve({ tls: { cert, key } }) -``` - -If you omit `tls`, the listener falls back to environment configuration: set **both** -`NETSCRIPT_TLS_CERT_FILE` and `NETSCRIPT_TLS_KEY_FILE` and the service serves HTTPS from those files. -Unlike the inline `cert`/`key` (PEM contents), these env vars are **file paths** — and both are -required; one alone is ignored. - -{{ comp.apiTable({ - caption: "ServiceTlsOptions (@netscript/service) + env fallback", - rows: [ - { name: "cert", type: "string (required)", desc: "PEM-encoded certificate chain — the contents, not a path." }, - { name: "key", type: "string (required)", desc: "PEM-encoded private key — the contents, not a path." }, - { name: "NETSCRIPT_TLS_CERT_FILE", type: "env (path)", desc: "Fallback when tls is omitted: file path to the PEM certificate chain. Both env vars must be set together." }, - { name: "NETSCRIPT_TLS_KEY_FILE", type: "env (path)", desc: "Fallback when tls is omitted: file path to the PEM private key. Both env vars must be set together." } - ] -}) }} - -## Service-layer authn / authz middleware - -The service builder ships a **provider-agnostic** authentication and authorization seam in -`@netscript/service/auth`. It is a thin Hono-middleware layer over the request pipeline — -deliberately **distinct from the [auth plugin](/capabilities/auth/)**, which composes a -sign-in/session backend (kv-oauth, WorkOS, better-auth). Use this seam when a service needs -to gate its own routes — verify a credential, trust an upstream identity header, or check a -scope — without taking on an interactive identity provider. - -Two stages wrap the request: `.withAuthn()` resolves a `Principal` (authentication) via an -`AuthenticatorPort`, and `.withAuthz()` makes an `AuthzDecision` from that principal -(authorization) via an `AuthorizerPort`. By default the `/api` surface is protected and -`/health` is anonymous — both configurable through `AuthnOptions.protect` / -`AuthnOptions.allowAnonymous`. The preset form is `defineService(router, { auth: { authn, authz } })`. - -{{ comp.apiTable({ - caption: "@netscript/service/auth surface", - rows: [ - { name: "createStaticCredentialAuthenticator", type: "authenticator", desc: "Matches a configured static credential (e.g. API key / shared secret) → Principal." }, - { name: "createTrustedHeaderAuthenticator", type: "authenticator", desc: "Trusts an identity asserted by an upstream proxy header (e.g. behind a gateway)." }, - { name: "createScopeAuthorizer", type: "authorizer", desc: "Allows/denies a request by checking the Principal's scopes against required scopes." }, - { name: ".withAuthn({ authenticator, protect?, allowAnonymous? })", type: "builder method", desc: "Installs the authentication gate (AuthnOptions). protect defaults to ['/api']; allowAnonymous defaults to ['/health']." }, - { name: ".withAuthz({ authorizer, denyByDefault? })", type: "builder method", desc: "Installs the authorization gate (AuthzOptions) from the Principal. denyByDefault fails closed and defaults to true." } - ] -}) }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "Fluent — withAuthn / withAuthz", - lang: "ts", - code: "// Gate a service with a static credential + scope check\nimport { createService } from '@netscript/service';\nimport {\n createStaticCredentialAuthenticator,\n createScopeAuthorizer,\n} from '@netscript/service/auth';\nimport { router } from './router.ts';\n\nconst authenticator = createStaticCredentialAuthenticator({\n credentials: {\n [Deno.env.get('SERVICE_API_KEY') ?? '']: {\n subject: 'service:ci',\n scopes: ['users:write'],\n },\n },\n});\nconst authorizer = createScopeAuthorizer({\n rules: [{ match: () => true, requireScopes: ['users:write'] }],\n denyByDefault: true,\n});\n\nawait createService(router, { name: 'users', version: '1.0.0', port: 3001 })\n .withRPC()\n .withAuthn({ authenticator })\n .withAuthz({ authorizer })\n .withHealth() // /health stays anonymous\n .serve();" - }, - { - label: "Trusted-header (behind a gateway)", - lang: "ts", - code: "// Trust identity asserted by an upstream proxy/gateway\nimport { createTrustedHeaderAuthenticator } from '@netscript/service/auth';\n\n// The Principal is assembled internally from these header NAMES — point at the\n// headers your gateway sets; you do not map values yourself.\nconst authenticator = createTrustedHeaderAuthenticator({\n subjectHeader: 'x-forwarded-user',\n scopesHeader: 'x-forwarded-scopes', // optional; space/comma-separated\n});\n\n// then: .withAuthn({ authenticator })" - } -] }) }} - -{{ comp callout { type: "note", title: "Service-auth seam vs. the auth plugin" } }} -This middleware is for machine-to-machine and gateway-fronted gating: static -credentials, trusted upstream headers, scope checks. It does not do interactive -sign-in, OAuth callbacks, or session cookies. For signed-in human users -(sign-in / callback / session / me), compose the -auth plugin backend instead — the two layers -are complementary and can run together. -{{ /comp }} - -## Production notes - -{{ comp callout { type: "warning", title: "Footguns before you ship" } }} -
    -
  • Don't wire liveness to dependency checks. /health/live must -stay dependency-free; only /health/ready should fail on an unreachable DB. Crossing -them turns a transient DB blip into a container restart loop.
    • -
  • Set drainTimeoutMs below your platform's kill grace. The drain -defaults to 30_000; if your orchestrator sends SIGKILL sooner, in-flight requests -are cut off — pick a timeout under the platform's termination grace period.
    • -
  • The served RPC path is /api/rpc/*, not /rpc. -Point generated clients at the same path the runtime mounts.
    • -
  • denyByDefault is true. With .withAuthz() -installed, any request that reaches no matching rule is rejected — add an explicit allow rule for -public-but-authenticated routes rather than relaxing the default.
    • -
  • Aspire first for DB-backed services. aspire start must bring up -Postgres/Redis before any netscript db command or the readiness probe will fail.
    • -
-{{ /comp }} - -## Reference - -This hub is intentionally thin — the full generated API lives in the reference. - -{{ comp.xref({ key: "ref:service" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — @netscript/service", - body: "The full generated API: defineService, createService, the fluent builder, the OpenAPI/Scalar and health primitives, the shutdown types, and the /auth middleware seam.", - href: "/reference/service/", - icon: "≡" - }, - { - title: "Look up — @netscript/contracts", - body: "Contract primitives plus the /crud, /query, and /transform helpers that generate CRUD contracts, paginated queries, and response transformers.", - href: "/reference/contracts/", - icon: "≡" - }, - { - title: "Understand — Contracts", - body: "The oRPC contract → implement → handler → typed client → query → island type flow, explained.", - href: "/explanation/contracts/", - icon: "◎" - }, - { - title: "Learn — Build a service", - body: "Guided tutorial: contract → users service on :3001 → typed client → island, built from scratch.", - href: "/tutorials/storefront/02-catalog-service/", - icon: "→" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Capabilities", href: "/capabilities/" }, next: { label: "Background jobs", href: "/capabilities/background-jobs/" } }) }} diff --git a/docs/site/capabilities/streams.md b/docs/site/capabilities/streams.md index 9ef0387cc..3c2551cbb 100644 --- a/docs/site/capabilities/streams.md +++ b/docs/site/capabilities/streams.md @@ -1,410 +1,4 @@ --- -layout: layouts/base.vto -title: Durable streams -templateEngine: [vento, md] -prev: { label: "Triggers & ingress", href: "/capabilities/triggers/" } -next: { label: "Database & Prisma", href: "/capabilities/database/" } +layout: layouts/redirect.vto +redirectTo: /durable-workflows/streams/ --- - -# Durable streams - -A NetScript **stream** is a typed, durable change-log: producers write entity -state into a durable-stream server, and any number of HTTP/SSE consumers -materialize the latest value per key. {{ comp.badge({ status: "alpha" }) }} - -{{ comp.diagram({ - src: "/assets/diagrams/streams-pipeline.svg", - alt: "A producer defines a typed stream schema and writes upsert/delete operations into the durable-stream server on port 4437; the durable log fans out over HTTP/SSE to Fresh consumers that materialize the latest value per key.", - caption: "Streams pipeline: producer (defineStreamSchema + createDurableStream) → durable log on :4437 → HTTP/SSE → Fresh consumers (latest value per key)." -}) }} - -NetScript's streams capability is the typed, change-data backbone the other -plugins lean on — workers, sagas, and the auth service all publish their live -state through it. The producer half is implemented: you define a -typed stream schema with [`defineStreamSchema`](/reference/streams/), open a -producer with `createDurableStream` (or the Service-facing -`createServiceStreamProducer`), and `upsert`/`delete`/`flush` entity state -over a durable-stream server that runs as an Aspire resource on port **:4437**. - -The producer runtime in `@netscript/plugin-streams-core` writes through -`@durable-streams/client` with idempotent delivery. The topic-centric -**manifest sugar** in `@netscript/plugin-streams` -(`defineStreamProducer` / `defineStreamConsumer`) is **not** wired to a -transport: a producer's `publish()` returns a **rejected** promise and a -consumer's `subscribe()` **throws** synchronously — both with -`StreamUnsupportedOperationError`, pointing you at the core package. There is -also no in-process consumer `subscribe()` yet — consumption is over the -durable-stream server's HTTP/SSE protocol, which Fresh clients read. Build -against the core producer package, not the manifest helpers. - -{{ comp callout { type: "important", title: "Status — producers write via the core package; manifest helpers fail loud" } }} -The producer is implemented: createDurableStream(...) from -@netscript/plugin-streams-core writes upsert/delete/flush -through @durable-streams/client to the :4437 Aspire service, and workers, -sagas, and auth already mirror their state through it. What is not supported: the -manifest helpers defineStreamProducer/defineStreamConsumer in -@netscript/plugin-streams fail loud — a producer's publish() returns a -rejected promise and a consumer's subscribe() throws -synchronously, both with StreamUnsupportedOperationError — use -@netscript/plugin-streams-core instead. There is also no -in-process consumer subscribe(); consumption is via the durable-stream HTTP/SSE server -(read by Fresh clients). -{{ /comp }} - -## What it is - -A NetScript stream is an **entity-oriented change log**. You describe a set of -collections — each a named entity type with a primary key — and the producer -publishes `upsert` and `delete` operations keyed by that primary key. Downstream -readers materialize the latest value per key and observe a live, replayable view -of your domain state. This is the same contracts-first instinct as oRPC services, -but applied to **state replication** instead of request/response: the schema is -the type contract that both producer and any HTTP/SSE consumer are locked to. - -Streams sit alongside the other long-running capabilities rather than replacing -them. Reach for a {{ comp.xref({ key: "cap:durable-sagas", text: "durable saga" }) }} -when you need message-driven orchestration with compensation; reach for a -{{ comp.xref({ key: "cap:triggers", text: "trigger" }) }} when inbound HTTP or a -file-watch should kick off work. A stream is the **read-model fan-out**: each of -those plugins runs a thin `streams/producer.ts` that mirrors its execution state -through `createDurableStream`, so a Fresh dashboard can watch saga, worker, and -trigger progress live without polling a request/response API. - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Learn — stream a live dashboard", - body: "Track D 05: publish execution state from a producer and read it over HTTP/SSE into a live Fresh dashboard.", - href: "/tutorials/live-dashboard/05-live-stream/", - icon: "→" - }, - { - title: "Do — add the streams plugin", - body: "Public package install adds the stream plugin dependency and user-owned glue; local netscript-dev scaffolding supports contributor-source samples.", - href: "/how-to/add-a-plugin/", - icon: "◆" - }, - { - title: "Understand — the plugin system", - body: "How thread-isolated plugins like streams register contributions and wire into the Aspire resource graph.", - href: "/explanation/plugin-system/", - icon: "◎" - } -] }) }} - -## Minimal example — produce, then consume - -The producer side is two calls: freeze a typed schema, open a stream, write -entity state. The consumer side is an HTTP/SSE read of the same `:4437` stream -path — there is no in-process `subscribe()` handle, so a Fresh island (or any -SSE client) reads the durable log directly and materializes the latest value per -key. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Define a schema (real)", - lang: "ts", - code: "// streams/executions-schema.ts\n// Author against the core package — this is the live surface.\nimport { defineStreamSchema } from '@netscript/plugin-streams-core';\nimport { z } from 'zod';\n\n// Each collection is an entity type with a primary key. The schema is the\n// type contract producers and HTTP/SSE consumers are locked to.\nexport const executionsSchema = defineStreamSchema({\n execution: {\n schema: z.object({\n id: z.string().min(1),\n status: z.enum(['queued', 'running', 'succeeded', 'failed']),\n updatedAt: z.string().datetime(),\n }),\n type: 'execution',\n primaryKey: 'id',\n },\n});" - }, - { - label: "Open a producer & write (real)", - lang: "ts", - code: "// streams/producer.ts\nimport { createDurableStream } from '@netscript/plugin-streams-core';\nimport { executionsSchema } from './executions-schema.ts';\n\n// createDurableStream returns a singleton producer per streamPath and begins\n// connecting to the :4437 durable-stream server immediately.\nconst producer = createDurableStream({\n streamPath: '/workers/executions',\n schema: executionsSchema,\n producerId: 'workers-service',\n});\n\n// upsert/delete are synchronous enqueues keyed by the collection primary key.\nproducer.upsert('execution', {\n id: 'exec-1',\n status: 'running',\n updatedAt: new Date().toISOString(),\n});\nproducer.delete('execution', 'exec-0');\n\n// flush before graceful shutdown; it rethrows the connect error if the\n// producer never connected (see known limitations).\nawait producer.flush();" - }, - { - label: "Consume over HTTP/SSE (Fresh client)", - lang: "ts", - code: "// islands/ExecutionsView.tsx — read the durable log directly\nimport { getStreamsUrl } from '@netscript/plugin-streams-core';\n\n// There is no in-process subscribe(); consumption is an HTTP/SSE read of the\n// same stream path the producer writes to. getStreamsUrl resolves the :4437\n// base from Aspire discovery / VITE env (see runtime resolvers below).\nconst base = getStreamsUrl();\nconst source = new EventSource(`${base}/workers/executions`);\n\nconst latest = new Map(); // materialize latest value per key\nsource.onmessage = (ev) => {\n const change = JSON.parse(ev.data) as { key: string; value?: unknown };\n if (change.value === undefined) latest.delete(change.key);\n else latest.set(change.key, change.value);\n};" - }, - { - label: "Manifest helpers (fail loud)", - lang: "ts", - code: "// @netscript/plugin-streams (the manifest root) re-exports topic-centric\n// helpers that are NOT implemented. They fail loud, by design.\nimport {\n defineStreamProducer,\n defineStreamConsumer,\n} from '@netscript/plugin-streams';\n\n// The producer handle is returned, but publish() rejects.\nconst producer = defineStreamProducer(/* topic */);\n// await producer.publish(event) // -> rejects with StreamUnsupportedOperationError\n\n// The consumer's subscribe() throws synchronously the moment you call it.\nconst consumer = defineStreamConsumer(/* topic */);\n// consumer.subscribe(handler) // -> throws StreamUnsupportedOperationError\n\n// Correct path: import createDurableStream / defineStreamSchema from\n// '@netscript/plugin-streams-core' instead (see the other tabs)." - } -] }) }} - -{{ comp callout { type: "warning", title: "Do not call the manifest topic helpers" } }} -defineStreamProducer and defineStreamConsumer exported from -@netscript/plugin-streams are not implemented: a producer's -publish() returns a rejected promise and a consumer's -subscribe() throws synchronously, both with -StreamUnsupportedOperationError — they are not silent no-ops and they will not publish -anything. Always reach -for createDurableStream / defineStreamSchema from -@netscript/plugin-streams-core for real producer work. See the -streams reference for the full export map. -{{ /comp }} - -## Key types first — the stream definition API - -A stream schema is a map of **collections**. Each collection is a -`CollectionDefinition` — a Standard-Schema validator, a State-Protocol `type` -discriminator, and the `primaryKey` property the producer keys writes by. -`defineStreamSchema(collections)` freezes that map into a `StateSchema` that both -the producer and any HTTP/SSE consumer are locked to. - -{{ comp.apiTable({ - caption: "CollectionDefinition — one entry per collection in defineStreamSchema", - rows: [ - { name: "schema", type: "unknown (Standard Schema validator)", desc: "Standard-Schema-compatible validator (e.g. a zod object) used by durable-streams to validate the collection payload." }, - { name: "type", type: "string (required)", desc: "State Protocol type discriminator emitted for every event in this collection (e.g. 'execution')." }, - { name: "primaryKey", type: "string (required)", desc: "Property name on the value used as the entity primary key; upsert/delete are keyed by this property." } - ] -}) }} - -`defineStreamSchema` returns a frozen `StateSchema` — the durable-streams -runtime attaches per-collection event helpers (`insert`/`update`/`upsert`/`delete`, -the `CollectionEventHelpers`) so the schema can both validate and emit -State-Protocol `ChangeEvent`s. The supported `Operation` set is -`insert | update | delete | upsert`. - -## Producer options — `createDurableStream` - -`createDurableStream(options)` takes a `DurableStreamProducerOptions` and returns -a `DurableStreamProducer`. It is a **singleton factory keyed by `streamPath`**: -calling it twice with the same path returns the same live producer (a closed one -is replaced). Writes are idempotent via `@durable-streams/client`'s -`IdempotentProducer` (stable `producerId` + auto-claim), so duplicate enqueues do -not double-apply downstream. - -{{ comp.apiTable({ - caption: "DurableStreamProducerOptions (createDurableStream argument)", - rows: [ - { name: "streamPath", type: "string (required)", desc: "Stream path relative to the base URL, e.g. '/workers/executions'. This is the singleton key and the path consumers read over HTTP/SSE." }, - { name: "schema", type: "StateSchema (required)", desc: "The frozen schema returned by defineStreamSchema; binds the producer to its collection map." }, - { name: "producerId", type: "string (required)", desc: "Stable producer identity used for idempotent delivery (IdempotentProducer auto-claim). Keep it stable across restarts for duplicate-safe writes." }, - { name: "signal", type: "AbortSignal?", desc: "Optional abort signal consulted while opening the stream connection; aborting cancels the connect (an AbortError is treated as expected, not a failure)." } - ] -}) }} - -The `DurableStreamProducer` it returns exposes a small, synchronous-write surface -with an async flush/close for shutdown. (`StreamProducerPort` is the -implemented-by interface — the same four members, with `entityType` widened to -`string`.) - -{{ comp.apiTable({ - title: "DurableStreamProducer — methods", - columns: ["Member", "Shape", "Behavior"], - rows: [ - ["upsert(entityType, value)", "(K, Record) => void", "Enqueue an upsert keyed by the collection primaryKey; skipped (warns) if the key is missing/empty or the collection is unknown."], - ["delete(entityType, key)", "(K, string) => void", "Enqueue a delete by primary key; skipped (warns) on an empty key."], - ["flush()", "() => Promise<void>", "Await pending writes before shutdown; rethrows the connect error if the producer never connected."], - ["close()", "() => Promise<void>", "Flush, close the underlying handle, and release the singleton for this streamPath."], - ["streamPath", "string (readonly)", "The stream path this producer owns."], - ["closed", "boolean (get)", "Whether shutdown has begun; further upsert/delete calls are ignored."] - ] -}) }} - -## Service-side producers — `createServiceStreamProducer` - -When the writer is a backend **Service** — for example an ingestion worker that -emits a `doc.ready` completion event from the callback where the work finishes — -reach for `createServiceStreamProducer` instead of wiring `createDurableStream` -and the URL/auth resolvers by hand. It is a thin wrapper over `createDurableStream` -that reuses the exact same singleton producer and Aspire discovery -(`getStreamsUrl` / `getStreamsAuth`), and it adds one guard: `assertResolvable` -(default `true`) eagerly resolves the streams URL and auth at construction, so a -Service that forgot to declare the `streams` reference **throws immediately** -rather than silently dropping every write. - -```ts -// services/ingestion/emit-completion.ts -import { - createServiceStreamProducer, - defineStreamSchema, -} from "@netscript/plugin-streams-core"; -import { z } from "zod"; - -const completions = defineStreamSchema({ - completion: { - schema: z.object({ id: z.string().min(1), status: z.string() }), - type: "completion", - primaryKey: "id", - }, -}); - -// assertResolvable defaults to true: this throws now if neither -// DURABLE_STREAMS_URL nor services__streams__http__0 is wired. -const producer = createServiceStreamProducer({ - streamPath: "/eischat/completions", - schema: completions, - producerId: "eischat-ingestion", -}); - -producer.upsert("completion", { id: "run-1", status: "done" }); -await producer.flush(); -``` - -A Service that declares `ServiceReferences: ["streams"]` always has the streams -endpoint wired, so the guard resolves without throwing. Set `assertResolvable: -false` to keep the tolerant, lazy-connect behavior of `createDurableStream` when -a Service may legitimately start before the streams service is reachable. -`ServiceStreamProducerOptions` is `DurableStreamProducerOptions` plus this -optional `assertResolvable` flag; the returned `DurableStreamProducer` is -identical, so the write/flush/close surface below applies unchanged. - -## Runtime & transport — URL and auth resolution - -The producer never hardcodes a host. `createDurableStream` resolves the -durable-stream base URL through `getStreamsUrl()` and the auth header through -`getStreamsAuth()`, both of which read the environment so the same code works -under Aspire, in a browser build, or against an explicit override. `buildStreamUrl` -joins a stream path onto that base, and `inspectStreamTopic` returns a JSON-stable -diagnostic report for a schema (handy in tests and CLI doctors). - -{{ comp.apiTable({ - caption: "Runtime resolvers (@netscript/plugin-streams-core)", - rows: [ - { name: "getStreamsUrl()", type: "() => string", desc: "Resolves the durable-stream base URL. Server: DURABLE_STREAMS_URL override, else Aspire's services__streams__http__0 discovery var. Browser: VITE_services__streams__http__0 (or the VITE_STREAMS_URL shorthand). Throws a descriptive error if none resolve." }, - { name: "getStreamsAuth()", type: "() => Record", desc: "Builds the auth header from STREAMS_SECRET (or DURABLE_STREAMS_SECRET) as { Authorization: 'Bearer ' }; returns {} when no secret is set." }, - { name: "buildStreamUrl(path, baseUrl?)", type: "(string, string?) => string", desc: "Joins a stream path onto the resolved base (or an explicit baseUrl), trimming a trailing slash on the base." }, - { name: "inspectStreamTopic(input)", type: "(input) => StreamTopicInspectionReport", desc: "Diagnostic: returns a JSON-stable report (package, target, summary, details with collections/streamPath/producerId) for a schema + optional producer metadata." } - ] -}) }} - -{{ comp.apiTable({ - caption: "Environment variables read by the resolvers", - rows: [ - { name: "DURABLE_STREAMS_URL", type: "server override", desc: "Explicit base URL; takes precedence over Aspire discovery (e.g. http://localhost:4437)." }, - { name: "services__streams__http__0", type: "Aspire (server)", desc: "Injected by the Aspire resource graph; the default server-side discovery path." }, - { name: "VITE_services__streams__http__0", type: "browser", desc: "Vite-injected reference for browser/Fresh consumers; VITE_STREAMS_URL is the convenience shorthand." }, - { name: "STREAMS_SECRET / DURABLE_STREAMS_SECRET", type: "auth", desc: "Bearer secret for getStreamsAuth(); when set, every connect sends Authorization: Bearer ." } - ] -}) }} - -## Known limitations - -Be deliberate about what the alpha producer does and does not guarantee. - - -{{ comp callout { type: "warning", title: "Writes are dropped after a connect failure (no reconnect)" } }} -If the producer cannot reach the :4437 durable-stream server at startup, it logs a -console.warn and then silently skips every subsequent upsert/delete -— there is no reconnect loop in the current alpha. flush() rethrows that connect error so a -graceful shutdown surfaces the failure. (Writes issued before the connection completes are -buffered and drained once it opens; the drop only applies after a connect error.) Treat a healthy -:4437 service as a hard precondition for durable delivery; do not assume buffered writes will -be replayed once the server returns. -{{ /comp }} - -{{ comp callout { type: "note", title: "No in-process consumer — read over HTTP/SSE" } }} -There is no in-process subscribe() handle. Consumption happens over the durable-stream -server's HTTP/SSE protocol, which Fresh clients read to materialize the latest value per key. Model your -read side as an HTTP/SSE consumer of the :4437 stream, not as an in-process callback. -{{ /comp }} - -{{ comp callout { type: "warning", title: "Local HTTP can limit concurrent stream consumers" } }} -The local AppHost serves the generated app over http://. Under HTTP/1.1, browsers -typically allow about six concurrent connections per origin, so several long-lived HTTP/SSE streams -from one Fresh page can starve later requests or stream subscriptions. Use HTTPS for HTTP/2 when a -local dashboard needs many simultaneous stream consumers. -{{ /comp }} - -{{ comp callout { type: "note", title: "Operator visibility is console.warn (alpha)" } }} -The producer reports skips and connect failures through console.warn today — tracked as -AP-13 architecture debt until the telemetry-integration wave supplies a structured -reporter. Until then, scrape the [DurableStreamProducer] warn lines if you need to alert on -dropped writes. -{{ /comp }} - -## Endpoints & manifest - -The streams plugin is registered as a utility/infra plugin — note it requires -**neither a database nor KV** (`requiresDb=false`, `requiresKv=false`), unlike -workers, sagas, and triggers. Its durable-stream service listens on `:4437` and -is wired into the Aspire resource graph so workers, sagas, and auth can publish -through it. The port is overridable via `STREAMS_PORT` or `PORT`. - -{{ comp.apiTable({ - title: "Streams plugin — runtime facts", - columns: ["Property", "Value"], - rows: [ - ["Plugin location", "plugins/streams/"], - ["Producer package", "@netscript/plugin-streams-core (createDurableStream, createServiceStreamProducer, defineStreamSchema)"], - ["Manifest import", "@netscript/plugin-streams — topic helpers throw StreamUnsupportedOperationError"], - ["Transport client", "@durable-streams/client (IdempotentProducer)"], - ["Dev service port", ":4437 (durable-stream Aspire service; override with STREAMS_PORT/PORT)"], - ["provider.kind", "stream · category plugin · pluginType utility"], - ["Requires DB / KV", "false / false"], - ["First-party producers", "workers, sagas, triggers, auth (each streams/producer.ts → createDurableStream)"], - ["Consumer surface", "HTTP/SSE from the :4437 server (Fresh clients) — no in-process subscribe()"] - ] -}) }} - -The plugin is referenced from `netscript.config.ts` as -`./plugins/streams/mod.ts`. Because workers, sagas, and triggers each list -`streams` in their `dependencies`, it is installed first in the dependency graph -and its `:4437` service comes up so dependent producers have somewhere to write. - -## Production notes - -{{ comp callout { type: "important", title: "Footguns before you ship" } }} -
    -
  • Build on the core package, not the manifest helpers. -createDurableStream / defineStreamSchema from -@netscript/plugin-streams-core are real; the -defineStreamProducer/defineStreamConsumer helpers in -@netscript/plugin-streams fail loud with -StreamUnsupportedOperationError.
    • -
  • A healthy :4437 server is a hard precondition. A startup connect -failure drops every later write with no reconnect — bring Aspire (or an explicit -DURABLE_STREAMS_URL) up first, and treat a flush() rejection on shutdown as a -real delivery failure, not noise.
    • -
  • Keep producerId stable across restarts. Idempotent delivery is keyed -by it; a churning id defeats the duplicate-safety guarantee.
    • -
  • Model the read side as HTTP/SSE. There is no in-process subscribe(); -resolve the base with getStreamsUrl() and consume the :4437 stream path -directly.
    • -
  • Set the auth secret on both ends. When the server expects a bearer token, export -STREAMS_SECRET (or DURABLE_STREAMS_SECRET) wherever the producer runs so -getStreamsAuth() can attach it.
    • -
-{{ /comp }} - -## Reference - -This hub is intentionally thin — the full generated API lives in the reference. - -{{ comp.xref({ key: "ref:streams" }) }} - -{{ comp callout { type: "tip", title: "Where the real surface comes from" } }} -The producer you build on lives in @netscript/plugin-streams-core: -createDurableStream, the Service-facing createServiceStreamProducer, -DurableStreamProducer, defineStreamSchema, -the buildStreamUrl/getStreamsUrl/getStreamsAuth resolvers, -inspectStreamTopic, and the StreamProducerPort / -DurableStreamProducerOptions types. The @netscript/plugin-streams manifest -re-exports streamsPlugin plus the fail-loud topic helpers. See the -full reference for every exported symbol and the -cli / scaffolding / e2e / aspire sub-paths. -{{ /comp }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — @netscript/plugin-streams-core", - body: "The generated API for createDurableStream, defineStreamSchema, the URL/auth resolvers, inspectStreamTopic, and the StreamProducerPort type.", - href: "/reference/streams/", - icon: "≡" - }, - { - title: "Learn — stream a live dashboard", - body: "Track D 05: publish execution state and read it over HTTP/SSE into a live Fresh dashboard.", - href: "/tutorials/live-dashboard/05-live-stream/", - icon: "→" - }, - { - title: "Understand — the plugin system", - body: "How thread-isolated plugins like streams register contributions and wire into the Aspire resource graph.", - href: "/explanation/plugin-system/", - icon: "◎" - }, - { - title: "Related — durable sagas", - body: "Message-driven orchestration with compensation; sagas mirror their state through a streams producer.", - href: "/capabilities/durable-sagas/", - icon: "◆" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Triggers & ingress", href: "/capabilities/triggers/" }, next: { label: "Database & Prisma", href: "/capabilities/database/" } }) }} diff --git a/docs/site/capabilities/telemetry.md b/docs/site/capabilities/telemetry.md index a25432199..390ee9460 100644 --- a/docs/site/capabilities/telemetry.md +++ b/docs/site/capabilities/telemetry.md @@ -1,436 +1,4 @@ --- -layout: layouts/base.vto -title: Telemetry & logging -templateEngine: [vento, md] -prev: { label: "KV, queues & cron", href: "/capabilities/kv-queues-cron/" } -next: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" } +layout: layouts/redirect.vto +redirectTo: /observability/telemetry/ --- - -# Telemetry & logging - -NetScript treats observability as a built-in, not a bolt-on. Every service and plugin -runtime is wired for **OpenTelemetry** — services serve their RPC handlers with -trace-context propagation, the worker runtime wraps job dispatch, execution, scheduling, -subprocess hand-off, and task execution in real OTel spans, and structured logs flow through -the framework `logger`. The viewing surface is the **Aspire dashboard at -`https://localhost:18888`**, which collects OTLP traces, metrics, and structured logs from -every resource in the app graph (services, plugin APIs, background processors) the moment you -run `aspire start`. You do not stand up Jaeger, Grafana, or a log shipper to get started — the -AppHost provisions the OTLP collector and the dashboard for you. - -{{ comp.diagram({ - src: "/assets/diagrams/otel-traceparent.svg", - alt: "A W3C traceparent header propagates from a service request through the worker runtime into a subprocess and across a saga boundary, keeping every span under one trace id.", - caption: "Trace-context propagation: a single traceparent (W3C) flows service → worker → subprocess → saga, so spans from every runtime join one distributed trace in Aspire." -}) }} - -This page is the capability hub: what telemetry exists, how to emit it, where to view it, -and the one place that is still a scaffold stub. For the full generated API of -each unit, follow the reference links: the telemetry primitives live at -[`/reference/telemetry/`](/reference/telemetry/) and the structured logger at -[`/reference/logger/`](/reference/logger/). - -{{ comp callout { type: "important", title: "Aspire first, then telemetry" } }} -The dashboard, the OTLP collector, and the per-resource trace/log views all come up with the -orchestrator — they are not separate processes you start by hand. Run -cd aspire && aspire start before any netscript db -command (Aspire provisions Postgres and Redis first), then open the dashboard URL printed in -the console (https://localhost:18888, with a one-time auth token). Until Aspire is -running there is no :18888 surface to view traces or logs on. See -Database & migration for the full startup order. -{{ /comp }} - -## What it is - -Telemetry in NetScript is three OpenTelemetry signals — **traces**, **metrics**, and -**logs** — wired through one package, `@netscript/telemetry`, and one viewing surface, the -Aspire dashboard. Traces are the headline: a request enters a service, the worker runtime -dispatches and executes a job, a subprocess runs a polyglot task, and a saga advances — and -because a single **W3C `traceparent`** header propagates across every one of those process -boundaries, the whole fan-out collapses into one trace tree. Metrics (worker counts, SSE -connection stats) and structured logs ride the same OTLP export. The framework owns transport -(OTLP/HTTP to the collector) and viewing (the dashboard); it now owns most of emission too — -the worker, scheduler, queue, saga, and SSE runtimes are instrumented for you. The full -mental model — what is framework-real versus a scaffold stub — is in -[Observability](/explanation/observability/). - -## Learn → / Do → - -{{ comp.featureGrid({ items: [ - { - title: "Do — Add OpenTelemetry", - body: "Task-oriented recipe: lean on the automatic job spans, add custom spans via @netscript/telemetry withChildSpan, propagate traceparent across services and subprocesses, and point the OTLP export (:4318) at the dashboard.", - href: "/how-to/add-opentelemetry/", - icon: "◆" - }, - { - title: "Understand — Observability", - body: "The mental model: how spans, structured logs, health endpoints, and Aspire traces fit together — and the precise framework-vs-scaffold map of what is real (traceJobExecution, task.execute) versus the tracked createJobTools(ctx) stub.", - href: "/explanation/observability/", - icon: "◎" - }, - { - title: "Look up — telemetry reference", - body: "The full generated deno doc API for @netscript/telemetry — tracer facade, context/traceparent helpers, the NetScript instrumentation helpers, and the oRPC tracing plugin.", - href: "/reference/telemetry/", - icon: "≡" - } -] }) }} - -## What you get out of the box - -Telemetry in NetScript spans three layers — emission (in framework runtimes and your handler -code), transport (OTLP over HTTP to the collector), and viewing (the Aspire dashboard). The -framework owns transport and viewing, and it now owns most of emission too: the worker runtime -traces the entire job lifecycle automatically. The only thing you still wire by hand is custom -spans *inside* a job handler — and even there the `@netscript/telemetry` helpers do the work. - -{{ comp.featureGrid({ items: [ - { - title: "Structured logging", - body: "The @netscript/logger unit gives every service and plugin a level-aware, structured logger. Config is declared in netscript.config.ts (logging: { level: 'info', format: 'text' }) and threaded through the runtime — text or JSON, filterable in the dashboard.", - icon: "▤" - }, - { - title: "OpenTelemetry traces (real)", - body: "Service request paths propagate W3C trace context, and the worker runtime emits real spans for job dispatch, execution, scheduling, subprocess hand-off, and task.execute. @opentelemetry/api (^1.9) is in the catalog and wired into the request and job paths so spans flow to the collector.", - icon: "◈" - }, - { - title: "OTLP → Aspire dashboard", - body: "The generated AppHost configures an OTLP endpoint (http://localhost:4318) and the Aspire dashboard (https://localhost:18888) so traces, metrics, and logs from every resource land in one place — no collector to deploy.", - icon: "◎" - }, - { - title: "Browser console logs (default)", - body: "Generated app resources call withBrowserLogs() automatically (the AppHost pins Aspire.Hosting.Browsers), so a Fresh/Vite app's browser console output — including a client-side error an island throws — is forwarded into the dashboard next to server logs and traces. No opt-in, nothing to wire.", - icon: "▷" - }, - { - title: "Per-resource health", - body: "Each plugin API exposes a liveness probe — workers :8091 GET /health, sagas :8092 GET /health/live, triggers :8093 GET /health, auth :8094 — surfaced as resource state in the dashboard.", - icon: "✚" - } -] }) }} - -{{ comp callout { type: "note", title: "Browser logs land in the dashboard too" } }} -Because generated app resources emit withBrowserLogs() by default, the class of failure -that is otherwise invisible from the server — a client-side TypeError a durable-streams -island throws in the browser, say — shows up in the Aspire dashboard's Console logs -for that app resource, correlated with the server-side spans of the same request. You do not start a -separate browser-log collector; the AppHost wires it when it scaffolds the app. -{{ /comp }} - -## Enable tracing & see a span - -Tracing turns on from the **environment** — `@netscript/telemetry/config` resolves a -`TelemetryConfig` from the standard `OTEL_*` variables, and the Aspire AppHost already sets -them for you. The fastest path is to run `aspire start`, trigger any job, and the -runtime's automatic spans show up in the dashboard with no code change. The tab below adds a -**custom** span on top of that automatic trace. - -{{ comp callout { type: "note", title: "Local vs deployed: same instrumentation, different collector" } }} -Locally, Aspire provisions the OTLP collector and the dashboard and injects the -OTEL_* variables into every resource, so telemetry works the moment you run -aspire start — nothing to configure. Note the dashboard and OTLP endpoints bind to -ephemeral ports (the generated profile uses localhost:0), so use the URL -aspire start prints rather than assuming :18888/:4318. Running -aspire start --isolated (an upstream Aspire CLI flag) also randomizes the dashboard, OTLP, -and resource-service ports and isolates secrets, so several apphosts can boot in parallel without -colliding. -Deployed, there is no Aspire and no dashboard: you point -OTEL_EXPORTER_OTLP_ENDPOINT at your own collector or a hosted backend (Grafana Tempo, -Honeycomb, Jaeger, …). The instrumentation code is identical — only the endpoint/protocol env and the -sampler change. That single OTLP seam is the whole porting story; see Production notes below. -{{ /comp }} - -{{ comp.tabbedCode({ tabs: [ - { - label: "1 — Turn it on (env-driven)", - lang: "ts", - code: "// netscript.config.ts — declare the structured-logging contract for the workspace.\n// Tracing itself is enabled by OTEL_* env vars (the Aspire AppHost sets these); the\n// config package reads them — there is no defineTelemetry() call to make.\nimport { defineConfig } from '@netscript/config';\n\nexport default defineConfig({\n name: 'my-app',\n version: '1.0.0',\n logging: { level: 'info', format: 'text' }, // level: debug|info|warn|error\n plugins: ['./plugins/workers/mod.ts'],\n});\n\n// Anywhere in app code you can inspect what telemetry resolved (log-safe, redacted):\nimport { describeTelemetryConfig, isTelemetryEnabled } from '@netscript/telemetry/config';\nif (isTelemetryEnabled()) console.log(describeTelemetryConfig());\n// → { enabled: true, endpoint: 'http://localhost:4318', protocol, serviceName, sampler }" - }, - { - label: "2 — Add a custom span", - lang: "ts", - code: "// plugins/workers/jobs/health-check.ts — a real child span inside a handler.\n// The dispatcher already wraps this job in traceJobExecution automatically;\n// withChildSpan opens a child span under that parent. recordJobProgress's 3rd arg\n// is a UNIT label (e.g. 'steps'), not a free-text message.\nimport { createSuccessResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { withChildSpan, recordJobProgress } from '@netscript/telemetry/instrumentation';\n\nconst handler = defineJobHandler(async (ctx) => {\n recordJobProgress(1, 2, 'steps'); // → job.progress event on the trace\n\n const envCheck = await withChildSpan('check.environment', async (span) => {\n span.setAttribute('check.name', 'environment');\n return { ok: true };\n });\n\n recordJobProgress(2, 2, 'steps');\n return createSuccessResult({ healthy: envCheck.ok });\n});\n\nexport default Object.assign(handler, { id: 'workers-plugin-health-check' as const });" - } -] }) }} - -{{ comp callout { type: "tip", title: "Lowest-effort observability path" } }} -Run aspire start, trigger a job, and open :18888 → Traces. -You will see the dispatch span, the execution span, progress and step events, and — for -subprocess or polyglot tasks — a continued trace across the process boundary. None of that -requires editing the scaffold. Reach for custom instrumentation only when you want spans -inside your own handler logic. -{{ /comp }} - -## Key types: the telemetry init/config surface - -Configuration is **read from the environment**, not constructed — `@netscript/telemetry/config` -resolves and caches a `TelemetryConfig` from the `OTEL_*` variables. These are the confirmed -fields and the helpers that read them. - -{{ comp.apiTable({ - caption: "TelemetryConfig (@netscript/telemetry/config — resolved from OTEL_* env vars)", - rows: [ - { name: "enabled", type: "boolean", desc: "Whether OpenTelemetry instrumentation is active (driven by OTEL_DENO / endpoint presence). Mirror it with isTelemetryEnabled()." }, - { name: "endpoint", type: "string | undefined", desc: "OTLP exporter endpoint URL (e.g. http://localhost:4318). Read directly via getOtlpEndpoint()." }, - { name: "protocol", type: "string", desc: "OTLP exporter protocol (e.g. http/protobuf), from OTEL_EXPORTER_OTLP_PROTOCOL." }, - { name: "serviceName", type: "string", desc: "Service name reported to backends, from OTEL_SERVICE_NAME (or the default). Read via getServiceName()." }, - { name: "serviceVersion", type: "string", desc: "Service version reported to backends." }, - { name: "resourceAttributes", type: "Record", desc: "Resource attributes parsed from OTEL_RESOURCE_ATTRIBUTES." }, - { name: "sampler", type: "string", desc: "Trace sampler name, from OTEL_TRACES_SAMPLER (e.g. parentbased_always_on, traceidratio)." }, - { name: "debug", type: "boolean", desc: "Whether debug-level telemetry logging is on (OTEL_LOG_LEVEL)." } - ] -}) }} - -{{ comp.apiTable({ - caption: "Config helpers (@netscript/telemetry/config)", - rows: [ - { name: "getTelemetryConfig()", type: "→ TelemetryConfig", desc: "Resolve telemetry config from the OTEL_* environment variables." }, - { name: "getConfig()", type: "→ TelemetryConfig", desc: "Return the process-cached config (resetConfig() clears the cache, mainly for tests)." }, - { name: "isTelemetryEnabled()", type: "→ boolean", desc: "Whether instrumentation is enabled for this process." }, - { name: "getOtlpEndpoint()", type: "→ string | undefined", desc: "The configured OTLP endpoint, when present." }, - { name: "getServiceName()", type: "→ string", desc: "The configured service name, or the default." }, - { name: "describeTelemetryConfig()", type: "→ TelemetryConfigDescription", desc: "A redacted, log-safe summary of the resolved config — safe to print in diagnostics." }, - { name: "OTEL_ENV_VARS", type: "const record", desc: "The OTEL_* variable NAMES NetScript reads: OTEL_DENO, OTEL_EXPORTER_OTLP_ENDPOINT/_PROTOCOL, OTEL_SERVICE_NAME, OTEL_RESOURCE_ATTRIBUTES, OTEL_TRACES_SAMPLER, OTEL_LOG_LEVEL, the BSP/BLRP schedule delays, and OTEL_METRIC_EXPORT_INTERVAL." } - ] -}) }} - -## Span & instrumentation helpers - -Two subpaths cover hand-written instrumentation. `@netscript/telemetry/tracer` is the -low-level facade over `@opentelemetry/api` — get a tracer, open a span, run a callback inside -one. `@netscript/telemetry/instrumentation` is the NetScript-domain layer the runtimes -themselves use — job, scheduler, queue, and SSE spans with the right attributes baked in. -The `/context` subpath holds the **`traceparent`** propagation helpers that make the -cross-boundary trace in the diagram above possible. - -{{ comp.apiTable({ - caption: "Tracer facade (@netscript/telemetry/tracer)", - rows: [ - { name: "getTracer(name, version)", type: "→ Tracer", desc: "Cached tracer for an instrumentation name/version. Domain shortcuts exist: getJobTracer/getQueueTracer/getWorkerTracer/getSchedulerTracer/getSagaTracer/getSSETracer/getKVTracer." }, - { name: "withSpan(tracer, name, fn, options)", type: "→ Promise", desc: "Run an async callback inside a span and close it on completion. withSpanSync is the synchronous form." }, - { name: "createSpan(tracer, name, options)", type: "→ Span", desc: "Create a span from a tracer + CreateSpanOptions (kind, attributes, parentContext, links). You end() it yourself." }, - { name: "setSpanAttributes / setSpanError / setSpanOk", type: "(span, …) → void", desc: "Bulk-set attributes, mark a span failed (with an optional Error), or mark it OK. addSpanEvent(span, name, attrs?) adds an event." }, - { name: "getActiveSpan() / getActiveContext()", type: "→ Span? / Context", desc: "Read the span / OTel context active on the current async path. isTracingEnabled() gates work." }, - { name: "SpanKind / SpanStatusCode / TracerNames", type: "const", desc: "Span-kind (INTERNAL/SERVER/CLIENT/PRODUCER/CONSUMER), status (UNSET/OK/ERROR), and the standard domain tracer-name strings." } - ] -}) }} - -{{ comp.apiTable({ - caption: "NetScript instrumentation helpers (@netscript/telemetry/instrumentation)", - rows: [ - { name: "traceJobExecution(options, fn)", type: "→ Promise", desc: "Wrap a job run in a span with job.started/completed/failed events. Emitted automatically by the dispatcher; the supported entry point for tracing a job by hand." }, - { name: "withChildSpan(name, fn, attributes?)", type: "→ Promise", desc: "Open a child span under the active span and run fn inside it. The go-to for custom spans in a handler." }, - { name: "recordJobProgress(current, total, unit)", type: "→ void", desc: "Emit a job.progress event. The 3rd arg is a UNIT label (e.g. 'steps'), NOT a description. addJobStepEvent(stepName, attrs?) records job.step.* events." }, - { name: "startJobDispatchSpan / traceJobDispatch", type: "span / → Promise", desc: "Open the dispatch span and return PropagationHeaders to carry traceparent to the executor. createJobSubprocessEnv injects traceparent/tracestate into a subprocess env." }, - { name: "scheduler & cron spans", type: "span / event", desc: "createSchedulerStartSpan, createScheduleJobSpan, startSchedulerTickSpan, traceSchedulerTick, recordCronJobRun — cron/schedule dispatch traced end to end." }, - { name: "traceQueue(queue, options) / TracedQueue", type: "→ TracedQueue", desc: "Wrap a MessageQueue so enqueue/consume carry trace context. SSE helpers (startSSEConnection, createSSEEventSpan, traceSSEEvent) trace Server-Sent-Event streams." } - ] -}) }} - -{{ comp.apiTable({ - caption: "Traceparent / context propagation (@netscript/telemetry/context)", - rows: [ - { name: "formatTraceparent(spanContext)", type: "→ string", desc: "Serialize a span context to a W3C traceparent header value. parseTraceparent(value) parses one back to a ParsedTraceparent | null." }, - { name: "injectContext(headers, ctx?) / extractContext(headers)", type: "→ headers / → Context", desc: "Inject the active trace context into outbound PropagationHeaders, or extract a remote one from inbound headers — the core of cross-service propagation." }, - { name: "withContext(ctx, fn) / withContextAsync(ctx, fn)", type: "→ T / → Promise", desc: "Run a callback with a given OTel context active so spans created inside attach to the right parent." }, - { name: "createMessageHeaders / resolveParentContextFromHeaders", type: "→ headers / → Context", desc: "Build propagation headers for a queue message, and resolve the parent context back out on the consumer side." }, - { name: "getTraceId(ctx?) / getSpanId(ctx?)", type: "→ string?", desc: "Read the current trace/span ids — handy for correlating structured log lines with a trace." } - ] -}) }} - -## Service & RPC tracing toggle - -Services opt into trace propagation through the oRPC layer rather than by hand. The -`@netscript/telemetry/orpc` subpath ships an oRPC **tracing plugin** that opens a SERVER span -per RPC and continues any inbound `traceparent`, plus an error-handling plugin that classifies -failures. On the service builder this is the `.withRPC({ traceContext: true })` toggle (see -[Services](/capabilities/services/)); the plugin -factory below is the underlying surface if you mount oRPC yourself. - -{{ comp.apiTable({ - caption: "oRPC tracing surface (@netscript/telemetry/orpc)", - rows: [ - { name: "createTracingPlugin(options?)", type: "→ TracingPlugin", desc: "oRPC plugin that opens a SERVER span per call and continues an inbound traceparent. Options are TracingPluginOptions." }, - { name: "createErrorHandlingPlugin(options?)", type: "→ ErrorHandlingPlugin", desc: "Classifies errors as client | server | transient and records them on the span; takes an optional ErrorClassifier and ErrorLogger." }, - { name: "createTraceContext()", type: "→ TraceContext", desc: "Build the per-call trace context the plugin threads through. addEvent / setAttributes / getTraceId / getSpanId operate on the active span." } - ] -}) }} - -{{ comp callout { type: "note", title: "Database & worker tracing toggles live next door" } }} -The Prisma query-tracing and worker-runtime tracing toggles are documented on their own hubs -to avoid duplication. Database query spans are wired through the -database client; the worker runtime's automatic job -spans are described in background jobs. This hub -owns the cross-cutting telemetry surface — config, the tracer facade, -traceparent propagation, and the oRPC plugin — that those runtimes build on. See -{{ comp.xref({ key: "cap:database", text: "Database & Prisma" }) }} for the per-query toggle. -{{ /comp }} - -## What is traced automatically - -Before you write a single instrumentation call, the worker runtime already produces a usable -trace tree in Aspire. The `@netscript/telemetry` instrumentation is wired into the dispatcher, -the executor, and the scheduler, so the moment a job runs you get spans with attributes, -durations, status, and lifecycle events — no scaffold changes required. - -{{ comp.apiTable({ - caption: "Automatic worker traces (emitted by the runtime — see /reference/telemetry/)", - rows: [ - { name: "traceJobExecution", type: "span", desc: "Wraps each job's execution with attributes, duration, status, and job.started / job.completed / job.failed / job.exception events. Emitted by the dispatcher." }, - { name: "recordJobProgress", type: "event", desc: "job.progress events carrying current / total / unit — the runtime records real progress as the job advances." }, - { name: "addJobStepEvent", type: "event", desc: "job.step.* events for each step the runtime walks through." }, - { name: "scheduler spans", type: "span", desc: "createSchedulerStartSpan / createScheduleJobSpan / startSchedulerTickSpan / recordCronJobRun — cron and schedule dispatch are traced end to end." }, - { name: "subprocess traceparent", type: "context", desc: "The dispatcher injects W3C traceparent / tracestate into the subprocess env so out-of-process job runs continue the same trace (initJobTracing / runTracedJob / createJobSubprocessEnv)." }, - { name: "task.execute", type: "span", desc: "The multi-runtime task executor wraps each task run in a task.execute span — polyglot and TS tasks alike show up in the trace tree." } - ] -}) }} - -## Instrument a handler - -The two tabs below show the two emission paths a developer touches: structured logging -through the framework logger, and **custom** spans inside a job handler. For custom spans, -call the `@netscript/telemetry` helpers (`traceJobExecution`, `withChildSpan`, -`recordJobProgress`) directly — they are the real, supported surface. Read the -callout under the tabs to understand why you reach for the telemetry package rather than the -scaffold's `createJobTools(ctx)` trace helpers. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Structured logging", - lang: "ts", - code: "// netscript.config.ts — declare the logging contract for the whole workspace.\nimport { defineConfig } from '@netscript/config';\n\nexport default defineConfig({\n name: 'my-app',\n version: '1.0.0',\n // level: 'debug' | 'info' | 'warn' | 'error'; format: 'text' | 'json'\n logging: { level: 'info', format: 'text' },\n plugins: ['./plugins/workers/mod.ts', './plugins/sagas/mod.ts'],\n});\n\n// Inside a job handler, the logger comes from the job tools (console-backed today,\n// and surfaced under the resource's Console logs view in the Aspire dashboard).\n// import { createJobTools } from './job-tools.ts';\nconst emit = (log) => {\n log.info('user provisioned', { userId: 'u_123', source: 'scaffold' });\n log.warn('rate limit approaching', { remaining: 4 });\n};" - }, - { - label: "Custom spans via @netscript/telemetry", - lang: "ts", - code: "// plugins/workers/jobs/health-check.ts — real custom spans inside a handler.\n// The dispatcher already wraps this job in traceJobExecution automatically;\n// withChildSpan + recordJobProgress let you add detail under that parent span.\nimport { createSuccessResult, createFailureResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { withChildSpan, recordJobProgress } from '@netscript/telemetry/instrumentation';\n\nconst handler = defineJobHandler(async (ctx) => {\n const { log } = ctx;\n log.info('Starting workers plugin health check');\n\n // Real progress event — 3rd arg is a UNIT label, not a message.\n recordJobProgress(1, 2, 'steps');\n\n // withChildSpan opens a real child span under the job-execution span.\n const envCheck = await withChildSpan('check.environment', async (span) => {\n span.setAttribute('check.name', 'environment');\n return { ok: true };\n });\n\n recordJobProgress(2, 2, 'steps');\n if (!envCheck.ok) return createFailureResult('environment check failed');\n return createSuccessResult({ status: 'healthy' });\n});\n\nexport default Object.assign(handler, { id: 'workers-plugin-health-check' as const });" - } -] }) }} - -{{ comp callout { type: "warning", title: "Known gap: the scaffold createJobTools(ctx) trace/progress helpers are no-op stubs" } }} -Job dispatch and execution are instrumented with real OTel spans — traces -show up in Aspire automatically (traceJobExecution, recordJobProgress, -scheduler spans, and subprocess traceparent propagation are all live in the runtime). The one -remaining gap is narrow and specific: the trace.{addEvent,withChildSpan,recordProgress} -and progress(...) helpers returned by the scaffold's createJobTools(ctx) -are currently no-op stubs in the generated copy. log writes to the -console; those particular trace/progress helpers do nothing. This is a known, tracked -limitation with a fix planned (debt workers-scaffold-job-tools-noop), not a -permanent design choice. Workaround: for custom spans and progress, call the -@netscript/telemetry helpers directly (as in the tab above) rather than the scaffold -createJobTools trace surface — those are real today. See -Observability for the full framework-vs-scaffold map. - -{{ /comp }} - -## Endpoints & ports - -Telemetry has no single service of its own — it is emitted by every runtime and aggregated -by Aspire. These are the real addresses you interact with, validated by the CLI E2E suite. - -{{ comp.apiTable({ - caption: "Observability surfaces (link to /reference/telemetry/ and /reference/logger/ for the generated APIs)", - rows: [ - { name: "https://localhost:18888", type: "dashboard", desc: "Aspire dashboard — traces, structured logs, metrics, and resource state for the whole app graph. Auth token printed by `aspire start`." }, - { name: "http://localhost:4318", type: "OTLP/HTTP", desc: "OTLP ingest endpoint the AppHost configures (aspire.config.json https profile). Runtimes export spans and logs here; the dashboard reads them back. This is the seam you point at a hosted backend." }, - { name: "GET :8091/health", type: "liveness", desc: "Workers API health probe — reported as resource health in the dashboard." }, - { name: "GET :8092/health/live", type: "liveness", desc: "Sagas API liveness route." }, - { name: "GET :8093/health", type: "liveness", desc: "Triggers API health probe (Hono service)." }, - { name: ":8094", type: "service", desc: "Auth API (auth-api) — its request spans propagate trace context like any other service." } - ] -}) }} - -## View it in Aspire - -With `aspire start` up, the dashboard at `https://localhost:18888` gives you four views over the -same telemetry stream — there is no separate tool to configure. - -{{ comp.apiTable({ - caption: "Aspire dashboard views (https://localhost:18888)", - rows: [ - { name: "Resources", type: "graph", desc: "Live state of every app-graph resource: postgres, redis, workers-api, workers, sagas-api, sagas, triggers-api, triggers, auth-api. Health probes drive the status colour." }, - { name: "Console logs", type: "stream", desc: "Per-resource stdout/stderr — the framework logger's text/JSON output lands here in real time." }, - { name: "Structured logs", type: "OTLP", desc: "Structured log records exported over OTLP, filterable by resource, level, and attributes." }, - { name: "Traces", type: "OTLP", desc: "Distributed traces collected from the OTLP endpoint (http://localhost:4318) — job dispatch/execution/scheduler/task spans and cross-service request spans as trace context propagates." } - ] -}) }} - -{{ comp callout { type: "tip", title: "Use this when…" } }} -Reach for the dashboard whenever you need to answer "what happened and where": -which resource is unhealthy, what a service logged for a given request, why a job failed, or how -a call fanned out across services and subprocesses. Because Aspire wires OTLP for you, the -lowest-effort observability path is to run aspire start and open -:18888 — no extra dependency, no collector to deploy. When you outgrow the local -dashboard, the same OTLP export (http://localhost:4318) is the seam you point at a -hosted backend. -{{ /comp }} - -## Production notes - -{{ comp callout { type: "warning", title: "Footguns before you ship telemetry" } }} -

❌ Trace 100% of traffic in production. The local Aspire default samples -everything — fine for dev, ruinous at scale.
-✅ Set a ratio sampler via env. Sampling is an env decision, not a code one: set -OTEL_TRACES_SAMPLER (surfaced as TelemetryConfig.sampler) to -traceidratio with a rate. Exception: keep always-on sampling for low-volume -services or a targeted debugging window.

-

❌ Fork the runtime to change telemetry backends. Editing instrumentation to -point at Grafana/Honeycomb is churn you will re-do every upgrade.
-✅ Point the exporter. The OTLP export (OTEL_EXPORTER_OTLP_ENDPOINT, -default http://localhost:4318) is the single seam — swap endpoint/protocol via env and -the instrumentation code does not change.

-

❌ Put unbounded values in span attributes. User ids, raw URLs, or full payloads -in span.setAttribute explode index cardinality on the backend.
-✅ Use stable, low-cardinality keys. Correlate to a specific record with -getTraceId() in a structured log line instead of widening the span.

-

❌ Pass a message as recordJobProgress's 3rd argument. Free text -there inflates cardinality and misreads the API.
-✅ Pass a stable unit label. 'steps' / 'items' — the -argument is a unit, not a description.

-

❌ Ship code that relies on the scaffold createJobTools(ctx) trace/progress -helpers. Those are no-op stubs today (debt workers-scaffold-job-tools-noop).
-✅ Call the @netscript/telemetry helpers directly for real spans and -progress, as in the handler tab above.

-{{ /comp }} - -## Reference - -This hub is intentionally thin — the full generated API lives in the reference. - -{{ comp.xref({ key: "ref:telemetry" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Do — Add OpenTelemetry", - body: "Task-oriented recipe: lean on the automatic job spans, add custom spans via @netscript/telemetry withChildSpan, propagate traceparent across services and subprocesses, and point the OTLP export (:4318) at the dashboard.", - href: "/how-to/add-opentelemetry/", - icon: "◆" - }, - { - title: "Understand — Observability", - body: "The mental model: how spans, structured logs, health endpoints, and Aspire traces fit together — and the precise framework-vs-scaffold map of what is real (traceJobExecution, task.execute) versus the tracked createJobTools(ctx) stub.", - href: "/explanation/observability/", - icon: "◎" - }, - { - title: "Look up — telemetry reference", - body: "The full generated deno doc API for @netscript/telemetry — tracer facade, context/traceparent helpers, the instrumentation helpers, scheduler spans, and the oRPC tracing plugin. The authority for signatures.", - href: "/reference/telemetry/", - icon: "≡" - }, - { - title: "Look up — logger reference", - body: "The full generated deno doc API for @netscript/logger — log levels, formats, and the structured-logging surface declared in netscript.config.ts.", - href: "/reference/logger/", - icon: "≡" - } -] }) }} - -{{ comp.nextPrev({ - prev: { label: "KV, queues & cron", href: "/capabilities/kv-queues-cron/" }, - next: { label: "Fresh UI & design", href: "/capabilities/fresh-ui/" } -}) }} diff --git a/docs/site/capabilities/triggers.md b/docs/site/capabilities/triggers.md index 8116927a6..16e87b58a 100644 --- a/docs/site/capabilities/triggers.md +++ b/docs/site/capabilities/triggers.md @@ -1,485 +1,4 @@ --- -layout: layouts/base.vto -title: Triggers & ingress -templateEngine: [vento, md] -prev: { label: "Durable sagas", href: "/capabilities/durable-sagas/" } -next: { label: "Durable streams", href: "/capabilities/streams/" } +layout: layouts/redirect.vto +redirectTo: /durable-workflows/triggers/ --- - -# Triggers & ingress - -The **triggers** plugin is NetScript's front door for the outside world. It turns -external events — inbound webhooks, files dropped on disk, and cron schedules — -into durable background work. A trigger handler does one job: receive an event and -return a list of *actions*. The canonical action is `enqueueJob(...)`, which hands -the payload to the [workers](/capabilities/background-jobs/) plugin so the heavy -lifting happens off the request path. The HTTP call returns immediately; the job -runs durably behind it. - -{{ comp.diagram({ - src: "/assets/diagrams/trigger-ingress-flow.svg", - alt: "Trigger ingress flow: a webhook POST is verified by an HMAC check, persisted as an event, and dispatched to the handler which returns an enqueueJob action; a file-watch event is debounced and stabilised before reaching the same handler path.", - caption: "Two ingress edges, one processor: webhook POST → HMAC verify → event store → handler → enqueueJob; file-watch → debounce + stability → handler. Both converge on the trigger processor, which retries and DLQs." -}) }} - -This closes the continuous-app loop. A webhook lands on the triggers service, it -`enqueueJob`s a workers job, that job publishes a saga message, and the -[saga](/capabilities/durable-sagas/) advances a workflow — all from one inbound -POST. - -{{ comp callout { type: "important", title: "Typed oRPC contract, raw webhook ingress" } }} -Like services, workers, -and sagas, the triggers API service serves a -typed v1 oRPC contract for trigger and event introspection plus management — -describing triggers, listing and fetching triggers and events, firing a trigger, testing a webhook, -previewing a schedule, enabling/disabling a trigger, and subscribing to an SSE event stream. The one -deliberate exception is the webhook ingress endpoint itself -(POST /api/v1/webhooks/:triggerId, plus a legacy GET /api/v1/events): it stays -a raw, signature-verifying route, not an oRPC procedure, because it must verify an HMAC -signature over the raw request bytes — incompatible with oRPC's Zod body parsing. That keeps it open to -any third-party webhook sender, which has no NetScript typed client and just POSTs to the plain URL. -{{ /comp }} - -## What it is - -A trigger is a **declarative event source** plus a handler. NetScript ships three -authorable trigger builders from `@netscript/plugin-triggers-core/builders` — -`defineWebhook` (inbound HTTP), `defineFileWatch` (filesystem events), and -`defineScheduledTrigger` (cron). All three return a frozen *definition* the runtime -walker discovers; all three share one handler contract — `(event, context) => -Promise` — and one runtime processor that verifies, persists, -retries, and dead-letters. Three more kinds (`queue`, `stream`, `manual`) are reserved -in the type surface but not yet executable. The processing engine itself lives in -[`@netscript/watchers`](/reference/watchers/) (file watching) and the trigger -processor/ingress runtime (`@netscript/plugin-triggers-core/runtime`). - -## Learn / Do - -{{ comp.featureGrid({ items: [ - { - title: "Learn — Ingest a webhook", - body: "Storefront Track A, rung 05: scaffold the triggers plugin, author a webhook, POST to :8093, and watch it enqueue a job — closing the continuous-app ladder.", - href: "/tutorials/storefront/05-shipping-webhook/", - icon: "→" - }, - { - title: "Do — Add a plugin", - body: "The how-to recipe for public plugin package install and the local netscript-dev contributor path for source-backed trigger samples.", - href: "/how-to/add-a-plugin/", - icon: "◆" - } -] }) }} - -## Minimal example — a verified payment webhook - -The 80% case: a webhook that verifies an HMAC signature, then fans the inbound -payment event out to a single durable job. The handler returns immediately; the -real work runs on the [workers](/capabilities/background-jobs/) plugin. - -```ts -// plugins/triggers/payment-status-webhook.ts -import { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders'; -import type { JobDefinition } from '@netscript/plugin-workers-core'; - -// The workers job this webhook will enqueue (authored in plugins/workers/jobs/). -const reconcilePaymentJob = { - id: 'reconcile-payment' as JobDefinition<'reconcile-payment'>['id'], - name: 'Reconcile Payment', - topic: 'default', -} satisfies JobDefinition<'reconcile-payment'>; - -// Inbound POST -> enqueue a workers job. Ingress verifies the HMAC signature -// (verifier: 'hmac-sha256' + secretEnv) BEFORE this handler ever runs, so by the -// time you see the event the sender is already proven. -export const paymentStatusWebhook = defineWebhook( - (event) => Promise.resolve([ - enqueueJob(reconcilePaymentJob, { - payload: { raw: event.payload.body }, - idempotencyKey: event.idempotencyKey, // de-dupe sender retries - priority: 80, - }), - ]), - { - id: 'payment-status-webhook', - path: 'payments/status', - verifier: 'hmac-sha256', - secretEnv: 'PAYMENT_WEBHOOK_SECRET', - description: 'Verified payment-status webhook that enqueues reconciliation.', - tags: ['webhook', 'payments', 'verified'], - }, -); - -export default paymentStatusWebhook; -``` - -## Key types first — the webhook definition - -`defineWebhook(handler, spec)` takes your handler and a static `WebhookSpec`. These -are the **actual** fields the builder accepts (confirmed via `deno doc`), shown -before any prose so you can read the contract at a glance. - -{{ comp.apiTable({ - caption: "WebhookSpec — the second argument to defineWebhook(handler, spec)", - rows: [ - { name: "id", type: "string (required)", desc: "Stable identifier for the webhook definition, e.g. 'payment-status-webhook'. Branded internally as a WebhookId." }, - { name: "path", type: "string (required)", desc: "Ingress sub-path the webhook answers on, e.g. 'payments/status'. Resolved as the :triggerId route segment under /api/v1/webhooks/." }, - { name: "verifier", type: "'hmac-sha256' | 'memory' | string (required)", desc: "Verification strategy selector (WebhookVerifierKind). 'hmac-sha256' performs a real signature check; 'memory' is open (dev only); a custom string selects a verifier you wire via selectVerifier." }, - { name: "secretEnv", type: "string?", desc: "Name of the env var holding the signing secret. The runtime reads it (resolveSecret) and hands it to the verifier — the secret is never inlined in the definition." }, - { name: "description", type: "string?", desc: "Human-readable summary surfaced by inspect/registry tooling." }, - { name: "tags", type: "readonly string[]?", desc: "Free-form labels for discovery/filtering, e.g. ['webhook','payments','verified']." }, - { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata attached to the definition." } - ] -}) }} - -The handler receives a `TriggerEvent<'webhook', WebhookTriggerPayload>`. The payload -carries the captured HTTP request: `body` (parsed/raw), `headers`, `method` (always -`"POST"`), `path`, and an optional `remoteAddr`. The event envelope also surfaces -`idempotencyKey`, `traceparent`/`tracestate`, and `requestHeaders` — so de-duplicating -sender retries is a one-liner, as in the example above. - -## HMAC verification — proving the sender - -`verifier: 'hmac-sha256'` activates the `HmacSha256WebhookVerifier` adapter from -`@netscript/plugin-triggers-core/adapters`. Ingress hashes the raw request body with -the secret (resolved from `secretEnv`) and compares it against the signature header -*before* persisting the event or running your handler — a bad signature is rejected at -the edge. The `'memory'` verifier (`MemoryWebhookVerifier`) accepts everything and is -for local iteration only. - -{{ comp.apiTable({ - caption: "HmacSha256WebhookVerifier — constructor options (from @netscript/plugin-triggers-core/adapters)", - rows: [ - { name: "secret", type: "string?", desc: "Signing secret. Usually supplied by the runtime from the definition's secretEnv rather than hard-coded here." }, - { name: "signatureHeader", type: "string?", desc: "Header carrying the sender's HMAC signature to compare against (e.g. 'x-signature')." }, - { name: "idempotencyHeader", type: "string?", desc: "Header whose value is surfaced as the event idempotencyKey when present, so sender retries collapse." } - ] -}) }} - -{{ comp callout { type: "note", title: "How the verifier is wired" } }} -You rarely construct the adapter yourself. The webhook spec declares -verifier: 'hmac-sha256' and secretEnv, and the ingress runtime -(createTriggerIngress) resolves the secret and selects the verifier per -definition (its verifier, selectVerifier, and -resolveSecret options). The verifier's verify(request) returns -{ ok, idempotencyKey?, reason? } — ok: false stops the event at the edge. -{{ /comp }} - -## File-watch triggers — react to dropped files - -`defineFileWatch(handler, spec)` turns filesystem events into the same durable-work -pipeline. The watching primitive lives in [`@netscript/watchers`](/reference/watchers/); -the trigger builder wraps it so a dropped file becomes a handler invocation that -returns `enqueueJob` actions. Below: a CSV drop-folder that enqueues one import job per -stable file. - -```ts -// plugins/triggers/product-import.ts -import { defineFileWatch, enqueueJob } from '@netscript/plugin-triggers-core/builders'; -import type { JobDefinition } from '@netscript/plugin-workers-core'; - -const importProductsJob = { - id: 'import-products' as JobDefinition<'import-products'>['id'], - name: 'Import Products', - topic: 'default', -} satisfies JobDefinition<'import-products'>; - -// Fires once a *.csv lands in ./incoming/products and stops growing. The -// stabilityThreshold guards against half-written files on a network share: -// the watcher waits for 3 unchanged checks 1s apart before yielding the event. -export const productImportWatch = defineFileWatch( - (event) => Promise.resolve([ - enqueueJob(importProductsJob, { - payload: { file: event.payload.path }, - idempotencyKey: event.payload.path, // one import per file path - priority: 50, - }), - ]), - { - id: 'product-import-watch', - paths: ['./incoming/products'], - patterns: ['*.csv'], - on: ['create', 'modify'], - stabilityThreshold: { checkIntervalMs: 1000, stableChecks: 3 }, - description: 'Imports a product CSV the moment it lands and stabilises.', - tags: ['file-watch', 'erp', 'import'], - }, -); - -export default productImportWatch; -``` - -{{ comp.apiTable({ - caption: "FileWatchSpec — the second argument to defineFileWatch(handler, spec)", - rows: [ - { name: "id", type: "string (required)", desc: "Stable identifier for the file-watch definition." }, - { name: "paths", type: "readonly string[] (required)", desc: "Directories to watch (at least one). Relative paths resolve against the process cwd." }, - { name: "patterns", type: "readonly string[] (required)", desc: "Glob patterns selecting which filenames fire, e.g. ['*.csv','sales_*.xlsx']." }, - { name: "on", type: "readonly ('create'|'modify'|'remove')[] (required)", desc: "Which filesystem lifecycle events yield to the handler (FileWatchLifecycle)." }, - { name: "ignored", type: "readonly string[]?", desc: "Glob patterns to exclude from the watch." }, - { name: "debounceMs", type: "number?", desc: "Per-file debounce window collapsing rapid successive events for the same path." }, - { name: "stabilityThreshold", type: "{ checkIntervalMs, stableChecks }?", desc: "Network-FS tolerant gate: waits for stableChecks unchanged polls (checkIntervalMs apart) so half-written files don't fire early." }, - { name: "description", type: "string?", desc: "Human-readable summary for tooling." }, - { name: "tags", type: "readonly string[]?", desc: "Free-form discovery labels." }, - { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata." } - ] -}) }} - -The handler receives a `TriggerEvent<'file-watch', FileWatchTriggerPayload>`; the -payload carries `path`, `kind` (the lifecycle event), and — when the watcher could read -them — `size`, `modifiedAt`, and `stableChecks`. - -### The underlying watcher - -For lower-level or standalone file watching (outside the trigger pipeline), call -`createWatcher(options)` from `@netscript/watchers` directly. It returns a `FileWatcher` -whose `watch()` is an async generator of `WatchEvent`s. - -```ts -// scripts/watch-incoming.ts -import { createWatcher } from "@netscript/watchers"; - -const watcher = createWatcher({ - paths: ["./incoming"], - patterns: ["*.csv"], - events: ["create", "modify"], - stabilityThreshold: { checkIntervalMs: 1000, stableChecks: 3 }, -}); - -for await (const event of watcher.watch()) { - console.log(`${event.kind}: ${event.path}`); -} -``` - -{{ comp.apiTable({ - caption: "createWatcher(options) — WatcherOptions (from @netscript/watchers)", - rows: [ - { name: "paths", type: "readonly string[] (required)", desc: "Directories to watch (at least one required)." }, - { name: "patterns", type: "readonly string[]?", desc: "Glob patterns for filtering files. Default ['*']." }, - { name: "events", type: "readonly EventKind[]?", desc: "Which FS events to yield ('create'|'modify'|'remove'). Default ['create']." }, - { name: "debounceMs", type: "number?", desc: "Per-file debounce in milliseconds. Default 2000." }, - { name: "contentHash", type: "boolean?", desc: "Compute a SHA-256 content hash for de-duplication. Default true." }, - { name: "processExisting", type: "boolean?", desc: "Scan existing files on startup and emit them as create events. Default false." }, - { name: "forcePolling", type: "boolean?", desc: "Force the polling strategy instead of native FS notifications. Default false (use for network paths)." }, - { name: "pollIntervalMs", type: "number?", desc: "Polling interval for the polling strategy. Minimum 500. Default 5000." }, - { name: "minFileSize", type: "number?", desc: "Skip files smaller than this many bytes. Default 0." }, - { name: "maxFileAge", type: "number?", desc: "Skip files older than this (ms); only applies during the startup scan." }, - { name: "stabilityThreshold", type: "StabilityOptions?", desc: "When set, waits for files to stop growing before yielding (network-FS tolerant)." }, - { name: "signal", type: "AbortSignal?", desc: "Abort signal for graceful shutdown of the watch loop." } - ] -}) }} - -The watcher selects the strategy for you: native OS notifications for local paths, -polling for network paths or when `forcePolling: true`, and a `HybridStrategy` that -blends both. Events pass a filter pipeline before you see them — `GlobFilter` limits -filenames, `StabilityFilter` waits for files to stop growing, and `DedupFilter` skips -repeated content hashes within its window. The concrete `NativeStrategy`, -`PollingStrategy`, and `HybridStrategy` classes are internal; construct watchers with -`createWatcher` or `new FileWatcher(...)`. - -## Scheduled triggers — cron without a daemon - -`defineScheduledTrigger(handler, spec)` fires a handler on a cron schedule. The spec -combines discovery metadata (`id`, `description`, `tags`, `metadata`) with the cron -`ScheduledTriggerSpec`. The scheduler runs inside the trigger processor entrypoint, not -the Hono API service. - -```ts -// plugins/triggers/nightly-reconcile.ts -import { defineScheduledTrigger, enqueueJob } from '@netscript/plugin-triggers-core/builders'; -import type { JobDefinition } from '@netscript/plugin-workers-core'; - -const nightlyReconcileJob = { - id: 'nightly-reconcile' as JobDefinition<'nightly-reconcile'>['id'], - name: 'Nightly Reconcile', - topic: 'default', -} satisfies JobDefinition<'nightly-reconcile'>; - -// Runs at 02:00 UTC. backfill replays fires the scheduler missed while down. -export const nightlyReconcile = defineScheduledTrigger( - () => Promise.resolve([enqueueJob(nightlyReconcileJob, { payload: {} })]), - { - id: 'nightly-reconcile', - description: 'Nightly reconciliation sweep.', - cron: '0 2 * * *', - timezone: 'UTC', - persistent: true, - backfill: { enabled: true, windowMs: 86_400_000, policy: 'fire-once' }, - }, -); - -export default nightlyReconcile; -``` - -{{ comp.apiTable({ - caption: "Scheduled trigger spec — DefineScheduledTriggerSpec + ScheduledTriggerSpec", - rows: [ - { name: "id", type: "string (required)", desc: "Stable identifier for the scheduled trigger definition." }, - { name: "cron", type: "CronExpression (required)", desc: "Cron expression governing when the handler fires, e.g. '0 2 * * *'." }, - { name: "timezone", type: "string?", desc: "IANA timezone the cron expression is evaluated in. Defaults to the runtime's zone." }, - { name: "persistent", type: "boolean?", desc: "Whether the schedule state survives restarts so missed fires can be reasoned about." }, - { name: "backfill", type: "TriggerBackfillSpec?", desc: "Quartz-style misfire handling: { enabled, windowMs, policy: 'fire-now'|'fire-once'|'do-nothing', maxMissedFires? } — replays fires the scheduler missed while down." }, - { name: "description", type: "string?", desc: "Human-readable summary for tooling." }, - { name: "tags", type: "readonly string[]?", desc: "Free-form discovery labels." }, - { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata." } - ] -}) }} - -## Supported trigger actions - -Every trigger handler returns **actions** — declarative descriptions of what should -happen after the event is accepted. The runtime processor reads each action and -dispatches it. Exactly one action is wired end-to-end today; a second is *defined in the -type surface* but not executable, and it now **fails loud** rather than silently dropping. - -{{ comp.apiTable({ - caption: "Trigger actions — what the runtime processor actually does", - rows: [ - { name: "enqueueJob(job, opts)", type: "✅ live", desc: "Hands the payload to the workers plugin. The supported, end-to-end path — it closes the continuous-app loop (event → job → saga). opts: { payload?, idempotencyKey?, concurrencyKey?, priority? }." }, - { name: "defer(...)", type: "⛔ unsupported", desc: "Defined in the action union (DeferAction) but NOT executable. The processor throws an unsupportedOperation error and routes the message to the dead-letter queue (DLQ). There is no deferred replay — do not author a trigger that relies on defer." } - ] -}) }} - -{{ comp callout { type: "warning", title: "defer is defined-but-unsupported — it fails loud" } }} -The defer action exists in the trigger action union, but the runtime processor does -not implement deferred replay. When a handler emits a defer action, the -processor throws an unsupportedOperation error and routes the message to the -dead-letter queue (DLQ) — it does not silently swallow it. Build ingress flows on -enqueueJob only; if you need delayed work, schedule it on the -workers plugin from the enqueued job, or use a -defineScheduledTrigger cron rather than deferring at the trigger edge. - -{{ /comp }} - -## Runtime — ingress, processor, retry - -The trigger runtime (`@netscript/plugin-triggers-core/runtime`) assembles the pieces -the API service and background processor use. You configure these when wiring a custom -host; the scaffold wires them for you. - -{{ comp.apiTable({ - caption: "Trigger runtime entry points (from @netscript/plugin-triggers-core/runtime)", - rows: [ - { name: "createTriggerIngress(options)", type: "→ TriggerIngressPort", desc: "Builds the webhook ingress edge. Options: { definitions, eventStore, processor, verifier, selectVerifier?, resolveSecret?, logger?, now?, createEventId? }. Verifies + persists, then hands off to the processor; responds 202 Accepted." }, - { name: "createTriggerProcessor(options)", type: "→ TriggerProcessor", desc: "Builds the processor that runs handlers and dispatches actions. Options: { idempotency, dlq, dispatchAction?, logger?, now?, random? }. Applies the retry policy and routes exhausted/unsupported events to the DLQ." }, - { name: "defaultRetryPolicy()", type: "→ TriggerRetryPolicy", desc: "The default { maxAttempts, initialDelayMs, maxDelayMs, backoffMultiplier, jitter } policy applied before DLQ handoff. Override per definition via the trigger's retry field." } - ] -}) }} - -{{ comp callout { type: "note", title: "Where the background work runs" } }} -The triggers API service (port 8093) is the HTTP edge — it hosts -createTriggerIngress behind a raw Hono router. The actual trigger -processing — cron, file-watch, and DLQ handling — runs in a separate background -processor entrypoint at plugins/triggers/src/runtime/trigger-processor.ts. Aspire wires -both as resources (triggers-api and trigger-processor) when you aspire start. -{{ /comp }} - -## Endpoints & port - -The triggers API service runs on **`:8093`**. The webhook router resolves the inbound -`:triggerId` segment against your registered `defineWebhook` definitions — so a -definition with `path: 'payments/status'` is reachable at -`POST :8093/api/v1/webhooks/payments/status`. - -{{ comp.apiTable({ - caption: "Triggers API — runtime surface (port 8093, Hono)", - rows: [ - { name: "POST /api/v1/webhooks/:triggerId", type: "Hono route", desc: "Generic dispatch — :triggerId matches the path of any registered defineWebhook definition. Verifies, persists, responds 202 Accepted." }, - { name: "POST /api/v1/webhooks/inbound/generic", type: "Hono route", desc: "The scaffold's generic inbound webhook. Posting JSON here resolves trigger id 'inbound/generic' and runs its actions." }, - { name: "GET /api/v1/events?limit=10", type: "Hono route", desc: "Recent ingress events recorded by the event store." }, - { name: "GET /health", type: "Hono route", desc: "Liveness check for the triggers API service." } - ] -}) }} - -## Author + call a trigger - -The simple case is one webhook that fans an inbound request out to a single job. The -advanced case validates the payload first, then enqueues. The simple case is adapted -from the scaffold's `plugins/triggers/generic-webhook.ts` sample and compiles as-is; the -advanced tab is an illustrative pattern, not scaffold code — the real -`plugins/triggers/webhook-validate-data.ts` is an accept-and-drop sample -(`() => Promise.resolve([])`) that imports no zod and enqueues no job. Every handler -returns an array of `enqueueJob` actions — the only supported trigger action. - -{{ comp.tabbedCode({ tabs: [ - { - label: "Simple — generic-webhook.ts", - lang: "ts", - code: "// plugins/triggers/generic-webhook.ts\nimport { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders';\nimport type { JobDefinition } from '@netscript/plugin-workers-core';\n\n// A reference to the workers job this webhook will enqueue.\nconst healthCheckJob = {\n id: 'workers-plugin-health-check' as JobDefinition<'workers-plugin-health-check'>['id'],\n name: 'Workers Health Check',\n topic: 'default',\n} satisfies JobDefinition<'workers-plugin-health-check'>;\n\n// Inbound POST -> enqueue a workers job. The HTTP call returns immediately;\n// the job runs durably on the workers plugin (:8091).\nexport const genericInboundWebhook = defineWebhook(\n () => Promise.resolve([\n enqueueJob(healthCheckJob, { payload: { verbose: false }, priority: 50 }),\n ]),\n {\n id: 'generic-inbound-webhook',\n path: 'inbound/generic',\n verifier: 'memory',\n description: 'Open webhook that enqueues the workers plugin health-check job.',\n tags: ['webhook', 'runtime-task', 'health-check'],\n },\n);\n\nexport default genericInboundWebhook;" - }, - { - label: "Advanced — validate + enqueue", - lang: "ts", - code: "// plugins/triggers/webhook-validate-data.ts (pattern)\nimport { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders';\nimport type { JobDefinition } from '@netscript/plugin-workers-core';\nimport { z } from 'zod';\n\n// Validate the inbound body before doing any work.\nconst InboundSchema = z.object({\n userId: z.string().min(1),\n source: z.string().default('webhook'),\n});\n\nconst createUserSettingsJob = {\n id: 'create-user-settings' as JobDefinition<'create-user-settings'>['id'],\n name: 'Create User Settings',\n topic: 'default',\n} satisfies JobDefinition<'create-user-settings'>;\n\nexport const validatedInboundWebhook = defineWebhook(\n (event) => {\n // Parse + validate. On a bad shape, accept-and-drop (return []),\n // or throw to reject the request.\n const parsed = InboundSchema.safeParse(event.payload.body ?? {});\n if (!parsed.success) return Promise.resolve([]);\n\n // Hand the validated payload to a workers job.\n return Promise.resolve([\n enqueueJob(createUserSettingsJob, {\n payload: { userId: parsed.data.userId },\n priority: 80,\n }),\n ]);\n },\n {\n id: 'validated-inbound-webhook',\n path: 'validate/data',\n verifier: 'memory',\n description: 'Validates the body with zod, then enqueues create-user-settings.',\n tags: ['webhook', 'validated'],\n },\n);\n\nexport default validatedInboundWebhook;" - }, - { - label: "Call it — curl", - lang: "bash", - code: "# Triggers API runs on :8093. POST to the webhook's resolved path.\ncurl -X POST http://localhost:8093/api/v1/webhooks/inbound/generic \\\n -H 'content-type: application/json' \\\n -d '{\"verbose\": false}'\n\n# Watch the resulting ingress events:\ncurl 'http://localhost:8093/api/v1/events?limit=10'\n\n# The enqueued job lands on the workers plugin (:8091):\ncurl http://localhost:8091/api/v1/workers/executions?limit=10" - } -] }) }} - -## How it wires to the rest of the app - -This is the last rung of the continuous-app thread. The `generic-inbound-webhook` -enqueues the workers health-check job; the `create-user-settings` job (authored in the -[background jobs](/capabilities/background-jobs/) tutorial) publishes a -`UserSettingsCreated` saga message; the saga from the -[durable workflow](/capabilities/durable-sagas/) tutorial handles it. One inbound POST -drives the whole chain — and every link is real scaffold code that compiles. - -## Production notes - -{{ comp callout { type: "warning", title: "Production pitfalls" } }} -
    -
  • Never ship the 'memory' verifier publicly. It accepts any POST so -you can iterate fast. Before exposing a webhook, set verifier: 'hmac-sha256' and a -secretEnv so only a sender holding the secret can enqueue work.
    • -
  • The signing secret lives in env, not the definition. secretEnv -names the env var; the runtime resolves it (resolveSecret) at ingress. Keep the secret -out of source and out of the definition object.
    • -
  • Return fast, work later. A handler should verify, validate, and -enqueueJob — not do the work inline. Webhook senders time out and retry; durable -processing belongs on the workers plugin.
    • -
  • De-dupe sender retries. Pass idempotencyKey (the event's key, or a -stable field) to enqueueJob so a retried POST collapses to one job.
    • -
  • File watches need stability on network shares. Set -stabilityThreshold so a half-written CSV does not fire mid-copy; use -forcePolling on the underlying watcher for network filesystems that miss native events.
    • -
  • Do not reach for defer. It is defined-but-unsupported, throws, and -routes to the DLQ. Schedule delayed work from the enqueued workers job, or use a -defineScheduledTrigger instead.
    • -
-{{ /comp }} - -## Reference - -{{ comp.xref({ key: "ref:triggers" }) }} - -{{ comp.featureGrid({ items: [ - { - title: "Look up — triggers core reference", - body: "The full generated API surface for the triggers unit: builders (defineWebhook / defineFileWatch / defineScheduledTrigger), runtime (ingress + processor), adapters (HMAC verifier), and the Hono router types.", - href: "/reference/triggers/", - icon: "≡" - }, - { - title: "Look up — watchers reference", - body: "createWatcher, FileWatcher, strategies (native/polling/hybrid), and the GlobFilter / StabilityFilter / DedupFilter pipeline.", - href: "/reference/watchers/", - icon: "≡" - }, - { - title: "Do — Background jobs", - body: "Where the enqueued work runs: defineJobHandler, worker runtimes, and the enqueue path triggers hand off to.", - href: "/capabilities/background-jobs/", - icon: "◆" - }, - { - title: "Understand — Durable sagas", - body: "The next link in the chain: how an enqueued job's saga message advances a durable workflow.", - href: "/capabilities/durable-sagas/", - icon: "◎" - } -] }) }} - -{{ comp.nextPrev({ prev: { label: "Durable sagas", href: "/capabilities/durable-sagas/" }, next: { label: "Durable streams", href: "/capabilities/streams/" } }) }} diff --git a/docs/site/data-persistence/database.md b/docs/site/data-persistence/database.md new file mode 100644 index 000000000..01c64e151 --- /dev/null +++ b/docs/site/data-persistence/database.md @@ -0,0 +1,392 @@ +--- +layout: layouts/base.vto +title: Database & Prisma +templateEngine: [vento, md] +prev: { label: "Durable streams", href: "/durable-workflows/streams/" } +next: { label: "KV, queues & cron", href: "/data-persistence/kv-queues-cron/" } +--- + +# Database & Prisma + +NetScript's persistence layer is **Prisma 7 over a driver adapter** (Postgres is the recommended engine, +with MSSQL and MySQL adapters shipped alongside), generated for the Deno runtime and +provisioned for you by **Aspire**. Every plugin contributes its own `.prisma` models, which +are aggregated into a single generated client over one primary datasource. + +{{ comp.diagram({ + src: "/assets/diagrams/database-schema-aggregation.svg", + alt: "Per-plugin .prisma schema files (root ExampleRecord, workers, sagas, triggers, auth) under database/postgres/schema/plugins are aggregated by netscript db generate into a single Deno-runtime Prisma client and matching zod schemas; the client talks to one datasource through a selected driver adapter — postgres, mssql, or mysql.", + caption: "One generated client, many contributors. Each plugin's .prisma file is aggregated into the same datasource; an adapter (postgres / mssql / mysql) selects the driver underneath." +}) }} + +{{ comp callout { type: "important", title: "Aspire is step 2 — before any db command" } }} +Every netscript db command talks to Postgres through Aspire. +In the default layout Postgres is a container that Aspire provisions, so +cd aspire && aspire start must be up before you run +netscript db init. Run a db command against a stopped AppHost and +it fails with aspire start failed: … Project file does not exist. That is the +dependency, not a bug — bring orchestration up first, then run the db workflow from the +workspace root. See Orchestration with Aspire for why. +{{ /comp }} + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for the database layer when you need durable, relational state — +records that survive restarts, are queried with a typed client, and are shared across your +service and its plugins. The engine is polyglot: pick it at scaffold time with +--db — Postgres (the recommended engine; or mysql / mssql / +sqlite). For ephemeral or high-throughput state (counters, locks, +work queues, scheduled triggers) reach for KV, +queues & cron instead; the scaffold uses both — Postgres for records, Redis/KV for +execution state. Postgres can also back the work queue itself — see the Postgres queue backend below. +{{ /comp }} + +## Learn → / Do → + +- **Learn** — the [Team Workspace tutorial, step 03](/tutorials/workspace/03-workspace-data/) + wires workspace data through the database from scratch. +- **Do** — the [Use a second database](/how-to/use-a-second-database/) recipe adds a second + adapter-backed datasource (MSSQL or MySQL) beside the primary Postgres. + +## How persistence is wired + +The scaffold engine is chosen with the `--db` flag — +`netscript init my-app --db postgres|mysql|mssql|sqlite`. Postgres is the recommended +engine (and what every tutorial uses); `mysql`, `mssql`, and `sqlite` are first-class +alternatives. `postgres` / `mysql` / `mssql` provision an Aspire container; `sqlite` is +file-backed with **no** Aspire container resource. The `--db postgres` scaffold +lays down a `database/postgres/` workspace (a different engine lays down +`database//`). A few facts are worth internalizing before you run anything, because +they differ from a typical single-file Prisma setup. + +```text +my-app/ +├── appsettings.json # NetScript.Databases.postgres — Engine/Mode/DatabaseName (the real config) +├── netscript.config.ts # databases.config is intentionally EMPTY here +└── database/postgres/ + ├── schema/ + │ ├── schema.prisma # root: generator client (runtime="deno"), generator zod, datasource db + │ └── plugins/ # per-plugin .prisma models aggregated here + │ ├── workers/workers.prisma + │ ├── sagas/sagas.prisma + │ ├── triggers/triggers.prisma + │ └── auth/auth.prisma # appears once the auth plugin is added (better-auth backend) + ├── prisma.config.ts # schema dir + migrations path + datasource url + ├── scripts/ # migrate.ts, seed.ts, generate-zod.ts, fix-zod-imports.ts, … + └── schema/.generated/ # appears after `db generate`: Prisma client + zod schemas +``` + +- **The datasource is driven by `appsettings.json`, not `netscript.config.ts`.** In the + scaffold, `netscript.config.ts` has `databases: { config: [] }` (empty). The connection + details live in `appsettings.json` under `NetScript.Databases.postgres` — + `Engine: "Postgres"`, `Mode: "Container"`, a generated `DatabaseName`, and a persistent + `DataPath` of `.data/postgres`, with `PrimaryDatabase: "postgres"`. Aspire reads that + block to provision the container. +- **One datasource, many contributors.** There is a single primary Postgres datasource. + Each first-party plugin ships its own `database/.prisma`, and those models are + aggregated under `database/postgres/schema/plugins//`. After adding `workers`, + `sagas`, and `triggers`, their tables (for example `model JobDefinition` from workers) + join the root `ExampleRecord` model in the same database — one `db init` / `db generate` + cycle covers the app and its plugins together. +- **The client is generated for Deno.** `schema.prisma` sets `generator client` with + `runtime = "deno"` and `output = "./.generated"`, plus a `generator zod` + (prisma-zod-generator) emitting matching zod schemas to `./.generated/zod`. You get a + type-safe client *and* runtime validation schemas from one generate. + +## Headline API: a model and a query + +Define models in Prisma schema files, then query them through the generated Deno client. +The root `schema.prisma` ships a single sample model, `ExampleRecord`; plugin schema files +add their own. The tabs below show a model alongside the typed query you'd write against +the generated client. + +{{ comp.tabbedCode({ tabs: [ + { + label: "schema.prisma", + lang: "prisma", + code: "// database/postgres/schema/schema.prisma\ngenerator client {\n provider = \"prisma-client\"\n output = \"./.generated\"\n runtime = \"deno\"\n}\n\ngenerator zod {\n provider = \"prisma-zod-generator\"\n output = \"./.generated/zod\"\n}\n\ndatasource db {\n provider = \"postgresql\"\n // The datasource URL is resolved by prisma.config.ts (POSTGRES_URI / DATABASE_URL),\n // not declared here.\n}\n\nmodel ExampleRecord {\n id String @id @default(uuid())\n name String\n createdAt DateTime @default(now())\n updatedAt DateTime @updatedAt\n}" + }, + { + label: "query.ts", + lang: "ts", + code: "// Import the Deno-runtime client generated by `netscript db generate`.\nimport { PrismaClient } from '../database/postgres/schema/.generated/client.server.ts';\n\nconst prisma = new PrismaClient();\n\n// Insert a record, then read it back — fully typed off the model above.\nconst created = await prisma.exampleRecord.create({\n data: { name: 'first' },\n});\n\nconst recent = await prisma.exampleRecord.findMany({\n orderBy: { createdAt: 'desc' },\n take: 10,\n});\n\nconsole.log(created.id, recent.length);" + } +] }) }} + +{{ comp callout { type: "note", title: "Generate after every schema change" } }} +The generated client under database/postgres/schema/.generated/ is a build +artifact. Edit a .prisma file and your code is still typed against the +old shape until you re-run netscript db generate. Generate +after every schema change, and re-run netscript db migrate to create and apply +the matching migration. +{{ /comp }} + +## The database workflow + +With `aspire start` up (Postgres reachable), run the public `netscript db` commands from the +workspace root in this order. The first three are the create-and-fill cycle; `status` +confirms it landed and `migrate` evolves the schema later. + +{{ comp.tabbedCode({ tabs: [ + { + label: "1 · init", + lang: "bash", + code: "# Create the first migration from the current Prisma schema and apply it.\n# --name labels the migration directory (here: a migration called \"init\").\nnetscript db init --name init" + }, + { + label: "2 · generate", + lang: "bash", + code: "# Generate the Deno-runtime Prisma client (and the zod schemas) into\n# database/postgres/schema/.generated so your code can import typed models.\nnetscript db generate" + }, + { + label: "3 · seed", + lang: "bash", + code: "# Run the workspace seed scripts (database/postgres/scripts/seed.ts)\n# to populate baseline rows.\nnetscript db seed" + }, + { + label: "4 · status", + lang: "bash", + code: "# Show migration / tooling status: which migrations are applied and\n# whether the database is in sync with the schema.\nnetscript db status" + }, + { + label: "5 · migrate", + lang: "bash", + code: "# Later, when the schema evolves: create + apply the next migration.\n# Edit a .prisma file, then:\nnetscript db migrate --name add_orders\nnetscript db generate # re-generate the client against the new shape" + } +] }) }} + +## Plugins contribute models + +Adding a first-party plugin installs its Prisma models to the *same* datasource — there is no +second database. The aggregated tables land under +`database/postgres/schema/plugins//` and migrate in the same `db init`/`migrate` +cycle as your app. + +{{ comp.apiTable({ + caption: "Plugin-contributed Prisma models (same primary datasource)", + rows: [ + { name: "workers", type: "@netscript/plugin-workers", desc: "Job and schedule definitions (for example JobDefinition) backing the durable background-job runtime." }, + { name: "sagas", type: "@netscript/plugin-sagas", desc: "Durable saga state — saga_runtime_state / saga_runtime_transition / saga_runtime_correlation — when the saga store backend is prisma (vs kv). See Durable sagas." }, + { name: "triggers", type: "@netscript/plugin-triggers", desc: "Trigger definitions and delivery bookkeeping for the inbound webhook/ingest runtime." }, + { name: "auth", type: "@netscript/plugin-auth", desc: "auth.prisma: auth_users / auth_sessions / auth_accounts / auth_verifications — used by the better-auth backend. (kv-oauth stores sessions in KV; WorkOS is stateless.)" } + ] +}) }} + +{{ comp callout { type: "note", title: "auth.prisma is backend-specific" } }} +The auth plugin contributes +auth.prisma with auth_users, auth_sessions, +auth_accounts, and auth_verifications. Those tables are the +persistence shape for the better-auth backend. The default +kv-oauth backend keeps sessions in KV and +WorkOS is stateless (sealed cookie), so they do not depend on these tables. +Run the auth migration only when you select the better-auth backend via +NETSCRIPT_AUTH_BACKEND. +{{ /comp }} + +{{ comp callout { type: "tip", title: "Postgres can also back the work queue" } }} +Persistence is not the only place Postgres shows up. The queue runtime now ships a +PostgreSQL backend alongside Deno KV, Redis, and RabbitMQ — a durable, +row-claimed (FOR UPDATE SKIP LOCKED) queue table in the same Postgres you +already provision. It is selectable only via an explicit +provider: 'postgres' (auto-discovery never picks it). See +KV, queues & cron for +connection.postgres.{url, tableName} and the full backend matrix. +{{ /comp }} + +## Adapter selection — postgres, mssql, mysql + +Prisma 7 talks to the database through a **driver adapter**. NetScript wraps each driver in +a small `DatabaseAdapter` that gives every backend the same lifecycle surface +(`getDriverAdapter` → `setClient` → `connect`/`disconnect`/`healthCheck`/`getStatus`). The +recommended scaffold is Postgres, but MSSQL and MySQL ship as sub-exports so you can target — or +add — a second engine without changing how your code consumes the client. + +The shared options shape comes from `DatabaseConnectionOptions` +(`@netscript/database/ports`): every adapter accepts either a `connectionString` or the +structured `host` / `port` / `database` / `username` / `password` / `ssl` / `poolSize` / +`timeout` parts, plus a few engine-specific extras. + +{{ comp.apiTable({ + caption: "Driver adapters (each implements DatabaseAdapter)", + rows: [ + { name: "adapters/postgres", type: "createPostgresAdapter(opts)", desc: "PostgreSQL via @prisma/adapter-pg. Re-exported from the @netscript/database/adapters barrel. PostgresConnectionOptions adds schema and applicationName. Builds a connection string under the hood." }, + { name: "adapters/mssql", type: "createMssqlAdapter(opts)", desc: "SQL Server via @prisma/adapter-mssql. Import from @netscript/database/adapters/mssql (sub-export — NOT in the barrel). MssqlConnectionOptions adds instanceName, trustServerCertificate, encrypt, integratedSecurity, applicationName, connectTimeout, requestTimeout. Accepts ADO.NET-style connection strings." }, + { name: "adapters/mysql", type: "createMysqlAdapter(opts)", desc: "MySQL 8.x / MariaDB via @netscript/prisma-adapter-mysql (a native-Deno driver). Import from @netscript/database/adapters/mysql (sub-export). MysqlConnectionOptions adds charset, timezone, connectionLimit, multipleStatements." } + ] +}) }} + +{{ comp.apiTable({ + caption: "DatabaseAdapter surface (every adapter)", + rows: [ + { name: "provider", type: "'postgres' | 'mssql' | 'mysql'", desc: "Readonly provider identity used in status reporting." }, + { name: "getDriverAdapter()", type: "→ driver adapter", desc: "Returns the Prisma driver adapter to pass as `new PrismaClient({ adapter })`. Lazily constructs it from the options." }, + { name: "setClient(client)", type: "(client) => void", desc: "Hand the constructed PrismaClient back to the adapter so connect / health / raw calls have a client to use." }, + { name: "getClient()", type: "→ PrismaClient", desc: "Return the configured Prisma client. Throws if setClient() has not been called — see the callout below." }, + { name: "connect() / disconnect()", type: "→ Promise", desc: "Explicit $connect / $disconnect. Prisma auto-connects on first query, so these are optional." }, + { name: "healthCheck()", type: "→ Promise", desc: "Lightweight `SELECT 1` probe; true when the connection answers." }, + { name: "getStatus()", type: "→ Promise", desc: "Snapshot: { connected, provider, database, host, lastConnected, error }." }, + { name: "executeRaw() / executeRawUnsafe()", type: "(query, ...params) => Promise", desc: "Raw query escape hatches routed through the configured client." } + ] +}) }} + +{{ comp callout { type: "important", title: "getClient() needs setClient() first" } }} +The adapter does not own the PrismaClient — you construct it. The flow is +always: build the adapter, call getDriverAdapter(), pass that to +new PrismaClient({ adapter }), then call adapter.setClient(client). +Only then does adapter.getClient() (and connect / +healthCheck / executeRaw) work — call getClient() +before setClient() and it throws +"… client not initialized. Call setClient() …". +{{ /comp }} + +## Tracing — Prisma OpenTelemetry spans + +Prisma query spans are wired through a lightweight tracing helper exported from +`@netscript/database/tracing`. It is a drop-in for `@prisma/instrumentation` that avoids the +CJS-heavy `@opentelemetry/instrumentation` dependency (which breaks Deno bundle/compile). +Call `enablePrismaTracing()` **once, before you construct any Prisma client**, and engine +spans dispatch into your OpenTelemetry tracer with the standard W3C `traceparent`. + +{{ comp.apiTable({ + caption: "@netscript/database/tracing — PrismaTracingConfig + hooks", + rows: [ + { name: "enablePrismaTracing(config?)", type: "(PrismaTracingConfig) => void", desc: "Register the tracing helper. Call once, before creating any PrismaClient." }, + { name: "config.tracerProvider", type: "PrismaTracingProvider", desc: "Tracer provider to use. Defaults to the globally registered OpenTelemetry provider." }, + { name: "config.ignoreSpanTypes", type: "(string | RegExp)[]", desc: "Span-name patterns to drop (string match or regex). Defaults to []." }, + { name: "disablePrismaTracing()", type: "() => void", desc: "Clear the global tracing helper." }, + { name: "isPrismaTracingEnabled()", type: "() => boolean", desc: "True when a tracing helper is currently registered." }, + { name: "enableInstrumentation()", type: "() => boolean (@netscript/database)", desc: "Convenience toggle from the package root: enables Prisma OTEL instrumentation only when OTEL_DENO=true; returns whether it was enabled." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Enable tracing", + lang: "ts", + code: "// services/orders/src/tracing.ts\nimport { enablePrismaTracing } from '@netscript/database/tracing';\n\n// Call ONCE, before constructing any PrismaClient. Engine spans then dispatch\n// into the globally registered OpenTelemetry tracer with a W3C traceparent.\nenablePrismaTracing({\n // tracerProvider defaults to the global provider; override only if you manage\n // your own. ignoreSpanTypes drops noisy internal spans by name or regex.\n ignoreSpanTypes: ['prisma:client:serialize', /detect_platform/],\n});\n\n// ... now construct the client (see the adapter example below).\nimport { PrismaClient } from '../database/postgres/schema/.generated/client.server.ts';\nconst prisma = new PrismaClient();" + }, + { + label: "Root toggle (OTEL_DENO)", + lang: "ts", + code: "// Alternatively, gate instrumentation on the OTEL_DENO flag from the package root.\nimport { enableInstrumentation } from '@netscript/database';\n\n// Returns true only when OTEL_DENO=true and wiring succeeded.\nconst tracing = enableInstrumentation();\nconsole.log('prisma tracing on:', tracing);" + } +] }) }} + +## Runnable example: wiring a second database (MySQL) + +The adapters let you stand up a **second** datasource beside the primary Postgres — for +example a reporting MySQL instance — without leaving the typed-client model. The pattern is +the same for every engine: create the adapter, pass `getDriverAdapter()` into a +`PrismaClient`, then `setClient()` so the adapter can manage lifecycle and health. Swap the +import to `adapters/mssql` and `createMssqlAdapter` for SQL Server — the surface is +identical. The MySQL adapter is a sub-export (not in the `adapters` barrel), so import it +from its own path. + +{{ comp.tabbedCode({ tabs: [ + { + label: "MySQL — createMysqlAdapter", + lang: "ts", + code: "// services/reporting/src/db.ts\n// MySQL ships as a sub-export — import from /adapters/mysql, not the barrel.\nimport { createMysqlAdapter } from '@netscript/database/adapters/mysql';\nimport { PrismaClient } from '../database/mysql/schema/.generated/client.server.ts';\n\n// 1) Build the adapter from structured parts (or pass { connectionString }).\nconst adapter = createMysqlAdapter({\n host: Deno.env.get('MYSQL_HOST') ?? 'localhost',\n port: 3306,\n database: 'reporting',\n username: Deno.env.get('MYSQL_USER') ?? 'root',\n password: Deno.env.get('MYSQL_PASSWORD'),\n ssl: false,\n});\n\n// 2) Pass the driver adapter into Prisma, then hand the client back.\nexport const reporting = new PrismaClient({ adapter: adapter.getDriverAdapter() });\nadapter.setClient(reporting);\n\n// 3) Now adapter lifecycle + health work off the same client.\nawait adapter.connect();\nconst ok = await adapter.healthCheck(); // SELECT 1\nconsole.log('mysql healthy:', ok, await adapter.getStatus());" + }, + { + label: "MSSQL — createMssqlAdapter", + lang: "ts", + code: "// services/reporting/src/db.ts — SQL Server variant (same shape)\nimport { createMssqlAdapter } from '@netscript/database/adapters/mssql';\nimport { PrismaClient } from '../database/mssql/schema/.generated/client.server.ts';\n\nconst adapter = createMssqlAdapter({\n host: Deno.env.get('MSSQL_SERVER') ?? 'localhost',\n port: 1433,\n database: 'reporting',\n username: 'sa',\n password: Deno.env.get('MSSQL_PASSWORD'),\n // local-dev TLS knobs; tighten for production\n encrypt: true,\n trustServerCertificate: true,\n});\n\nexport const reporting = new PrismaClient({ adapter: adapter.getDriverAdapter() });\nadapter.setClient(reporting);\nawait adapter.connect();" + }, + { + label: "Env config helper", + lang: "ts", + code: "// Prefer env-driven config? Each engine ships a getter that reads structured\n// vars (MYSQL_HOST/…) and falls back to a connection-string env var.\nimport { createMysqlAdapter } from '@netscript/database/adapters/mysql';\nimport { getMysqlConfig } from '@netscript/database/adapters/mysql';\n\n// getMysqlConfig() → MysqlAdapterConfig from MYSQL_* or MYSQLDB_URI / DATABASE_URL.\nconst cfg = getMysqlConfig();\nconst adapter = createMysqlAdapter({\n host: cfg.hostname,\n port: cfg.port,\n database: cfg.db,\n username: cfg.username,\n password: cfg.password,\n});\n// (getMssqlConfig / getPostgres equivalents exist per adapter.)" + } +] }) }} + +{{ comp callout { type: "note", title: "A second datasource is a second schema dir" } }} +A second engine is a separate Prisma schema workspace (for example +database/mysql/schema/) with its own generate output and its own +migrations — it does not merge into the Postgres aggregation above. Generate and +migrate each datasource independently. The full step-by-step lives in the +Use a second database recipe. +{{ /comp }} + +## Endpoints & ports + +The database itself isn't an HTTP service you call — it's a Postgres container Aspire +provisions and the generated Prisma client connects to. The relevant surfaces are the +Aspire resources and the connection-string env vars the workspace resolves. + +{{ comp.apiTable({ + caption: "Database surface (provisioned by Aspire)", + rows: [ + { name: "postgres", type: "Aspire resource", desc: "The Postgres container Aspire provisions from appsettings.json NetScript.Databases.postgres. Watch it go green in the dashboard." }, + { name: "redis", type: "Aspire resource", desc: "Redis cache — the default `--cache-backend`; Redis-compatible — backing KV/queues. A separate concern from Postgres; see KV, queues & cron. (`garnet` and `deno-kv` are alternative backends.)" }, + { name: "https://localhost:18888", type: "dashboard", desc: "Aspire dashboard (token printed by `aspire start`) — confirm the postgres resource is healthy." }, + { name: "POSTGRES_URI / DATABASE_URL", type: "env", desc: "Connection string resolved by prisma.config.ts and normalized to a URL. Set these yourself under --no-aspire." } + ] +}) }} + +{{ comp callout { type: "warning", title: "Production pitfalls" } }} +
    +
  • Forgetting Aspire. The most common failure: running netscript db init with no aspire start up. Start orchestration first, from the aspire/ folder.
    • +
  • Wrong directory. Run aspire start from aspire/, but run the netscript db commands from the workspace root (or pass --project-root).
    • +
  • Stale client after a schema change. Editing a .prisma file without re-running netscript db generate leaves your code typed against the old model.
    • +
  • Calling getClient() before setClient(). A second-database adapter throws until you hand it the constructed PrismaClient via setClient().
    • +
  • Tracing wired too late. enablePrismaTracing() must run before the first PrismaClient is constructed, or early query spans are lost.
    • +
  • Docker not running. Aspire provisions Postgres and Redis as containers; if Docker/Podman is down the postgres resource never goes green.
    • +
+{{ /comp }} + +{{ comp callout { type: "note", title: "MSSQL & MySQL are sub-exports" } }} +Postgres is the recommended engine and lives in the @netscript/database/adapters barrel. +The SQL Server and MySQL adapters are imported from their +own sub-paths — @netscript/database/adapters/mssql and +@netscript/database/adapters/mysql — so a Postgres-only app never pulls in the +heavier mssql / native-Deno MySQL drivers. The MySQL driver itself lives in the +standalone @netscript/prisma-adapter-mysql +package. +{{ /comp }} + +## Where to go next + +This hub is intentionally thin — the full generated API lives in the reference. Pick the +lane that matches what you're doing. + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Workspace data (Track B 03)", + body: "Guided tutorial: wire workspace data through Prisma, end to end, in the Team Workspace track.", + href: "/tutorials/workspace/03-workspace-data/", + icon: "→" + }, + { + title: "Do — Use a second database", + body: "Task recipe: add an MSSQL or MySQL datasource beside Postgres with a driver adapter.", + href: "/how-to/use-a-second-database/", + icon: "◆" + }, + { + title: "Do — Database & migration", + body: "Task recipe: bring up Postgres with Aspire, then run db init → generate → seed → status → migrate.", + href: "/how-to/database-migration/", + icon: "◆" + }, + { + title: "Look up — @netscript/database reference", + body: "The full generated API: ports, the postgres/mssql/mysql adapters, extensions, and tracing.", + href: "/reference/database/", + icon: "≡" + }, + { + title: "Understand — Orchestration with Aspire", + body: "Why the AppHost (aspire/apphost.mts) provisions Postgres and Redis, and how the resource graph fits together.", + href: "/explanation/aspire/", + icon: "◎" + }, + { + title: "MySQL — prisma-adapter-mysql", + body: "The native-Deno MySQL Prisma driver adapter surface, generated.", + href: "/reference/prisma-adapter-mysql/", + icon: "→" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Durable streams", href: "/durable-workflows/streams/" }, next: { label: "KV, queues & cron", href: "/data-persistence/kv-queues-cron/" } }) }} diff --git a/docs/site/data-persistence/index.md b/docs/site/data-persistence/index.md index 60f89b708..a788a8d4d 100644 --- a/docs/site/data-persistence/index.md +++ b/docs/site/data-persistence/index.md @@ -11,7 +11,7 @@ packages. Start here when the question is where state lives, how schema changes workspace resolves a second database. {{ comp.cardsGrid({ columns: 3, cards: [ - { eyebrow: "Overview & Concepts", title: "Database and KV", body: "The persistence layer behind services, queues, and workflow state.", href: "/capabilities/database/", icon: "O" }, + { eyebrow: "Overview & Concepts", title: "Database and KV", body: "The persistence layer behind services, queues, and workflow state.", href: "/data-persistence/database/", icon: "O" }, { eyebrow: "Quickstart", title: "Data contracts", body: "Follow the Storefront data step before service and saga work expand it.", href: "/tutorials/storefront/03-cart-contracts/", icon: "Q" }, { eyebrow: "How-To", title: "Database and migration", body: "Initialize and migrate the primary datasource.", href: "/how-to/database-migration/", icon: "H" }, { eyebrow: "How-To", title: "Second database", body: "Add and address another datasource from the workspace.", href: "/how-to/use-a-second-database/", icon: "H" }, diff --git a/docs/site/data-persistence/kv-queues-cron.md b/docs/site/data-persistence/kv-queues-cron.md new file mode 100644 index 000000000..7fc524909 --- /dev/null +++ b/docs/site/data-persistence/kv-queues-cron.md @@ -0,0 +1,378 @@ +--- +layout: layouts/base.vto +title: KV, queues & cron +templateEngine: [vento, md] +prev: { label: "Database & Prisma", href: "/data-persistence/database/" } +next: { label: "Telemetry & logging", href: "/observability/telemetry/" } +--- + +# KV, queues & cron + +The integration trio. Most backends end up hand-assembling three boring-but-load-bearing +seams: a **key-value store** for cache and session state, a **message queue** for +fire-and-forget work, and a **cron scheduler** for time-driven jobs. NetScript ships all +three as provider-agnostic packages — `@netscript/kv`, `@netscript/queue`, and +`@netscript/cron` — each exposing one typed contract across several backends and +**auto-detecting** the best available adapter from the Aspire environment, with a zero-config +local fallback so the same code runs on a laptop and in production. + +{{ comp.diagram({ + src: "/assets/diagrams/queue-worker-scheduler.svg", + alt: "A cron schedule fires a tick that enqueues work onto a provider-agnostic queue; one or more worker listeners drain the queue in parallel while a KV store holds cursors, dedupe keys, and counters that keep the loop idempotent.", + caption: "The trio in one loop: cron tick → enqueue → parallel queue listeners → KV state holds the cursor, dedupe keys, and counters." +}) }} + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for KV when you need fast, typed read/write state — caches, sessions, +feature flags, counters — with optional TTL and reactive watches. Reach for a +queue when you have fire-and-forget or fan-out work that should not +block a request: emails, webhooks, image processing. Reach for cron when work +is time-driven: nightly reports, hourly cleanups, weekday digests. For +message-driven, long-running orchestration with state, step up to a +durable saga; for inbound HTTP that kicks off +work use a trigger. +{{ /comp }} + +## What it is + +Each package is built on the same NetScript opinion: a single typed contract, several adapters +behind it, and provider selection that resolves automatically from the environment so you never +branch on "is this local or Aspire?" in product code. KV unifies **Deno KV and Redis-compatible caches (`redis` by default, or `garnet` via `--cache-backend`), plus +in-memory**; the queue unifies **Deno KV, Redis, RabbitMQ (AMQP), and PostgreSQL** (plus a +KV-polling fallback for remote KV Connect endpoints); cron unifies **native `Deno.cron()` and an +in-memory** scheduler. None of the three requires you to operate Redis, RabbitMQ, or PostgreSQL +during development — the local adapters carry the same code until [Aspire](/explanation/aspire/) +provisions the real backends. When [telemetry](/observability/telemetry/) is on, queues are traced +for you with no manual instrumentation. + +{{ comp.featureGrid({ items: [ + { + title: "@netscript/kv", + body: "Reactive key-value storage. getKv() auto-detects Deno KV or Redis/Garnet; construct MemoryKvAdapter directly for tests. get/set/watch/atomic with per-key TTL.", + href: "/reference/kv/", + icon: "◆" + }, + { + title: "@netscript/queue", + body: "Provider-agnostic message queues. createQueue() wraps Deno KV, Redis, RabbitMQ (AMQP), and PostgreSQL behind one MessageQueue contract; createParallelQueue() adds concurrency.", + href: "/reference/queue/", + icon: "≡" + }, + { + title: "@netscript/cron", + body: "Runtime-agnostic scheduling. createScheduler() uses native Deno.cron() when available and an in-memory adapter for tests.", + href: "/reference/cron/", + icon: "◎" + } +] }) }} + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Do — Choose a queue provider", + body: "Recipe: pick between Deno KV, Redis, RabbitMQ, and PostgreSQL — and wire the Aspire-provisioned, local-fallback, and explicit-provider paths.", + href: "/how-to/choose-a-queue-provider/", + icon: "→" + }, + { + title: "Understand — Orchestration with Aspire", + body: "How the scaffold wires Redis (the default cache backend; or Garnet via --cache-backend) and message brokers as resources, and why auto-detection just works once aspire start is up.", + href: "/explanation/aspire/", + icon: "◎" + } +] }) }} + +## One API, many backends + +Each package keeps a single typed contract with several adapters behind it, and provider +selection resolves automatically from the environment. The matrix below is the full provider +surface — read across a capability to see which backends it supports and how each is chosen. Note +one deliberate exception: the **PostgreSQL queue backend is explicit-provider only** — it is real +and durable, but auto-detection never selects it (see the callout below). + +{{ comp.apiTable({ + caption: "Adapter × capability — supported backends and how they are selected", + rows: [ + { name: "Deno KV", type: "kv · queue · cron", desc: "Default zero-config fallback. KV stores locally; the queue uses native Deno KV queue ops; cron shares the same local-first philosophy via Deno.cron(). No external service required." }, + { name: "memory", type: "kv · cron", desc: "Process-local. MemoryKvAdapter for KV (must be constructed explicitly) and the in-memory cron adapter (provider: 'memory') — deterministic for tests. The KV default for local development without Redis is Deno KV (local file), not the in-memory adapter." }, + { name: "Redis / Garnet", type: "kv · queue", desc: "Production cache and queue backend. Selected when CACHE_PROVIDER=redis|garnet, REDIS_URI/GARNET_URI, or Aspire services__redis__*/services__garnet__* are present. Redis is the default cache Aspire provisions; Garnet is the Redis-compatible alternative via --cache-backend garnet." }, + { name: "RabbitMQ (AMQP)", type: "queue", desc: "Durable broker for high-throughput, multi-consumer queues. Chosen first by auto-detection when Aspire reports a rabbitmq service. Imported via @netscript/queue/adapters/amqp for direct access." }, + { name: "PostgreSQL", type: "queue", desc: "Durable SQL-backed queue (FOR UPDATE SKIP LOCKED row-claim, visibility timeout, ack/nack, DLQ). EXPLICIT-PROVIDER ONLY — set provider: QueueProvider.Postgres; never auto-detected. Configure connection.postgres.{url,tableName}." }, + { name: "KV-polling", type: "queue", desc: "KvPollingAdapter — used automatically when the Deno KV path is a remote HTTP/HTTPS endpoint (KV Connect), where native queue ops are unavailable. Tunable via connection.denoKv.{pollInterval,visibilityTimeout,maxRetries}." }, + { name: "Deno.cron()", type: "cron", desc: "Native runtime scheduler. Used by createScheduler() whenever the runtime exposes Deno.cron(); falls back to the in-memory adapter otherwise." } + ] +}) }} + +{{ comp callout { type: "warning", title: "PostgreSQL queue is explicit-provider only" } }} +The queue's auto-detection order is RabbitMQ → Redis → Deno KV — it never +selects PostgreSQL. The Postgres backend is a real, durable adapter +(FOR UPDATE SKIP LOCKED row-claiming, visibility timeout, ack/nack, dead-letter +store), but you must opt in explicitly with provider: QueueProvider.Postgres (or +the string 'postgres'). The url in +connection.postgres is optional — when omitted, the adapter falls back to the +Aspire-provisioned Postgres URI — and tableName defaults to +message_queue. Reach for it when you want queue state to live in the same +transactional database as your domain data. +{{ /comp }} + +{{ comp callout { type: "important", title: "Aspire first — then the production adapters appear" } }} +Auto-detection upgrades to Redis/Garnet and RabbitMQ only when those resources are +running. Bring orchestration up first: +cd aspire && aspire start provisions Postgres, Redis, and any message broker +(dashboard at https://localhost:18888) +before any netscript db command or service that expects those +backends. Without Aspire, KV falls back to local Deno KV, the queue falls back to Deno KV, and +cron falls back to the in-memory adapter — your code is identical either way. See +Orchestration with Aspire. +{{ /comp }} + +## KV — key types first + +`getKv(config?)` returns a `WatchableKv` bound to the auto-detected backend. Keys are typed +tuples, values are fully typed, and `KvSetOptions.expireIn` (milliseconds) gives per-key TTL on +every adapter. Pass a `SharedKvConfig` to override provider selection on first access. + +{{ comp.apiTable({ + caption: "SharedKvConfig — first-access overrides for getKv()", + rows: [ + { name: "provider", type: "KvProvider?", desc: "Force a specific provider. The default 'auto' inspects the environment and chooses the best available backend." }, + { name: "path", type: "string?", desc: "Deno KV path or URL." }, + { name: "redisUrl", type: "string?", desc: "Explicit Redis connection URL." }, + { name: "redisNamespace", type: "string?", desc: "Prefix used for Redis-backed keys." }, + { name: "skipServiceDiscovery", type: "boolean?", desc: "Skip environment-based provider detection." } + ] +}) }} + +{{ comp.apiTable({ + caption: "KvSetOptions — per-key write options", + rows: [ + { name: "expireIn", type: "number?", desc: "Time-to-live in milliseconds; the key is removed after the window elapses. Honoured by every adapter." } + ] +}) }} + +`getKv()` is the headline entrypoint; the same module also exposes `closeKv()`, `resetKv()` +(for tests), `getRawKv()` (the underlying `Deno.Kv` when the provider is Deno KV), +`getActiveProvider()`, `isKvInitialized()`, and the `isWatchable()` type guard. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — auto-detected getKv()", + lang: "ts", + code: "// main.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv(); // picks Redis/Garnet under Aspire, else local Deno KV\n\nawait kv.set(['users', 'alice'], { name: 'Alice', role: 'admin' });\n\nconst entry = await kv.get<{ name: string; role: string }>(['users', 'alice']);\nconsole.log(entry?.value.name); // \"Alice\"\n\n// Per-key TTL works on every adapter — expire a session after one hour.\nawait kv.set(['sessions', 'tok_abc'], { userId: 'u1' }, { expireIn: 3_600_000 });" + }, + { + label: "Reactive — watch a prefix", + lang: "ts", + code: "// watch-jobs.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv();\n\n// Stream every change under a prefix, including newly created keys.\nfor await (const event of kv.watchPrefix(['jobs', 'order-processor'])) {\n console.log(`${event.key.join('/')} -> ${event.type}`, event.value);\n}" + }, + { + label: "Atomic — compare-and-swap", + lang: "ts", + code: "// counter.ts\nimport { getKv } from '@netscript/kv';\n\nconst kv = await getKv();\n\nconst entry = await kv.get(['counters', 'visits']);\nconst result = await kv.atomic(\n [{ key: ['counters', 'visits'], versionstamp: entry?.versionstamp ?? null }],\n [{ type: 'set', key: ['counters', 'visits'], value: (entry?.value ?? 0) + 1 }],\n);\n\nif (!result.ok) {\n // A concurrent write landed since the read — retry the cycle.\n}" + }, + { + label: "Tests / explicit backend", + lang: "ts", + code: "// kv.test.ts — deterministic, process-local store for tests.\nimport { MemoryKvAdapter } from '@netscript/kv';\n\nconst kv = new MemoryKvAdapter();\nawait kv.set(['test', 'key'], 'value');\n\n// Or pin Redis explicitly via the subpath import (keeps ioredis out of other graphs).\nimport { RedisKvAdapter } from '@netscript/kv/redis';\n\nconst redisKv = new RedisKvAdapter({ url: 'redis://localhost:6379', namespace: 'myapp' });\nawait redisKv.set(['cache', 'featured'], items, { expireIn: 60_000 });" + } +] }) }} + +## Queues — key types first + +`createQueue(name, options?)` returns a `MessageQueue` over the auto-detected backend. The +factory stays synchronous; the heavy Redis, RabbitMQ (AMQP), and PostgreSQL adapters resolve +lazily on first use and never enter your module graph until then. Producers call +`enqueue(message, options?)`; consumers call `listen(handler, options?)`; `stop()` drains and +releases the connection. The `QueueProvider` enum names the four backends — +`DenoKv`, `Redis`, `RabbitMQ`, `Postgres`. + +{{ comp.apiTable({ + caption: "QueueOptions — passed to createQueue / createTypedQueue / createParallelQueue", + rows: [ + { name: "provider", type: "QueueProvider?", desc: "Pin a backend. Omit to auto-discover (RabbitMQ → Redis → Deno KV). Set QueueProvider.Postgres to opt into the explicit-only Postgres adapter." }, + { name: "autoDiscover", type: "boolean? = true", desc: "Enable Aspire-environment discovery. Priority: RabbitMQ > Redis > Deno KV." }, + { name: "retryAttempts", type: "number? = 3", desc: "Max retries for failed messages — only applies when the backend has no native retry." }, + { name: "retryDelay", type: "number? = 1000", desc: "Delay between retry attempts in milliseconds — only when the backend has no native retry." }, + { name: "connection", type: "QueueConnectionOptions?", desc: "Provider-specific connection: connection.{denoKv,redis,rabbitmq,postgres}. KV Connect tuning lives under connection.denoKv (pollInterval, visibilityTimeout, maxRetries)." }, + { name: "deadLetterStore", type: "DeadLetterStorePort?", desc: "Custom terminal-failure store. When omitted, adapters use their provider-specific durable default." }, + { name: "disableAutoTracing", type: "boolean? = false", desc: "Skip the automatic TracedQueue wrapper when you prefer to wire spans by hand." } + ] +}) }} + +`createTypedQueue(name, schema, options?)` wraps a Zod schema and validates on enqueue/dequeue; +its `TypedQueueOptions` adds `validateOnEnqueue` (default true), `validateOnDequeue` (default +true), and `onValidationError` (`'discard'` default, `'dlq'`, or `'throw'`). `createParallelQueue` +takes the same options plus a `concurrency` count. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Enqueue — producer", + lang: "ts", + code: "// enqueue-email.ts\nimport { createQueue } from '@netscript/queue';\n\n// Auto-detects: RabbitMQ (AMQP) under Aspire, else Redis, else Deno KV.\nconst emails = createQueue<{ to: string; body: string }>('emails');\n\nawait emails.enqueue({\n to: 'user@example.com',\n body: 'Welcome to NetScript.',\n});\n\n// Delay availability (e.g. a 5-minute reminder) via EnqueueOptions.\nawait emails.enqueue({ to: 'user@example.com', body: 'Still there?' }, { delay: 5 * 60 * 1000 });" + }, + { + label: "Consume — listener", + lang: "ts", + code: "// consume-email.ts\nimport { createQueue } from '@netscript/queue';\n\nconst emails = createQueue<{ to: string; body: string }>('emails');\n\n// The handler receives the message and a MessageContext (messageId, deliveryCount, ack/nack).\nawait emails.listen(async (message, context) => {\n await sendEmail(message.to, message.body);\n // Some backends ack/retry natively (queue.nativeRetrial === true).\n});\n\n// On shutdown, drain in-flight work and release the backend connection.\nawait emails.stop();" + }, + { + label: "Parallel — bounded concurrency", + lang: "ts", + code: "// parallel-jobs.ts\nimport { createParallelQueue } from '@netscript/queue';\n\n// Process up to 4 I/O-bound messages concurrently on a single listener.\nconst jobs = createParallelQueue<{ orderId: string }>('order-jobs', { concurrency: 4 });\n\nawait jobs.listen(async (message) => {\n await fulfilOrder(message.orderId); // these run in parallel, up to 4 at a time\n});\n\n// concurrency <= 1 behaves exactly like createQueue. For CPU-bound work prefer Web Workers." + }, + { + label: "Typed — runtime validation", + lang: "ts", + code: "// typed-notifications.ts\nimport { z } from 'zod';\nimport { createTypedQueue } from '@netscript/queue';\n\nconst NotificationSchema = z.object({\n type: z.enum(['email', 'sms']),\n to: z.string(),\n body: z.string(),\n});\n\n// Invalid payloads route to a dead-letter queue instead of being discarded.\nconst notifications = createTypedQueue('notifications', NotificationSchema, {\n onValidationError: 'dlq',\n});" + }, + { + label: "PostgreSQL — durable, explicit", + lang: "ts", + code: "// pg-queue.ts\nimport { createQueue, QueueProvider } from '@netscript/queue';\n\n// PostgreSQL is never auto-detected — opt in explicitly.\nconst jobs = createQueue<{ orderId: string }>('order-jobs', {\n provider: QueueProvider.Postgres, // or provider: 'postgres'\n connection: {\n postgres: {\n // url is optional — falls back to the Aspire-provisioned Postgres URI.\n url: 'postgres://app:secret@localhost:5432/app',\n tableName: 'message_queue', // default table name\n },\n },\n});\n\nawait jobs.enqueue({ orderId: 'ord_123' });" + }, + { + label: "Pin another provider", + lang: "ts", + code: "// pin-provider.ts\nimport { createQueue, QueueProvider } from '@netscript/queue';\n\n// Force Redis regardless of what auto-detection would pick.\nconst jobs = createQueue('jobs', { provider: QueueProvider.Redis });\n\n// Or tune the Deno KV / KV-polling adapter for a remote KV Connect endpoint.\nconst remote = createQueue('jobs', {\n connection: {\n denoKv: { path: 'https://kv.example.com', pollInterval: 500, visibilityTimeout: 60_000 },\n },\n});" + } +] }) }} + +{{ comp callout { type: "note", title: "Queues are traced for you" } }} +When telemetry is enabled, createQueue() auto-wraps the queue with +@netscript/telemetry, so enqueue and consume spans show up in the Aspire dashboard +alongside RPC and SSE work — no manual instrumentation. Opt out with +disableAutoTracing: true when you want to wire spans by hand. See +Telemetry & logging. +{{ /comp }} + +## Cron — key types first + +`createScheduler(options?)` returns a `CronScheduler` that uses native `Deno.cron()` when the +runtime exposes it and an in-memory adapter otherwise. +`schedule(id, expression, handler, options?)` registers a job from a standard cron expression or +a `CronPresets` constant and resolves to a `ScheduledJob`; `trigger(id)` fires it manually (handy +in tests); `unschedule(id)` removes one; `stop()` tears the scheduler down. For simple +single-scheduler apps, `getScheduler()` returns a shared singleton (`stopScheduler()` resets it). +The handler may be a bare `JobHandler` or a `ContextualJobHandler` that receives a `JobContext` +(`jobId`, `scheduledTime`, `actualTime`, `attempt`, `signal`). + +{{ comp.apiTable({ + caption: "CreateSchedulerOptions — passed to createScheduler / getScheduler", + rows: [ + { name: "provider", type: "CronProvider?", desc: "Pin a provider. Omit to auto-detect by runtime — Deno.cron() when available, else the in-memory adapter." }, + { name: "tickInterval", type: "number? = 60000", desc: "Poll interval in milliseconds for the memory adapter; only used when provider is 'memory'." } + ] +}) }} + +{{ comp.apiTable({ + caption: "ScheduleOptions — per-job scheduling options", + rows: [ + { name: "timezone", type: "string? = 'UTC'", desc: "IANA timezone the cron expression is evaluated against, e.g. 'America/New_York'." }, + { name: "runOnInit", type: "boolean?", desc: "Run the handler once immediately on registration, in addition to the schedule." }, + { name: "enabled", type: "boolean? = true", desc: "Whether the job starts enabled. Register a job disabled and turn it on later." }, + { name: "backoff", type: "object?", desc: "Retry backoff: { type: 'fixed' | 'exponential' | 'linear', initialDelay, maxDelay?, multiplier? }. Delays are milliseconds." }, + { name: "maxRetries", type: "number?", desc: "Maximum retries on handler failure before the run is recorded as failed." }, + { name: "metadata", type: "Record?", desc: "Arbitrary metadata stored with the job and surfaced on its ScheduledJob record." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — preset schedule", + lang: "ts", + code: "// daily-report.ts\nimport { createScheduler, CronPresets } from '@netscript/cron';\n\nconst scheduler = createScheduler(); // native Deno.cron() when available\n\nconst report = async () => {\n await generateDailyReport();\n};\n\nawait scheduler.schedule('daily-report', CronPresets.WEEKDAYS_9AM, report, {\n timezone: 'America/New_York',\n});\n\n// Fire once on demand — useful for verifying wiring.\nawait scheduler.trigger('daily-report');\nawait scheduler.stop();" + }, + { + label: "Raw expression + context", + lang: "ts", + code: "// hourly-cleanup.ts\nimport { createScheduler } from '@netscript/cron';\n\nconst scheduler = createScheduler({ provider: 'memory', tickInterval: 100 });\n\n// Hourly cleanup; the contextual handler receives scheduling metadata.\nconst cleanup = async (context) => {\n console.log(context.jobId, context.attempt, context.scheduledTime.toISOString());\n};\n\nawait scheduler.schedule('cleanup', '0 * * * *', cleanup);" + }, + { + label: "Lifecycle events", + lang: "ts", + code: "// observe-jobs.ts\nimport { createScheduler } from '@netscript/cron';\n\nconst scheduler = createScheduler({ provider: 'memory' });\n\n// Observe jobRun / jobError / jobScheduled / jobUnscheduled.\nscheduler.on('jobRun', (event) => {\n if (!event.result.success) {\n console.error(`Job ${event.jobId} failed`, event.result.error);\n }\n});" + }, + { + label: "Validate expressions", + lang: "ts", + code: "// validate-cron.ts\nimport { isValidCronExpression, parseCronExpression } from '@netscript/cron';\n\nif (isValidCronExpression('0 9 * * 1-5')) {\n const parsed = parseCronExpression('0 9 * * 1-5');\n console.log(parsed?.hour, parsed?.dayOfWeek);\n}" + } +] }) }} + +## How they compose + +The trio is strongest together. A common pattern: a **cron** job wakes on a schedule and +**enqueues** a batch of work; **queue** consumers process each message in parallel; and a **KV** +store holds the cursor, dedupe keys, or rate-limit counters that keep the whole thing idempotent +across restarts. None of these requires you to operate Redis, RabbitMQ, or PostgreSQL during +development — the local Deno KV / in-memory adapters carry the same code until Aspire provisions +the real backends (or until you opt in to the PostgreSQL queue explicitly). + +{{ comp.apiTable({ + caption: "Picking the right primitive", + rows: [ + { name: "Key-value state", type: "@netscript/kv", desc: "Synchronous-feeling read/write state: caches, sessions, flags, counters, cursors. Use TTL for ephemerality and watches for reactivity." }, + { name: "Fire-and-forget work", type: "@netscript/queue", desc: "Decouple slow work from the request path. Fan out to multiple consumers; let the backend handle retries where nativeRetrial is true. Four backends: Deno KV, Redis, AMQP, PostgreSQL." }, + { name: "Time-driven work", type: "@netscript/cron", desc: "Run handlers on a schedule. Pair with a queue to fan a scheduled tick out into many parallel jobs." }, + { name: "Stateful orchestration", type: "durable saga", desc: "When work spans steps with correlation and compensation, a queue is not enough — model it as a durable saga instead." } + ] +}) }} + +{{ comp callout { type: "important", title: "Production notes" } }} +Review these production constraints before deploying: (1) the PostgreSQL queue is +never auto-detected — if you forget provider: QueueProvider.Postgres you +silently get the Deno KV fallback. (2) retryAttempts / +retryDelay apply only when the backend lacks native retry; check +queue.nativeRetrial before assuming your config is in effect. (3) +KV per-key TTL is set via expireIn (milliseconds), not an absolute timestamp. +(4) cron timezone defaults to UTC — set it explicitly +for business-hours schedules. (5) always await scheduler.stop() and +await queue.stop() on shutdown so in-flight work drains and connections release. +{{ /comp }} + +## Reference → + +{{ comp.xref({ key: "ref:kv" }) }} · {{ comp.xref({ key: "ref:queue" }) }} · +{{ comp.xref({ key: "ref:cron" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Do — Choose a queue provider", + body: "Task recipe: stand up each primitive in an existing workspace, with the Aspire-provisioned, local-fallback, and explicit-PostgreSQL paths spelled out.", + href: "/how-to/choose-a-queue-provider/", + icon: "◆" + }, + { + title: "Look up — @netscript/kv reference", + body: "Full generated API: getKv, WatchableKv, KvSetOptions, SharedKvConfig, the Deno KV / memory / Redis adapters, and the auto-detection table.", + href: "/reference/kv/", + icon: "≡" + }, + { + title: "Look up — @netscript/queue reference", + body: "createQueue, createTypedQueue, createParallelQueue, QueueProvider, QueueOptions, MessageQueue, and the Deno KV / Redis / AMQP / PostgreSQL / KV-polling adapters.", + href: "/reference/queue/", + icon: "≡" + }, + { + title: "Look up — @netscript/cron reference", + body: "createScheduler, getScheduler, CronPresets, CronScheduler, ScheduleOptions, JobContext, and the Deno.cron() and memory adapters.", + href: "/reference/cron/", + icon: "≡" + }, + { + title: "Understand — Orchestration with Aspire", + body: "How the scaffold wires Redis (the default cache backend; or Garnet via --cache-backend) and message brokers as resources, and why auto-detection just works once aspire start is up.", + href: "/explanation/aspire/", + icon: "◎" + }, + { + title: "Understand — Telemetry & logging", + body: "Why queues are traced by default and how KV/queue/cron spans land in the Aspire dashboard.", + href: "/observability/telemetry/", + icon: "◎" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Database & Prisma", href: "/data-persistence/database/" }, next: { label: "Telemetry & logging", href: "/observability/telemetry/" } }) }} diff --git a/docs/site/durable-workflows/sagas.md b/docs/site/durable-workflows/sagas.md new file mode 100644 index 000000000..d97c76e3d --- /dev/null +++ b/docs/site/durable-workflows/sagas.md @@ -0,0 +1,508 @@ +--- +layout: layouts/base.vto +title: Durable sagas +templateEngine: [vento, md] +prev: { label: "Background jobs", href: "/background-processing/workers/" } +next: { label: "Triggers & ingress", href: "/durable-workflows/triggers/" } +--- + +# Durable sagas + +A saga is the answer to the question every retry loop dodges: *what happens between +step three and step four when the process dies?* In NetScript a saga is an explicit, +message-driven state machine — you declare the state it carries, the messages it +reacts to, and the effects each handler emits. It is authored with a single fluent +builder, persisted to a durable store, and served by the **sagas plugin** on port +`:8092`. The model is closer to Temporal than to a job queue, but it lives in plain +TypeScript inside your workspace — no separate cluster to operate. + +{{ comp.diagram({ + src: "/assets/diagrams/saga-state-machine.svg", + alt: "Saga state machine: an event advances the saga through typed states, a compensation branch unwinds a failed step, and every transition checkpoints to a kv or prisma store.", + caption: "A saga is a state machine. Each inbound message runs a handler that mutates typed state and returns an effect ledger (advance, complete, fail, compensate); the runtime checkpoints state between messages to the durable store (kv or prisma) so an instance survives a crash." +}) }} + +This is the third capability in the continuous-app thread. The +[background jobs](/background-processing/workers/) capability ended with a job, +`create-user-settings`, that publishes a `UserSettingsCreated` message. Here that +message stops being fire-and-forget: a saga **consumes it**, advances its own state, +and emits a `sagaComplete(...)` effect that the runtime records as a first-class +outcome — and persists durably so the instance survives a process restart. + +{{ comp callout { type: "note", title: "Where it lives" } }} +The plugin is installed at plugins/sagas/ and referenced by +netscript.config.ts as ./plugins/sagas/mod.ts. The fluent +builder and effect helpers come from @netscript/plugin-sagas-core; the +config-time companion (defineSagaConfig) comes from +@netscript/plugin-sagas-core/config; the durable runtime factory +(createDurableSagaRuntime) and the HTTP publisher +(createSagaPublisher) come from the +@netscript/plugin-sagas/runtime subpath. Add it to a workspace with +the public package install flow (netscript plugin install @netscript/plugin-sagas). +For local-source contributor samples inside this monorepo, run +deno run -A packages/cli/bin/netscript-dev.ts plugin install saga --name sagas --samples. +{{ /comp }} + +## What it is + +A saga is a **durable, message-driven state machine**: a typed state object plus a set +of handlers, one per message type, that advance that state and return an effect ledger. +The runtime persists state between every message, so an instance that is mid-flight when +the process restarts resumes exactly where it left off. Unlike a background job — a +single idempotent unit — a saga correlates many messages into one long-running instance +and gives compensation (the *undo* of an already-applied step) a first-class place in +the model. Read the conceptual companion, the +{{ comp.xref({ key: "explain:durable-workflows" }) }}, for why state must outlive the +process and how effect-based outcomes differ from a retry loop. + +## Learn → · Do → + +{{ comp.featureGrid({ + columns: 2, + items: [ + { icon: "◆", title: "Learn — Build the checkout saga", body: "The storefront tutorial rung that adds the sagas plugin, correlates the order events, and compensates a failed payment step.", href: "/tutorials/storefront/04-checkout-saga/" }, + { icon: "→", title: "Do — Wire a durable workflow", body: "Add the sagas plugin to a running app, consume a cross-plugin message, and emit sagaComplete end to end.", href: "/tutorials/storefront/04-checkout-saga/" } + ] +}) }} + +{{ comp.featureGrid({ + columns: 3, + items: [ + { icon: "◆", title: "Fluent builder", body: "defineSaga(id).durability().state().on().compensate().build() — id, durability tier, typed state, message handlers, compensations, then build(). One chain, fully type-checked." }, + { icon: "▣", title: "Durable store backend", body: "Runtime state persists to kv or prisma, chosen by NETSCRIPT_SAGA_STORE / appsettings. createDurableSagaRuntime({ backend, prisma }) owns the resources." }, + { icon: "≋", title: "Effect-based outcomes", body: "Every handler returns an array of effects — sagaComplete, sagaFail, sagaCompensate, send, schedule, spawn are named outcomes returned from handlers, never a fall-through." }, + { icon: "⊡", title: "Served on :8092", body: "An oRPC API lists registered sagas, inspects running instances, publishes messages, and streams activity over SSE." }, + { icon: "⇄", title: "Cross-plugin choreography", body: "The workers create-user-settings job publishes UserSettingsCreated; this saga consumes it — one message crossing the plugin boundary, type-checked on both sides." }, + { icon: "◷", title: "Crash-survivable", body: "State checkpoints between messages, so an instance picks up exactly where it left off after a restart. That survival is the entire point of a saga." } + ] +}) }} + +## Minimal example — an order saga with compensation + +The runnable shape you grow into: a real state type, multiple `.on(...)` handlers, and a +`.compensate(...)` branch that unwinds an already-charged payment when fulfillment fails. +The first handler reserves stock and emits a `send(...)` effect to charge payment; the +failure path returns `sagaCompensate(...)` so the engine routes the matching compensation +handler, which refunds and then fails the instance. + +```ts +// plugins/sagas/order-saga.ts +import { + defineSaga, + sagaComplete, + sagaCompensate, + sagaFail, + send, +} from '@netscript/plugin-sagas-core'; + +// The state is the source of truth that survives across messages and crashes. +type OrderState = Readonly<{ + status: 'awaiting-payment' | 'awaiting-fulfillment' | 'completed' | 'refunded'; + orderId?: string; + paymentId?: string; +}>; + +type OrderPlaced = Readonly<{ orderId: string }>; +type PaymentCaptured = Readonly<{ orderId: string; paymentId: string }>; +type FulfillmentFailed = Readonly<{ orderId: string; reason: string }>; + +export const orderSaga = defineSaga('order-saga') + // Durability tier: where the runtime checkpoints state between messages. + .durability('t1') + // Typed initial state, seeded once at instance creation. + .state({ status: 'awaiting-payment' }) + // Handler 1: an order was placed — record it and ask the payment service to charge. + .on<'OrderPlaced', OrderPlaced>('OrderPlaced', (saga, event) => { + saga.state = { ...saga.state, orderId: event.payload.orderId }; + // `send` is an effect: dispatch a command to another target, not a direct call. + return [send({ kind: 'service', id: 'payments' }, { orderId: event.payload.orderId }, {})]; + }) + // Handler 2: payment captured — advance toward fulfillment. + .on<'PaymentCaptured', PaymentCaptured>('PaymentCaptured', (saga, event) => { + saga.state = { + ...saga.state, + status: 'awaiting-fulfillment', + paymentId: event.payload.paymentId, + }; + return []; + }) + // Handler 3: fulfillment failed AFTER we charged — request the compensation branch. + .on<'FulfillmentFailed', FulfillmentFailed>('FulfillmentFailed', (saga, event) => { + return [sagaCompensate({ type: 'FulfillmentFailed', payload: event.payload }, event.payload.reason)]; + }) + // Compensation: undo the already-applied payment, then fail the instance. + .compensate<'FulfillmentFailed', FulfillmentFailed>('FulfillmentFailed', (saga, event) => { + saga.state = { ...saga.state, status: 'refunded' }; + return [ + send({ kind: 'service', id: 'payments' }, { refund: saga.state.paymentId }, {}), + sagaFail(`order ${event.payload.orderId} unfulfilled: ${event.payload.reason}`), + ]; + }) + .build(); + +export default orderSaga; +``` + +{{ comp callout { type: "important", title: "Compensation: a .compensate() branch OR an effect" } }} +There are two ways to compensate, and they work together. The builder +exposes a first-class .compensate(eventType, handler) that registers an undo +handler keyed by message type. A running handler routes into it by returning the +sagaCompensate(message, reason?) effect. So a saga's whole lifecycle — +advance, sagaComplete, sagaFail, sagaCompensate, +send, schedule, spawn — is named outcomes +returned from handlers, never a fall-through or an unhandled throw. The undo logic +itself lives in the .compensate() handler, not inline in the forward path. +{{ /comp }} + +## Key types first + +Before the options, the primary interfaces the DSL works in. `SagaState` is the base +shape your typed state must satisfy; `SagaContext` is the read-only context passed to +every handler; `SagaDefinition` is the frozen object `build()` returns and the runtime +registers. + +{{ comp.apiTable({ + caption: "Primary saga types (@netscript/plugin-sagas-core)", + rows: [ + { name: "SagaState", type: "Readonly>", desc: "Base state shape every saga's typed state must extend. Your State type is intersected with this." }, + { name: "SagaMessage", type: "{ type, payload, correlationKey?, idempotencyKey?, concurrencyKey?, occurredAt?, traceparent? }", desc: "The event or command delivered to a handler. `type` discriminates; `payload` is your typed body." }, + { name: "SagaContext", type: "{ sagaId, instanceId, correlationKey, state, message, attempt, now, traceparent? }", desc: "Read-only handler context. `now` is the injected clock; `attempt` is the retry count; trace fields carry W3C context." }, + { name: "SagaHandler", type: "(saga, event, context) => readonly CascadedMessage[]", desc: "A synchronous handler: it mutates saga.state and RETURNS an effect ledger. No async, no direct I/O." }, + { name: "SagaDefinition", type: "Readonly<{ id, durability, initialState, handlers, compensations, correlations, retry?, concurrency?, schedule? }>", desc: "The frozen definition build() produces. Registered into the runtime and into KV under ['saga','registry', id]." }, + { name: "SAGA_DURABILITY_TIERS", type: "readonly ['t1','t2','t3']", desc: "The durability tiers a definition may declare via .durability(tier). 't1' is the scaffolded default." } + ] +}) }} + +## The builder API + +A saga is built with `defineSaga(id)` and a typestate chain: `.state()` must come before +any handler, and `.build()` requires at least one handler. These are the methods on the +returned `SagaBuilder`. + +{{ comp.apiTable({ + caption: "defineSaga(id) — SagaBuilder methods", + rows: [ + { name: ".durability(tier)", type: "SagaDurabilityTier ('t1' | 't2' | 't3')", desc: "Set the persistence tier the runtime checkpoints state to. Defaults to t1. Distinct from the kv/prisma store backend (see below)." }, + { name: ".state(initial)", type: "S extends SagaState", desc: "Declare the typed initial state, seeded once at instance creation. MUST be called before any handler (typestate-enforced)." }, + { name: ".on(type, handler)", type: "SagaHandler", desc: "Register a forward handler for a message type. Receives (saga, event, context), mutates saga.state, returns an effect ledger." }, + { name: ".compensate(type, handler)", type: "SagaHandler", desc: "Register a compensation (undo) handler keyed by message type. Routed when a handler returns sagaCompensate(...)." }, + { name: ".correlate(rule)", type: "SagaCorrelation", desc: "Extract a correlation key from an incoming message so it routes to the right running instance." }, + { name: ".concurrency(opts)", type: "{ limit: number; key?: (m) => string }", desc: "Bound how many messages run at once, optionally per derived key. Overlapping publishes for one key are rejected." }, + { name: ".schedule(cron)", type: "string", desc: "Attach a cron expression that ticks the saga definition on a schedule." }, + { name: ".onSignal(signal, handler)", type: "SignalDefinition, handler", desc: "Register a reserved signal handler (defineSignal). Runtime dispatch is deferred in the alpha." }, + { name: ".onQuery(query, handler)", type: "QueryDefinition, handler", desc: "Register a reserved synchronous read-only query handler (defineQuery). Promises are rejected at type level." }, + { name: ".build()", type: "=> SagaDefinition", desc: "Freeze the chain into a SagaDefinition. Requires at least one .on(...) handler (typestate-enforced)." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — minimal saga", + lang: "ts", + code: "// plugins/sagas/.ts\nimport { defineSaga, sagaComplete } from '@netscript/plugin-sagas-core';\n\ntype State = Readonly<{ status: string; processedAt?: string }>;\n\n// Build a saga: id -> durability tier -> typed state -> message handlers -> build().\nexport const userSettingsSaga = defineSaga('user-settings-saga')\n .durability('t1')\n .state({ status: 'pending' })\n .on<'UserSettingsCreated', { userId: string }>(\n 'UserSettingsCreated',\n (saga, event, context) => {\n // Advance the saga's own state, then emit completion as a recorded outcome.\n saga.state = {\n ...saga.state,\n status: 'completed',\n processedAt: context.now.toISOString(),\n };\n return [sagaComplete({\n userId: event.payload.userId,\n processedAt: context.now.toISOString(),\n })];\n },\n )\n .build();\n\nexport default userSettingsSaga;" + }, + { + label: "Config-time companion", + lang: "ts", + code: "// netscript.config.ts (excerpt)\nimport { defineSagaConfig } from '@netscript/plugin-sagas-core/config';\n\n// The config-time entry the scaffolder + CLI read. SEPARATE from the runtime\n// definition that defineSaga(...).build() produces.\nexport const orderSagaEntry = defineSagaConfig('order-saga', './plugins/sagas/order-saga.ts')\n .name('Order saga')\n .description('Reserve, charge, fulfill — with a compensation branch.')\n .topic('orders')\n .tags(['orders', 'checkout'])\n .build();" + } +] }) }} + +## Effect helpers + +A handler's only side effect is the array of **cascaded messages** it returns. These +helpers (`@netscript/plugin-sagas-core`) construct the named effects. Every kind in +`CASCADED_MESSAGE_KINDS` (`send | scheduled | spawn | complete | fail | compensate`) has +a constructor. + +{{ comp.apiTable({ + caption: "Effect helpers — saga handler outcomes", + rows: [ + { name: "sagaComplete(result?)", type: "=> CascadedMessage<'complete'>", desc: "Terminal success. Marks the instance finished and records the optional result payload." }, + { name: "sagaFail(reason)", type: "string | Error => CascadedMessage<'fail'>", desc: "Terminal failure. Records the reason; no further messages are applied to the instance." }, + { name: "sagaCompensate(message, reason?)", type: "=> CascadedMessage<'compensate'>", desc: "Route into the matching .compensate(type, ...) handler to undo an already-applied step." }, + { name: "send(target, payload, options)", type: "=> CascadedMessage<'send'>", desc: "Dispatch a command to a target (job, saga, or runtime adapter). options carry idempotencyKey / concurrencyKey / retry / queue." }, + { name: "schedule(message, delay)", type: "delay: Date | number | '5m' => CascadedMessage<'scheduled'>", desc: "Deliver a wrapped message after a delay (a Date, ms, or a '30s'/'5m'/'2h'/'1d' string)." }, + { name: "spawn(child, input, options)", type: "=> CascadedMessage<'spawn'>", desc: "Start a child saga from a definition or id, passing typed input. options take idempotencyKey / concurrencyKey." } + ] +}) }} + +## createParallelQueue — fan-out and concurrent processing + +Saga handlers stay synchronous and pure; the *transport* layer that carries cascaded +messages and feeds fan-out work is where you tune concurrency. `createParallelQueue` (from +{{ comp.xref({ key: "ref:queue", text: "@netscript/queue" }) }}) is the primitive for +that: it wraps a base queue so a single listener processes several messages at once. It is +the right tool when a saga spawns many independent children or pushes I/O-bound side work +(API calls, DB writes) that should run in parallel rather than one at a time. The +`concurrency` option is the whole story — `1` is plain sequential, anything higher enables +parallel processing. + +{{ comp.apiTable({ + caption: "createParallelQueue(name, options) — ParallelQueueOptions (extends QueueOptions)", + rows: [ + { name: "concurrency", type: "number (default 1)", desc: "Number of concurrent processors. Must be >= 1; values > 1 wrap the queue for parallel listening. Use for I/O-bound work; for CPU-bound work prefer web-worker tasks." }, + { name: "provider", type: "QueueProvider ('deno-kv' | 'redis' | 'rabbitmq' | 'postgres')", desc: "Backing queue provider. Omit to auto-discover from the Aspire environment." }, + { name: "autoDiscover", type: "boolean (default true)", desc: "Discover a queue service from Aspire. Priority RabbitMQ > Redis > Deno KV." }, + { name: "retryAttempts", type: "number (default 3)", desc: "Max retry attempts for failed messages, when the backend lacks native retry." }, + { name: "retryDelay", type: "number ms (default 1000)", desc: "Delay between retries, when the backend lacks native retry." }, + { name: "connection", type: "QueueConnectionOptions", desc: "Provider-specific connection options (denoKv / redis / rabbitmq / postgres)." }, + { name: "deadLetterStore", type: "DeadLetterStorePort", desc: "Where terminal message failures land. Omit to use the provider's durable default." }, + { name: "disableAutoTracing", type: "boolean (default false)", desc: "Skip the automatic TracedQueue wrapper when you trace manually." } + ] +}) }} + +```ts +// plugins/sagas/fan-out.ts — process saga side-effects concurrently +import { createParallelQueue, QueueProvider } from '@netscript/queue'; + +type NotifyMessage = Readonly<{ orderId: string; channel: 'email' | 'sms' }>; + +// Four messages processed at a time on a single listener — for I/O-bound fan-out +// (notifications, webhook calls) a saga emits as it advances. +const notifications = createParallelQueue('order-notifications', { + concurrency: 4, + provider: QueueProvider.Redis, +}); + +await notifications.listen(async (message) => { + // These run up to 4 at a time, not serially. + await deliverNotification(message.orderId, message.channel); +}); +``` + +{{ comp callout { type: "note", title: "Concurrency lives in two places" } }} +createParallelQueue({ concurrency }) tunes how many queue messages a +listener processes at once. defineSaga().concurrency({ limit, key }) bounds +how many messages run against one saga instance (or per derived key) — overlapping +publishes for the same key are rejected. Use the queue knob for fan-out throughput; use the +builder knob to serialize work on a single business entity (one order, one tenant). +{{ /comp }} + +## Choosing a durable store backend + +Authoring a saga decides *what* it does. The **durable store backend** decides *where +its runtime state lives between messages and across crashes*. NetScript ships two +backends, and the choice is **explicit and mandatory** — the runtime refuses to start +without one. + +- **`kv`** — durable saga state in Deno KV (the orchestration store stood up by + Aspire). Zero extra schema; the natural default for a single-service app and for + local development. +- **`prisma`** — durable saga state in your scaffolded relational database via Prisma + (Postgres in the recommended setup; equally `mysql` / `mssql` / `sqlite`, since the store writes through + your project's Prisma client — it is not Postgres-specific). The `PrismaSagaStore` + writes the dedicated runtime tables `saga_runtime_state`, `saga_runtime_transition`, + and `saga_runtime_correlation`. Reach for this when you want the saga's own write + path in your relational database alongside the rest of your data, with SQL-level + inspection of in-flight state and transition history. + +You select the backend with the `NETSCRIPT_SAGA_STORE` environment variable (`kv` | `prisma`) or the appsettings key `sagas.store.backend`. The plugin service resolves this on startup via `resolveSagaStoreBackend(...)`, which **throws** if neither is set — there is no silent default in the resolver, by design, so a deployment can never guess wrong about where durable state lands. (Calling `createDurableSagaRuntime(...)` directly without a `backend` falls back to the KV store; the mandatory-selection guarantee comes from the resolver the service runs at startup.) + +{{ comp.apiTable({ + caption: "Durable saga store backends — trait matrix", + rows: [ + { name: "kv", type: "Deno KV", desc: "Default for local/single-service. No extra schema. Provisioned by Aspire (Redis/KV). Resolved via NETSCRIPT_SAGA_STORE=kv or sagas.store.backend=kv." }, + { name: "prisma", type: "Relational / Prisma", desc: "Writes saga_runtime_state, saga_runtime_transition, saga_runtime_correlation. Requires a Prisma client passed to createDurableSagaRuntime — so it follows whatever engine you scaffolded (Postgres in the recommended setup; mysql / mssql / sqlite all work). SQL-inspectable in-flight state. Resolved via NETSCRIPT_SAGA_STORE=prisma or sagas.store.backend=prisma." }, + { name: "selection", type: "mandatory", desc: "No implicit default. resolveSagaStoreBackend(...) throws when neither NETSCRIPT_SAGA_STORE nor sagas.store.backend is set." }, + { name: "client requirement", type: "prisma only", desc: "backend: 'prisma' (or passing prisma) without a Prisma client throws 'Prisma saga store backend requires a Prisma client.'" } + ] +}) }} + +The factory that owns these resources is `createDurableSagaRuntime(...)` from the +`@netscript/plugin-sagas/runtime` subpath. It resolves a `SagaStorePort`, builds the +native runtime over it, and hands you back a `dispose()` that closes the store (and the +KV handle it opened). + +{{ comp.tabbedCode({ tabs: [ + { + label: "kv backend", + lang: "ts", + code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\n\n// Deno KV durable store — the default for local/single-service apps.\n// Selected at deploy time by NETSCRIPT_SAGA_STORE=kv (or appsettings sagas.store.backend=kv).\nconst { runtime, store, dispose } = await createDurableSagaRuntime({\n backend: 'kv',\n // kv is opened for you if you don't inject one (openSagaRuntimeKv()).\n});\n\n// ... register saga definitions on `runtime`, process messages ...\n\nawait dispose(); // closes the KV-backed store + handle" + }, + { + label: "prisma backend", + lang: "ts", + code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\nimport { PrismaClient } from './generated/prisma/client.ts';\n\n// Postgres/Prisma durable store — writes saga_runtime_* tables.\n// Selected at deploy time by NETSCRIPT_SAGA_STORE=prisma.\nconst prisma = new PrismaClient();\nconst { runtime, store, dispose } = await createDurableSagaRuntime({\n backend: 'prisma',\n prisma, // REQUIRED for prisma — omitting it throws.\n});\n\n// ... register saga definitions, process messages — transitions land in\n// saga_runtime_state / saga_runtime_transition / saga_runtime_correlation ...\n\nawait dispose();" + }, + { + label: "resolve from env / appsettings", + lang: "ts", + code: "import { createDurableSagaRuntime } from '@netscript/plugin-sagas/runtime';\nimport { resolveSagaStoreBackend } from '@netscript/plugin-sagas-core/stores';\n\n// Read the backend from the environment (or appsettings) — throws if unset.\nconst backend = resolveSagaStoreBackend({\n env: Deno.env.toObject(),\n // appsettings: loadedAppsettings, // sagas.store.backend\n});\n\nconst runtime = await createDurableSagaRuntime({\n backend,\n prisma: backend === 'prisma' ? prismaClient : undefined,\n});" + } +] }) }} + +{{ comp callout { type: "important", title: "Two durability concepts — don't conflate them" } }} +.durability('t1') on the saga definition selects a +durability tier (SAGA_DURABILITY_TIERS) — a property of the saga +itself. The store backend (kv | prisma) is a +runtime/deployment choice about which database holds durable state, +resolved by NETSCRIPT_SAGA_STORE / sagas.store.backend. A +saga keeps its declared tier regardless of which backend you deploy against. +{{ /comp }} + +{{ comp callout { type: "note", title: "Read model vs. durable write path" } }} +The Prisma backend's saga_runtime_* tables are the durable +write path the engine checkpoints to. They are distinct from +saga_instances (and saga_execution_history) — a +read-model projection into Postgres for API queries, analytics, and +debugging. The runtime persists to the former; the API surface lists from the latter. +{{ /comp }} + +## The HTTP publisher — createSagaPublisher + +A message reaches a saga through a **publisher**. `createSagaPublisher` (from +`@netscript/plugin-sagas/runtime`) returns a `SagaPublisherPort` whose `publish(...)` POSTs +to the sagas API publish endpoint, discovering the service URL from the Aspire environment +by default. The workers `create-user-settings` job uses exactly this to emit +`UserSettingsCreated` across the plugin boundary. + +{{ comp.apiTable({ + caption: "createSagaPublisher(options) — HttpSagaPublisherOptions", + rows: [ + { name: "serviceName", type: "string", desc: "Aspire service name to resolve a base URL from. Defaults to the sagas service discovery name." }, + { name: "baseUrl", type: "string", desc: "Explicit base URL override; skips discovery when set." }, + { name: "publishPath", type: "string (default '/api/v1/sagas/publish')", desc: "Path the publisher POSTs each message to." }, + { name: "headers", type: "Record", desc: "Extra headers sent with every publish (auth, tenant routing)." }, + { name: "retryableStatusCodes", type: "readonly number[] (default 408,409,425,429,5xx)", desc: "HTTP statuses treated as retryable so the receipt is marked retryable." }, + { name: "id", type: "string (default 'http-saga-publisher')", desc: "Stable publisher id surfaced in diagnostics." }, + { name: "fetcher / readEnv", type: "boundary fns", desc: "Test/injection seams for the fetch implementation and the env reader used for discovery." } + ] +}) }} + +The port returns a typed **receipt** rather than throwing: `publish(...)` resolves a +`SagaPublisherResult` — either `{ published: true, ... }` (a `SagaPublisherReceipt`) or +`{ published: false, reason, retryable }` (a `SagaPublisherRejected`). `publishMany(...)` +takes a `mode: 'sequential' | 'parallel'` so a batch can fan out. + +{{ comp.tabbedCode({ tabs: [ + { + label: "The job that publishes (workers)", + lang: "ts", + code: "// plugins/workers/jobs/create-user-settings.ts (core, verbatim from the scaffold)\nimport { createSagaPublisher } from '@netscript/plugin-sagas/runtime';\nimport { createSuccessResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { z } from 'zod';\n\nconst PayloadSchema = z.object({ userId: z.string().min(1) });\nconst sagaPublisher = createSagaPublisher();\n\nconst handler = defineJobHandler(async (ctx) => {\n const { userId } = PayloadSchema.parse(ctx.payload ?? {});\n // This is the message the saga below consumes — a typed receipt comes back.\n await sagaPublisher.publish({ type: 'UserSettingsCreated', payload: { userId } });\n return createSuccessResult({ userId, settingsCreated: true });\n});\n\nexport default Object.assign(handler, { id: 'create-user-settings' });" + }, + { + label: "The saga that consumes (sagas)", + lang: "ts", + code: "// plugins/sagas/user-settings-saga.ts\nimport { defineSaga, sagaComplete } from '@netscript/plugin-sagas-core';\n\ntype State = Readonly<{ status: string; processedAt?: string }>;\n\nexport const userSettingsSaga = defineSaga('user-settings-saga')\n .durability('t1')\n .state({ status: 'pending' })\n .on<'UserSettingsCreated', { userId: string }>(\n 'UserSettingsCreated',\n (saga, event, context) => {\n saga.state = { ...saga.state, status: 'completed', processedAt: context.now.toISOString() };\n return [sagaComplete({ userId: event.payload.userId, processedAt: context.now.toISOString() })];\n },\n )\n .build();" + } +] }) }} + +## Extension points + +The current scaffold uses the curated defaults, but the core package exposes the seams the +plugin is composed from. Each is a subpath of `@netscript/plugin-sagas-core` — cite the +sub-path in prose and look it up under {{ comp.xref({ key: "ref:sagas" }) }} (e.g. +`reference/sagas/presets`, `reference/sagas/transports`). + +{{ comp.apiTable({ + caption: "Saga extension seams (@netscript/plugin-sagas-core subpaths)", + rows: [ + { name: "/presets", type: "startSagas(), startSagaHandlers()", desc: "Composition helpers that build a runtime from explicit definitions and return a { runtime, bus, sagaCount, shutdown } bundle. startSagaHandlers is the distributed-handler alias." }, + { name: "/middleware", type: "createSagaMiddleware(), createSSEEventsMiddleware()", desc: "Hono middleware that injects saga helpers into request context, plus SSE event emission with optional durable history (SagaHistoryWriter)." }, + { name: "/transports", type: "createGarnetListTransport(), createNetScriptRedisTransport()", desc: "At-least-once delivery adapters: a Garnet/Redis LIST transport and a Redis Streams transport, both with immediate + delayed publish and ack/nack." }, + { name: "/agent", type: "agent surface", desc: "Agent integration seam for the plugin's agent-facing surface (alpha)." }, + { name: "/integration/publisher", type: "SagaPublisherPort", desc: "The publisher boundary createSagaPublisher implements — publish() / publishMany() with typed receipts." }, + { name: "/integration/workers", type: "workers bridge", desc: "The seam that lets a workers job emit and consume saga messages across the plugin boundary." } + ] +}) }} + +{{ comp callout { type: "tip", title: "Default transport vs. Redis/Garnet" } }} +The scaffolded sagas service runs over the in-process native runtime and Deno KV. The +/transports adapters (createGarnetListTransport, +createNetScriptRedisTransport) are for multi-process or distributed +delivery — when saga handlers run as separate processes and need at-least-once delivery with +consumer groups, delayed messages, and dead-letter handling. Reach for them only when a single +process is no longer enough; the KV path covers local and single-service apps. +{{ /comp }} + +## Endpoints and ports + +The sagas plugin runs an oRPC API service on `:8092`. It lists registered sagas and +inspects running instances; the registry is backed by Deno KV. These are the routes +the live scaffold serves — see {{ comp.xref({ key: "ref:sagas" }) }} for the full +generated surface. + +{{ comp.apiTable({ + caption: "Sagas plugin — runtime endpoints (port :8092)", + rows: [ + { name: "GET /health/live", type: "liveness", desc: "Liveness probe for the sagas API service." }, + { name: "GET /api/v1/sagas/sagas", type: "registry", desc: "List the saga definitions registered into KV (id, name, topic, handled message types, enabled)." }, + { name: "GET /api/v1/sagas/instances", type: "instances", desc: "List running and completed saga instances. Inspect one with /instances/{sagaName}/{correlationId}." }, + { name: "POST /api/v1/sagas/publish", type: "publish", desc: "Publish a message to the saga bus — the same path createSagaPublisher POSTs to and the workers create-user-settings job uses." }, + { name: "GET /api/v1/sagas/subscribe", type: "stream (SSE)", desc: "Server-sent-events stream of saga activity (saga:started / state_changed / completed / failed / compensating), KV-watch backed." } + ] +}) }} + +## The continuous-app choreography + +The thread that ties the capabilities together is real and it compiles. The workers +plugin's `create-user-settings` job calls a saga publisher and emits +`UserSettingsCreated`; this saga's `.on('UserSettingsCreated', ...)` handler consumes +it and emits `sagaComplete(...)`. One message crosses the plugin boundary, and both +halves are type-checked against the same message type. + +After both plugins are running under Aspire, trigger the workers job +(`POST :8091/api/v1/workers/jobs/create-user-settings/trigger`) and watch the saga +appear at `GET :8092/api/v1/sagas/instances` — the message crossed the boundary and a +durable instance recorded its completion. Whether that durable instance lives in Deno +KV or in your `saga_runtime_*` Postgres tables is exactly the `NETSCRIPT_SAGA_STORE` +choice above. + +## Production notes + +{{ comp callout { type: "warning", title: "Aspire first, then anything stateful" } }} +The sagas service needs Postgres and Redis up before it can persist and list +instances. Bring orchestration up first — cd aspire && aspire start +(dashboard at https://localhost:18888) — +before any netscript db command or before you expect +/api/v1/sagas/instances to return durable state. This holds for both +backends: kv needs Redis/KV up, and prisma needs Postgres up +with the saga_runtime_* tables migrated. DB commands require Aspire running +first. +{{ /comp }} + +{{ comp callout { type: "important", title: "Handlers are synchronous and pure" } }} +A SagaHandler returns readonly CascadedMessage[] — it is +synchronous and does no direct I/O. All side effects (charging payment, +sending mail, spawning a child) are expressed as effects the runtime applies, so +the handler is replayable and crash-safe. Do not await a network call inside a +handler; emit a send(...) effect and let the target do the work. Compensation +follows the same rule — its undo runs in the .compensate() handler as more +effects, not as inline cleanup. +{{ /comp }} + +## Why a saga, and why not + +{{ comp callout { type: "tip", title: "Reach for a saga when…" } }} +You have a multi-step process where steps can fail independently and the +state between steps must survive a crash — onboarding, checkout, +provisioning. The correlation, persistence, and named completion/failure/compensation +outcomes are exactly what a hand-rolled retry loop lacks. Pick prisma as the +store backend when you also want that in-flight state queryable in your relational database. +{{ /comp }} + +{{ comp callout { type: "warning", title: "Don't reach for a saga when…" } }} +The work is a single idempotent unit with no inter-step state — that is a +background job, not a saga. And remember +the alpha reality: durability and the instance store depend on the orchestration stack +being up via Aspire (Redis/KV for kv, Postgres for prisma), +so a saga is not a substitute for a database transaction within one handler. +{{ /comp }} + +## Reference → + +{{ comp.featureGrid({ + columns: 2, + items: [ + { icon: "≡", title: "Look up — sagas reference", body: "The full generated @netscript/plugin-sagas API: the defineSaga builder, durability tiers, createDurableSagaRuntime, the kv/prisma store backends, effect helpers, presets/middleware/transports seams, and every :8092 route.", href: "/reference/sagas/" }, + { icon: "✲", title: "Understand — the durability model", body: "Why a saga's state must outlive the process, how effect-based outcomes differ from a retry loop, and where the kv vs prisma backends fit.", href: "/explanation/durability-model/" }, + { icon: "◆", title: "Learn — Storefront checkout saga", body: "Build the order/checkout saga with a compensation branch as part of the storefront tutorial track.", href: "/tutorials/storefront/04-checkout-saga/" }, + { icon: "▣", title: "Look up — queue reference", body: "createParallelQueue, createQueue, providers, and connection options for the fan-out concurrency primitive.", href: "/reference/queue/" } + ] +}) }} + +{{ comp.nextPrev({ + prev: { label: "Background jobs", href: "/background-processing/workers/" }, + next: { label: "Triggers & ingress", href: "/durable-workflows/triggers/" } +}) }} diff --git a/docs/site/durable-workflows/streams.md b/docs/site/durable-workflows/streams.md new file mode 100644 index 000000000..1a0345107 --- /dev/null +++ b/docs/site/durable-workflows/streams.md @@ -0,0 +1,410 @@ +--- +layout: layouts/base.vto +title: Durable streams +templateEngine: [vento, md] +prev: { label: "Triggers & ingress", href: "/durable-workflows/triggers/" } +next: { label: "Database & Prisma", href: "/data-persistence/database/" } +--- + +# Durable streams + +A NetScript **stream** is a typed, durable change-log: producers write entity +state into a durable-stream server, and any number of HTTP/SSE consumers +materialize the latest value per key. {{ comp.badge({ status: "alpha" }) }} + +{{ comp.diagram({ + src: "/assets/diagrams/streams-pipeline.svg", + alt: "A producer defines a typed stream schema and writes upsert/delete operations into the durable-stream server on port 4437; the durable log fans out over HTTP/SSE to Fresh consumers that materialize the latest value per key.", + caption: "Streams pipeline: producer (defineStreamSchema + createDurableStream) → durable log on :4437 → HTTP/SSE → Fresh consumers (latest value per key)." +}) }} + +NetScript's streams capability is the typed, change-data backbone the other +plugins lean on — workers, sagas, and the auth service all publish their live +state through it. The producer half is implemented: you define a +typed stream schema with [`defineStreamSchema`](/reference/streams/), open a +producer with `createDurableStream` (or the Service-facing +`createServiceStreamProducer`), and `upsert`/`delete`/`flush` entity state +over a durable-stream server that runs as an Aspire resource on port **:4437**. + +The producer runtime in `@netscript/plugin-streams-core` writes through +`@durable-streams/client` with idempotent delivery. The topic-centric +**manifest sugar** in `@netscript/plugin-streams` +(`defineStreamProducer` / `defineStreamConsumer`) is **not** wired to a +transport: a producer's `publish()` returns a **rejected** promise and a +consumer's `subscribe()` **throws** synchronously — both with +`StreamUnsupportedOperationError`, pointing you at the core package. There is +also no in-process consumer `subscribe()` yet — consumption is over the +durable-stream server's HTTP/SSE protocol, which Fresh clients read. Build +against the core producer package, not the manifest helpers. + +{{ comp callout { type: "important", title: "Status — producers write via the core package; manifest helpers fail loud" } }} +The producer is implemented: createDurableStream(...) from +@netscript/plugin-streams-core writes upsert/delete/flush +through @durable-streams/client to the :4437 Aspire service, and workers, +sagas, and auth already mirror their state through it. What is not supported: the +manifest helpers defineStreamProducer/defineStreamConsumer in +@netscript/plugin-streams fail loud — a producer's publish() returns a +rejected promise and a consumer's subscribe() throws +synchronously, both with StreamUnsupportedOperationError — use +@netscript/plugin-streams-core instead. There is also no +in-process consumer subscribe(); consumption is via the durable-stream HTTP/SSE server +(read by Fresh clients). +{{ /comp }} + +## What it is + +A NetScript stream is an **entity-oriented change log**. You describe a set of +collections — each a named entity type with a primary key — and the producer +publishes `upsert` and `delete` operations keyed by that primary key. Downstream +readers materialize the latest value per key and observe a live, replayable view +of your domain state. This is the same contracts-first instinct as oRPC services, +but applied to **state replication** instead of request/response: the schema is +the type contract that both producer and any HTTP/SSE consumer are locked to. + +Streams sit alongside the other long-running capabilities rather than replacing +them. Reach for a {{ comp.xref({ key: "cap:durable-sagas", text: "durable saga" }) }} +when you need message-driven orchestration with compensation; reach for a +{{ comp.xref({ key: "cap:triggers", text: "trigger" }) }} when inbound HTTP or a +file-watch should kick off work. A stream is the **read-model fan-out**: each of +those plugins runs a thin `streams/producer.ts` that mirrors its execution state +through `createDurableStream`, so a Fresh dashboard can watch saga, worker, and +trigger progress live without polling a request/response API. + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — stream a live dashboard", + body: "Track D 05: publish execution state from a producer and read it over HTTP/SSE into a live Fresh dashboard.", + href: "/tutorials/live-dashboard/05-live-stream/", + icon: "→" + }, + { + title: "Do — add the streams plugin", + body: "Public package install adds the stream plugin dependency and user-owned glue; local netscript-dev scaffolding supports contributor-source samples.", + href: "/how-to/add-a-plugin/", + icon: "◆" + }, + { + title: "Understand — the plugin system", + body: "How thread-isolated plugins like streams register contributions and wire into the Aspire resource graph.", + href: "/explanation/plugin-system/", + icon: "◎" + } +] }) }} + +## Minimal example — produce, then consume + +The producer side is two calls: freeze a typed schema, open a stream, write +entity state. The consumer side is an HTTP/SSE read of the same `:4437` stream +path — there is no in-process `subscribe()` handle, so a Fresh island (or any +SSE client) reads the durable log directly and materializes the latest value per +key. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Define a schema (real)", + lang: "ts", + code: "// streams/executions-schema.ts\n// Author against the core package — this is the live surface.\nimport { defineStreamSchema } from '@netscript/plugin-streams-core';\nimport { z } from 'zod';\n\n// Each collection is an entity type with a primary key. The schema is the\n// type contract producers and HTTP/SSE consumers are locked to.\nexport const executionsSchema = defineStreamSchema({\n execution: {\n schema: z.object({\n id: z.string().min(1),\n status: z.enum(['queued', 'running', 'succeeded', 'failed']),\n updatedAt: z.string().datetime(),\n }),\n type: 'execution',\n primaryKey: 'id',\n },\n});" + }, + { + label: "Open a producer & write (real)", + lang: "ts", + code: "// streams/producer.ts\nimport { createDurableStream } from '@netscript/plugin-streams-core';\nimport { executionsSchema } from './executions-schema.ts';\n\n// createDurableStream returns a singleton producer per streamPath and begins\n// connecting to the :4437 durable-stream server immediately.\nconst producer = createDurableStream({\n streamPath: '/workers/executions',\n schema: executionsSchema,\n producerId: 'workers-service',\n});\n\n// upsert/delete are synchronous enqueues keyed by the collection primary key.\nproducer.upsert('execution', {\n id: 'exec-1',\n status: 'running',\n updatedAt: new Date().toISOString(),\n});\nproducer.delete('execution', 'exec-0');\n\n// flush before graceful shutdown; it rethrows the connect error if the\n// producer never connected (see known limitations).\nawait producer.flush();" + }, + { + label: "Consume over HTTP/SSE (Fresh client)", + lang: "ts", + code: "// islands/ExecutionsView.tsx — read the durable log directly\nimport { getStreamsUrl } from '@netscript/plugin-streams-core';\n\n// There is no in-process subscribe(); consumption is an HTTP/SSE read of the\n// same stream path the producer writes to. getStreamsUrl resolves the :4437\n// base from Aspire discovery / VITE env (see runtime resolvers below).\nconst base = getStreamsUrl();\nconst source = new EventSource(`${base}/workers/executions`);\n\nconst latest = new Map(); // materialize latest value per key\nsource.onmessage = (ev) => {\n const change = JSON.parse(ev.data) as { key: string; value?: unknown };\n if (change.value === undefined) latest.delete(change.key);\n else latest.set(change.key, change.value);\n};" + }, + { + label: "Manifest helpers (fail loud)", + lang: "ts", + code: "// @netscript/plugin-streams (the manifest root) re-exports topic-centric\n// helpers that are NOT implemented. They fail loud, by design.\nimport {\n defineStreamProducer,\n defineStreamConsumer,\n} from '@netscript/plugin-streams';\n\n// The producer handle is returned, but publish() rejects.\nconst producer = defineStreamProducer(/* topic */);\n// await producer.publish(event) // -> rejects with StreamUnsupportedOperationError\n\n// The consumer's subscribe() throws synchronously the moment you call it.\nconst consumer = defineStreamConsumer(/* topic */);\n// consumer.subscribe(handler) // -> throws StreamUnsupportedOperationError\n\n// Correct path: import createDurableStream / defineStreamSchema from\n// '@netscript/plugin-streams-core' instead (see the other tabs)." + } +] }) }} + +{{ comp callout { type: "warning", title: "Do not call the manifest topic helpers" } }} +defineStreamProducer and defineStreamConsumer exported from +@netscript/plugin-streams are not implemented: a producer's +publish() returns a rejected promise and a consumer's +subscribe() throws synchronously, both with +StreamUnsupportedOperationError — they are not silent no-ops and they will not publish +anything. Always reach +for createDurableStream / defineStreamSchema from +@netscript/plugin-streams-core for real producer work. See the +streams reference for the full export map. +{{ /comp }} + +## Key types first — the stream definition API + +A stream schema is a map of **collections**. Each collection is a +`CollectionDefinition` — a Standard-Schema validator, a State-Protocol `type` +discriminator, and the `primaryKey` property the producer keys writes by. +`defineStreamSchema(collections)` freezes that map into a `StateSchema` that both +the producer and any HTTP/SSE consumer are locked to. + +{{ comp.apiTable({ + caption: "CollectionDefinition — one entry per collection in defineStreamSchema", + rows: [ + { name: "schema", type: "unknown (Standard Schema validator)", desc: "Standard-Schema-compatible validator (e.g. a zod object) used by durable-streams to validate the collection payload." }, + { name: "type", type: "string (required)", desc: "State Protocol type discriminator emitted for every event in this collection (e.g. 'execution')." }, + { name: "primaryKey", type: "string (required)", desc: "Property name on the value used as the entity primary key; upsert/delete are keyed by this property." } + ] +}) }} + +`defineStreamSchema` returns a frozen `StateSchema` — the durable-streams +runtime attaches per-collection event helpers (`insert`/`update`/`upsert`/`delete`, +the `CollectionEventHelpers`) so the schema can both validate and emit +State-Protocol `ChangeEvent`s. The supported `Operation` set is +`insert | update | delete | upsert`. + +## Producer options — `createDurableStream` + +`createDurableStream(options)` takes a `DurableStreamProducerOptions` and returns +a `DurableStreamProducer`. It is a **singleton factory keyed by `streamPath`**: +calling it twice with the same path returns the same live producer (a closed one +is replaced). Writes are idempotent via `@durable-streams/client`'s +`IdempotentProducer` (stable `producerId` + auto-claim), so duplicate enqueues do +not double-apply downstream. + +{{ comp.apiTable({ + caption: "DurableStreamProducerOptions (createDurableStream argument)", + rows: [ + { name: "streamPath", type: "string (required)", desc: "Stream path relative to the base URL, e.g. '/workers/executions'. This is the singleton key and the path consumers read over HTTP/SSE." }, + { name: "schema", type: "StateSchema (required)", desc: "The frozen schema returned by defineStreamSchema; binds the producer to its collection map." }, + { name: "producerId", type: "string (required)", desc: "Stable producer identity used for idempotent delivery (IdempotentProducer auto-claim). Keep it stable across restarts for duplicate-safe writes." }, + { name: "signal", type: "AbortSignal?", desc: "Optional abort signal consulted while opening the stream connection; aborting cancels the connect (an AbortError is treated as expected, not a failure)." } + ] +}) }} + +The `DurableStreamProducer` it returns exposes a small, synchronous-write surface +with an async flush/close for shutdown. (`StreamProducerPort` is the +implemented-by interface — the same four members, with `entityType` widened to +`string`.) + +{{ comp.apiTable({ + title: "DurableStreamProducer — methods", + columns: ["Member", "Shape", "Behavior"], + rows: [ + ["upsert(entityType, value)", "(K, Record) => void", "Enqueue an upsert keyed by the collection primaryKey; skipped (warns) if the key is missing/empty or the collection is unknown."], + ["delete(entityType, key)", "(K, string) => void", "Enqueue a delete by primary key; skipped (warns) on an empty key."], + ["flush()", "() => Promise<void>", "Await pending writes before shutdown; rethrows the connect error if the producer never connected."], + ["close()", "() => Promise<void>", "Flush, close the underlying handle, and release the singleton for this streamPath."], + ["streamPath", "string (readonly)", "The stream path this producer owns."], + ["closed", "boolean (get)", "Whether shutdown has begun; further upsert/delete calls are ignored."] + ] +}) }} + +## Service-side producers — `createServiceStreamProducer` + +When the writer is a backend **Service** — for example an ingestion worker that +emits a `doc.ready` completion event from the callback where the work finishes — +reach for `createServiceStreamProducer` instead of wiring `createDurableStream` +and the URL/auth resolvers by hand. It is a thin wrapper over `createDurableStream` +that reuses the exact same singleton producer and Aspire discovery +(`getStreamsUrl` / `getStreamsAuth`), and it adds one guard: `assertResolvable` +(default `true`) eagerly resolves the streams URL and auth at construction, so a +Service that forgot to declare the `streams` reference **throws immediately** +rather than silently dropping every write. + +```ts +// services/ingestion/emit-completion.ts +import { + createServiceStreamProducer, + defineStreamSchema, +} from "@netscript/plugin-streams-core"; +import { z } from "zod"; + +const completions = defineStreamSchema({ + completion: { + schema: z.object({ id: z.string().min(1), status: z.string() }), + type: "completion", + primaryKey: "id", + }, +}); + +// assertResolvable defaults to true: this throws now if neither +// DURABLE_STREAMS_URL nor services__streams__http__0 is wired. +const producer = createServiceStreamProducer({ + streamPath: "/eischat/completions", + schema: completions, + producerId: "eischat-ingestion", +}); + +producer.upsert("completion", { id: "run-1", status: "done" }); +await producer.flush(); +``` + +A Service that declares `ServiceReferences: ["streams"]` always has the streams +endpoint wired, so the guard resolves without throwing. Set `assertResolvable: +false` to keep the tolerant, lazy-connect behavior of `createDurableStream` when +a Service may legitimately start before the streams service is reachable. +`ServiceStreamProducerOptions` is `DurableStreamProducerOptions` plus this +optional `assertResolvable` flag; the returned `DurableStreamProducer` is +identical, so the write/flush/close surface below applies unchanged. + +## Runtime & transport — URL and auth resolution + +The producer never hardcodes a host. `createDurableStream` resolves the +durable-stream base URL through `getStreamsUrl()` and the auth header through +`getStreamsAuth()`, both of which read the environment so the same code works +under Aspire, in a browser build, or against an explicit override. `buildStreamUrl` +joins a stream path onto that base, and `inspectStreamTopic` returns a JSON-stable +diagnostic report for a schema (handy in tests and CLI doctors). + +{{ comp.apiTable({ + caption: "Runtime resolvers (@netscript/plugin-streams-core)", + rows: [ + { name: "getStreamsUrl()", type: "() => string", desc: "Resolves the durable-stream base URL. Server: DURABLE_STREAMS_URL override, else Aspire's services__streams__http__0 discovery var. Browser: VITE_services__streams__http__0 (or the VITE_STREAMS_URL shorthand). Throws a descriptive error if none resolve." }, + { name: "getStreamsAuth()", type: "() => Record", desc: "Builds the auth header from STREAMS_SECRET (or DURABLE_STREAMS_SECRET) as { Authorization: 'Bearer ' }; returns {} when no secret is set." }, + { name: "buildStreamUrl(path, baseUrl?)", type: "(string, string?) => string", desc: "Joins a stream path onto the resolved base (or an explicit baseUrl), trimming a trailing slash on the base." }, + { name: "inspectStreamTopic(input)", type: "(input) => StreamTopicInspectionReport", desc: "Diagnostic: returns a JSON-stable report (package, target, summary, details with collections/streamPath/producerId) for a schema + optional producer metadata." } + ] +}) }} + +{{ comp.apiTable({ + caption: "Environment variables read by the resolvers", + rows: [ + { name: "DURABLE_STREAMS_URL", type: "server override", desc: "Explicit base URL; takes precedence over Aspire discovery (e.g. http://localhost:4437)." }, + { name: "services__streams__http__0", type: "Aspire (server)", desc: "Injected by the Aspire resource graph; the default server-side discovery path." }, + { name: "VITE_services__streams__http__0", type: "browser", desc: "Vite-injected reference for browser/Fresh consumers; VITE_STREAMS_URL is the convenience shorthand." }, + { name: "STREAMS_SECRET / DURABLE_STREAMS_SECRET", type: "auth", desc: "Bearer secret for getStreamsAuth(); when set, every connect sends Authorization: Bearer ." } + ] +}) }} + +## Known limitations + +Be deliberate about what the alpha producer does and does not guarantee. + + +{{ comp callout { type: "warning", title: "Writes are dropped after a connect failure (no reconnect)" } }} +If the producer cannot reach the :4437 durable-stream server at startup, it logs a +console.warn and then silently skips every subsequent upsert/delete +— there is no reconnect loop in the current alpha. flush() rethrows that connect error so a +graceful shutdown surfaces the failure. (Writes issued before the connection completes are +buffered and drained once it opens; the drop only applies after a connect error.) Treat a healthy +:4437 service as a hard precondition for durable delivery; do not assume buffered writes will +be replayed once the server returns. +{{ /comp }} + +{{ comp callout { type: "note", title: "No in-process consumer — read over HTTP/SSE" } }} +There is no in-process subscribe() handle. Consumption happens over the durable-stream +server's HTTP/SSE protocol, which Fresh clients read to materialize the latest value per key. Model your +read side as an HTTP/SSE consumer of the :4437 stream, not as an in-process callback. +{{ /comp }} + +{{ comp callout { type: "warning", title: "Local HTTP can limit concurrent stream consumers" } }} +The local AppHost serves the generated app over http://. Under HTTP/1.1, browsers +typically allow about six concurrent connections per origin, so several long-lived HTTP/SSE streams +from one Fresh page can starve later requests or stream subscriptions. Use HTTPS for HTTP/2 when a +local dashboard needs many simultaneous stream consumers. +{{ /comp }} + +{{ comp callout { type: "note", title: "Operator visibility is console.warn (alpha)" } }} +The producer reports skips and connect failures through console.warn today — tracked as +AP-13 architecture debt until the telemetry-integration wave supplies a structured +reporter. Until then, scrape the [DurableStreamProducer] warn lines if you need to alert on +dropped writes. +{{ /comp }} + +## Endpoints & manifest + +The streams plugin is registered as a utility/infra plugin — note it requires +**neither a database nor KV** (`requiresDb=false`, `requiresKv=false`), unlike +workers, sagas, and triggers. Its durable-stream service listens on `:4437` and +is wired into the Aspire resource graph so workers, sagas, and auth can publish +through it. The port is overridable via `STREAMS_PORT` or `PORT`. + +{{ comp.apiTable({ + title: "Streams plugin — runtime facts", + columns: ["Property", "Value"], + rows: [ + ["Plugin location", "plugins/streams/"], + ["Producer package", "@netscript/plugin-streams-core (createDurableStream, createServiceStreamProducer, defineStreamSchema)"], + ["Manifest import", "@netscript/plugin-streams — topic helpers throw StreamUnsupportedOperationError"], + ["Transport client", "@durable-streams/client (IdempotentProducer)"], + ["Dev service port", ":4437 (durable-stream Aspire service; override with STREAMS_PORT/PORT)"], + ["provider.kind", "stream · category plugin · pluginType utility"], + ["Requires DB / KV", "false / false"], + ["First-party producers", "workers, sagas, triggers, auth (each streams/producer.ts → createDurableStream)"], + ["Consumer surface", "HTTP/SSE from the :4437 server (Fresh clients) — no in-process subscribe()"] + ] +}) }} + +The plugin is referenced from `netscript.config.ts` as +`./plugins/streams/mod.ts`. Because workers, sagas, and triggers each list +`streams` in their `dependencies`, it is installed first in the dependency graph +and its `:4437` service comes up so dependent producers have somewhere to write. + +## Production notes + +{{ comp callout { type: "important", title: "Footguns before you ship" } }} +
    +
  • Build on the core package, not the manifest helpers. +createDurableStream / defineStreamSchema from +@netscript/plugin-streams-core are real; the +defineStreamProducer/defineStreamConsumer helpers in +@netscript/plugin-streams fail loud with +StreamUnsupportedOperationError.
    • +
  • A healthy :4437 server is a hard precondition. A startup connect +failure drops every later write with no reconnect — bring Aspire (or an explicit +DURABLE_STREAMS_URL) up first, and treat a flush() rejection on shutdown as a +real delivery failure, not noise.
    • +
  • Keep producerId stable across restarts. Idempotent delivery is keyed +by it; a churning id defeats the duplicate-safety guarantee.
    • +
  • Model the read side as HTTP/SSE. There is no in-process subscribe(); +resolve the base with getStreamsUrl() and consume the :4437 stream path +directly.
    • +
  • Set the auth secret on both ends. When the server expects a bearer token, export +STREAMS_SECRET (or DURABLE_STREAMS_SECRET) wherever the producer runs so +getStreamsAuth() can attach it.
    • +
+{{ /comp }} + +## Reference + +This hub is intentionally thin — the full generated API lives in the reference. + +{{ comp.xref({ key: "ref:streams" }) }} + +{{ comp callout { type: "tip", title: "Where the real surface comes from" } }} +The producer you build on lives in @netscript/plugin-streams-core: +createDurableStream, the Service-facing createServiceStreamProducer, +DurableStreamProducer, defineStreamSchema, +the buildStreamUrl/getStreamsUrl/getStreamsAuth resolvers, +inspectStreamTopic, and the StreamProducerPort / +DurableStreamProducerOptions types. The @netscript/plugin-streams manifest +re-exports streamsPlugin plus the fail-loud topic helpers. See the +full reference for every exported symbol and the +cli / scaffolding / e2e / aspire sub-paths. +{{ /comp }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — @netscript/plugin-streams-core", + body: "The generated API for createDurableStream, defineStreamSchema, the URL/auth resolvers, inspectStreamTopic, and the StreamProducerPort type.", + href: "/reference/streams/", + icon: "≡" + }, + { + title: "Learn — stream a live dashboard", + body: "Track D 05: publish execution state and read it over HTTP/SSE into a live Fresh dashboard.", + href: "/tutorials/live-dashboard/05-live-stream/", + icon: "→" + }, + { + title: "Understand — the plugin system", + body: "How thread-isolated plugins like streams register contributions and wire into the Aspire resource graph.", + href: "/explanation/plugin-system/", + icon: "◎" + }, + { + title: "Related — durable sagas", + body: "Message-driven orchestration with compensation; sagas mirror their state through a streams producer.", + href: "/durable-workflows/sagas/", + icon: "◆" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Triggers & ingress", href: "/durable-workflows/triggers/" }, next: { label: "Database & Prisma", href: "/data-persistence/database/" } }) }} diff --git a/docs/site/durable-workflows/triggers.md b/docs/site/durable-workflows/triggers.md new file mode 100644 index 000000000..02181b9fc --- /dev/null +++ b/docs/site/durable-workflows/triggers.md @@ -0,0 +1,485 @@ +--- +layout: layouts/base.vto +title: Triggers & ingress +templateEngine: [vento, md] +prev: { label: "Durable sagas", href: "/durable-workflows/sagas/" } +next: { label: "Durable streams", href: "/durable-workflows/streams/" } +--- + +# Triggers & ingress + +The **triggers** plugin is NetScript's front door for the outside world. It turns +external events — inbound webhooks, files dropped on disk, and cron schedules — +into durable background work. A trigger handler does one job: receive an event and +return a list of *actions*. The canonical action is `enqueueJob(...)`, which hands +the payload to the [workers](/background-processing/workers/) plugin so the heavy +lifting happens off the request path. The HTTP call returns immediately; the job +runs durably behind it. + +{{ comp.diagram({ + src: "/assets/diagrams/trigger-ingress-flow.svg", + alt: "Trigger ingress flow: a webhook POST is verified by an HMAC check, persisted as an event, and dispatched to the handler which returns an enqueueJob action; a file-watch event is debounced and stabilised before reaching the same handler path.", + caption: "Two ingress edges, one processor: webhook POST → HMAC verify → event store → handler → enqueueJob; file-watch → debounce + stability → handler. Both converge on the trigger processor, which retries and DLQs." +}) }} + +This closes the continuous-app loop. A webhook lands on the triggers service, it +`enqueueJob`s a workers job, that job publishes a saga message, and the +[saga](/durable-workflows/sagas/) advances a workflow — all from one inbound +POST. + +{{ comp callout { type: "important", title: "Typed oRPC contract, raw webhook ingress" } }} +Like services, workers, +and sagas, the triggers API service serves a +typed v1 oRPC contract for trigger and event introspection plus management — +describing triggers, listing and fetching triggers and events, firing a trigger, testing a webhook, +previewing a schedule, enabling/disabling a trigger, and subscribing to an SSE event stream. The one +deliberate exception is the webhook ingress endpoint itself +(POST /api/v1/webhooks/:triggerId, plus a legacy GET /api/v1/events): it stays +a raw, signature-verifying route, not an oRPC procedure, because it must verify an HMAC +signature over the raw request bytes — incompatible with oRPC's Zod body parsing. That keeps it open to +any third-party webhook sender, which has no NetScript typed client and just POSTs to the plain URL. +{{ /comp }} + +## What it is + +A trigger is a **declarative event source** plus a handler. NetScript ships three +authorable trigger builders from `@netscript/plugin-triggers-core/builders` — +`defineWebhook` (inbound HTTP), `defineFileWatch` (filesystem events), and +`defineScheduledTrigger` (cron). All three return a frozen *definition* the runtime +walker discovers; all three share one handler contract — `(event, context) => +Promise` — and one runtime processor that verifies, persists, +retries, and dead-letters. Three more kinds (`queue`, `stream`, `manual`) are reserved +in the type surface but not yet executable. The processing engine itself lives in +[`@netscript/watchers`](/reference/watchers/) (file watching) and the trigger +processor/ingress runtime (`@netscript/plugin-triggers-core/runtime`). + +## Learn / Do + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Ingest a webhook", + body: "Storefront Track A, rung 05: scaffold the triggers plugin, author a webhook, POST to :8093, and watch it enqueue a job — closing the continuous-app ladder.", + href: "/tutorials/storefront/05-shipping-webhook/", + icon: "→" + }, + { + title: "Do — Add a plugin", + body: "The how-to recipe for public plugin package install and the local netscript-dev contributor path for source-backed trigger samples.", + href: "/how-to/add-a-plugin/", + icon: "◆" + } +] }) }} + +## Minimal example — a verified payment webhook + +The 80% case: a webhook that verifies an HMAC signature, then fans the inbound +payment event out to a single durable job. The handler returns immediately; the +real work runs on the [workers](/background-processing/workers/) plugin. + +```ts +// plugins/triggers/payment-status-webhook.ts +import { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders'; +import type { JobDefinition } from '@netscript/plugin-workers-core'; + +// The workers job this webhook will enqueue (authored in plugins/workers/jobs/). +const reconcilePaymentJob = { + id: 'reconcile-payment' as JobDefinition<'reconcile-payment'>['id'], + name: 'Reconcile Payment', + topic: 'default', +} satisfies JobDefinition<'reconcile-payment'>; + +// Inbound POST -> enqueue a workers job. Ingress verifies the HMAC signature +// (verifier: 'hmac-sha256' + secretEnv) BEFORE this handler ever runs, so by the +// time you see the event the sender is already proven. +export const paymentStatusWebhook = defineWebhook( + (event) => Promise.resolve([ + enqueueJob(reconcilePaymentJob, { + payload: { raw: event.payload.body }, + idempotencyKey: event.idempotencyKey, // de-dupe sender retries + priority: 80, + }), + ]), + { + id: 'payment-status-webhook', + path: 'payments/status', + verifier: 'hmac-sha256', + secretEnv: 'PAYMENT_WEBHOOK_SECRET', + description: 'Verified payment-status webhook that enqueues reconciliation.', + tags: ['webhook', 'payments', 'verified'], + }, +); + +export default paymentStatusWebhook; +``` + +## Key types first — the webhook definition + +`defineWebhook(handler, spec)` takes your handler and a static `WebhookSpec`. These +are the **actual** fields the builder accepts (confirmed via `deno doc`), shown +before any prose so you can read the contract at a glance. + +{{ comp.apiTable({ + caption: "WebhookSpec — the second argument to defineWebhook(handler, spec)", + rows: [ + { name: "id", type: "string (required)", desc: "Stable identifier for the webhook definition, e.g. 'payment-status-webhook'. Branded internally as a WebhookId." }, + { name: "path", type: "string (required)", desc: "Ingress sub-path the webhook answers on, e.g. 'payments/status'. Resolved as the :triggerId route segment under /api/v1/webhooks/." }, + { name: "verifier", type: "'hmac-sha256' | 'memory' | string (required)", desc: "Verification strategy selector (WebhookVerifierKind). 'hmac-sha256' performs a real signature check; 'memory' is open (dev only); a custom string selects a verifier you wire via selectVerifier." }, + { name: "secretEnv", type: "string?", desc: "Name of the env var holding the signing secret. The runtime reads it (resolveSecret) and hands it to the verifier — the secret is never inlined in the definition." }, + { name: "description", type: "string?", desc: "Human-readable summary surfaced by inspect/registry tooling." }, + { name: "tags", type: "readonly string[]?", desc: "Free-form labels for discovery/filtering, e.g. ['webhook','payments','verified']." }, + { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata attached to the definition." } + ] +}) }} + +The handler receives a `TriggerEvent<'webhook', WebhookTriggerPayload>`. The payload +carries the captured HTTP request: `body` (parsed/raw), `headers`, `method` (always +`"POST"`), `path`, and an optional `remoteAddr`. The event envelope also surfaces +`idempotencyKey`, `traceparent`/`tracestate`, and `requestHeaders` — so de-duplicating +sender retries is a one-liner, as in the example above. + +## HMAC verification — proving the sender + +`verifier: 'hmac-sha256'` activates the `HmacSha256WebhookVerifier` adapter from +`@netscript/plugin-triggers-core/adapters`. Ingress hashes the raw request body with +the secret (resolved from `secretEnv`) and compares it against the signature header +*before* persisting the event or running your handler — a bad signature is rejected at +the edge. The `'memory'` verifier (`MemoryWebhookVerifier`) accepts everything and is +for local iteration only. + +{{ comp.apiTable({ + caption: "HmacSha256WebhookVerifier — constructor options (from @netscript/plugin-triggers-core/adapters)", + rows: [ + { name: "secret", type: "string?", desc: "Signing secret. Usually supplied by the runtime from the definition's secretEnv rather than hard-coded here." }, + { name: "signatureHeader", type: "string?", desc: "Header carrying the sender's HMAC signature to compare against (e.g. 'x-signature')." }, + { name: "idempotencyHeader", type: "string?", desc: "Header whose value is surfaced as the event idempotencyKey when present, so sender retries collapse." } + ] +}) }} + +{{ comp callout { type: "note", title: "How the verifier is wired" } }} +You rarely construct the adapter yourself. The webhook spec declares +verifier: 'hmac-sha256' and secretEnv, and the ingress runtime +(createTriggerIngress) resolves the secret and selects the verifier per +definition (its verifier, selectVerifier, and +resolveSecret options). The verifier's verify(request) returns +{ ok, idempotencyKey?, reason? } — ok: false stops the event at the edge. +{{ /comp }} + +## File-watch triggers — react to dropped files + +`defineFileWatch(handler, spec)` turns filesystem events into the same durable-work +pipeline. The watching primitive lives in [`@netscript/watchers`](/reference/watchers/); +the trigger builder wraps it so a dropped file becomes a handler invocation that +returns `enqueueJob` actions. Below: a CSV drop-folder that enqueues one import job per +stable file. + +```ts +// plugins/triggers/product-import.ts +import { defineFileWatch, enqueueJob } from '@netscript/plugin-triggers-core/builders'; +import type { JobDefinition } from '@netscript/plugin-workers-core'; + +const importProductsJob = { + id: 'import-products' as JobDefinition<'import-products'>['id'], + name: 'Import Products', + topic: 'default', +} satisfies JobDefinition<'import-products'>; + +// Fires once a *.csv lands in ./incoming/products and stops growing. The +// stabilityThreshold guards against half-written files on a network share: +// the watcher waits for 3 unchanged checks 1s apart before yielding the event. +export const productImportWatch = defineFileWatch( + (event) => Promise.resolve([ + enqueueJob(importProductsJob, { + payload: { file: event.payload.path }, + idempotencyKey: event.payload.path, // one import per file path + priority: 50, + }), + ]), + { + id: 'product-import-watch', + paths: ['./incoming/products'], + patterns: ['*.csv'], + on: ['create', 'modify'], + stabilityThreshold: { checkIntervalMs: 1000, stableChecks: 3 }, + description: 'Imports a product CSV the moment it lands and stabilises.', + tags: ['file-watch', 'erp', 'import'], + }, +); + +export default productImportWatch; +``` + +{{ comp.apiTable({ + caption: "FileWatchSpec — the second argument to defineFileWatch(handler, spec)", + rows: [ + { name: "id", type: "string (required)", desc: "Stable identifier for the file-watch definition." }, + { name: "paths", type: "readonly string[] (required)", desc: "Directories to watch (at least one). Relative paths resolve against the process cwd." }, + { name: "patterns", type: "readonly string[] (required)", desc: "Glob patterns selecting which filenames fire, e.g. ['*.csv','sales_*.xlsx']." }, + { name: "on", type: "readonly ('create'|'modify'|'remove')[] (required)", desc: "Which filesystem lifecycle events yield to the handler (FileWatchLifecycle)." }, + { name: "ignored", type: "readonly string[]?", desc: "Glob patterns to exclude from the watch." }, + { name: "debounceMs", type: "number?", desc: "Per-file debounce window collapsing rapid successive events for the same path." }, + { name: "stabilityThreshold", type: "{ checkIntervalMs, stableChecks }?", desc: "Network-FS tolerant gate: waits for stableChecks unchanged polls (checkIntervalMs apart) so half-written files don't fire early." }, + { name: "description", type: "string?", desc: "Human-readable summary for tooling." }, + { name: "tags", type: "readonly string[]?", desc: "Free-form discovery labels." }, + { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata." } + ] +}) }} + +The handler receives a `TriggerEvent<'file-watch', FileWatchTriggerPayload>`; the +payload carries `path`, `kind` (the lifecycle event), and — when the watcher could read +them — `size`, `modifiedAt`, and `stableChecks`. + +### The underlying watcher + +For lower-level or standalone file watching (outside the trigger pipeline), call +`createWatcher(options)` from `@netscript/watchers` directly. It returns a `FileWatcher` +whose `watch()` is an async generator of `WatchEvent`s. + +```ts +// scripts/watch-incoming.ts +import { createWatcher } from "@netscript/watchers"; + +const watcher = createWatcher({ + paths: ["./incoming"], + patterns: ["*.csv"], + events: ["create", "modify"], + stabilityThreshold: { checkIntervalMs: 1000, stableChecks: 3 }, +}); + +for await (const event of watcher.watch()) { + console.log(`${event.kind}: ${event.path}`); +} +``` + +{{ comp.apiTable({ + caption: "createWatcher(options) — WatcherOptions (from @netscript/watchers)", + rows: [ + { name: "paths", type: "readonly string[] (required)", desc: "Directories to watch (at least one required)." }, + { name: "patterns", type: "readonly string[]?", desc: "Glob patterns for filtering files. Default ['*']." }, + { name: "events", type: "readonly EventKind[]?", desc: "Which FS events to yield ('create'|'modify'|'remove'). Default ['create']." }, + { name: "debounceMs", type: "number?", desc: "Per-file debounce in milliseconds. Default 2000." }, + { name: "contentHash", type: "boolean?", desc: "Compute a SHA-256 content hash for de-duplication. Default true." }, + { name: "processExisting", type: "boolean?", desc: "Scan existing files on startup and emit them as create events. Default false." }, + { name: "forcePolling", type: "boolean?", desc: "Force the polling strategy instead of native FS notifications. Default false (use for network paths)." }, + { name: "pollIntervalMs", type: "number?", desc: "Polling interval for the polling strategy. Minimum 500. Default 5000." }, + { name: "minFileSize", type: "number?", desc: "Skip files smaller than this many bytes. Default 0." }, + { name: "maxFileAge", type: "number?", desc: "Skip files older than this (ms); only applies during the startup scan." }, + { name: "stabilityThreshold", type: "StabilityOptions?", desc: "When set, waits for files to stop growing before yielding (network-FS tolerant)." }, + { name: "signal", type: "AbortSignal?", desc: "Abort signal for graceful shutdown of the watch loop." } + ] +}) }} + +The watcher selects the strategy for you: native OS notifications for local paths, +polling for network paths or when `forcePolling: true`, and a `HybridStrategy` that +blends both. Events pass a filter pipeline before you see them — `GlobFilter` limits +filenames, `StabilityFilter` waits for files to stop growing, and `DedupFilter` skips +repeated content hashes within its window. The concrete `NativeStrategy`, +`PollingStrategy`, and `HybridStrategy` classes are internal; construct watchers with +`createWatcher` or `new FileWatcher(...)`. + +## Scheduled triggers — cron without a daemon + +`defineScheduledTrigger(handler, spec)` fires a handler on a cron schedule. The spec +combines discovery metadata (`id`, `description`, `tags`, `metadata`) with the cron +`ScheduledTriggerSpec`. The scheduler runs inside the trigger processor entrypoint, not +the Hono API service. + +```ts +// plugins/triggers/nightly-reconcile.ts +import { defineScheduledTrigger, enqueueJob } from '@netscript/plugin-triggers-core/builders'; +import type { JobDefinition } from '@netscript/plugin-workers-core'; + +const nightlyReconcileJob = { + id: 'nightly-reconcile' as JobDefinition<'nightly-reconcile'>['id'], + name: 'Nightly Reconcile', + topic: 'default', +} satisfies JobDefinition<'nightly-reconcile'>; + +// Runs at 02:00 UTC. backfill replays fires the scheduler missed while down. +export const nightlyReconcile = defineScheduledTrigger( + () => Promise.resolve([enqueueJob(nightlyReconcileJob, { payload: {} })]), + { + id: 'nightly-reconcile', + description: 'Nightly reconciliation sweep.', + cron: '0 2 * * *', + timezone: 'UTC', + persistent: true, + backfill: { enabled: true, windowMs: 86_400_000, policy: 'fire-once' }, + }, +); + +export default nightlyReconcile; +``` + +{{ comp.apiTable({ + caption: "Scheduled trigger spec — DefineScheduledTriggerSpec + ScheduledTriggerSpec", + rows: [ + { name: "id", type: "string (required)", desc: "Stable identifier for the scheduled trigger definition." }, + { name: "cron", type: "CronExpression (required)", desc: "Cron expression governing when the handler fires, e.g. '0 2 * * *'." }, + { name: "timezone", type: "string?", desc: "IANA timezone the cron expression is evaluated in. Defaults to the runtime's zone." }, + { name: "persistent", type: "boolean?", desc: "Whether the schedule state survives restarts so missed fires can be reasoned about." }, + { name: "backfill", type: "TriggerBackfillSpec?", desc: "Quartz-style misfire handling: { enabled, windowMs, policy: 'fire-now'|'fire-once'|'do-nothing', maxMissedFires? } — replays fires the scheduler missed while down." }, + { name: "description", type: "string?", desc: "Human-readable summary for tooling." }, + { name: "tags", type: "readonly string[]?", desc: "Free-form discovery labels." }, + { name: "metadata", type: "Readonly>?", desc: "Arbitrary structured metadata." } + ] +}) }} + +## Supported trigger actions + +Every trigger handler returns **actions** — declarative descriptions of what should +happen after the event is accepted. The runtime processor reads each action and +dispatches it. Exactly one action is wired end-to-end today; a second is *defined in the +type surface* but not executable, and it now **fails loud** rather than silently dropping. + +{{ comp.apiTable({ + caption: "Trigger actions — what the runtime processor actually does", + rows: [ + { name: "enqueueJob(job, opts)", type: "✅ live", desc: "Hands the payload to the workers plugin. The supported, end-to-end path — it closes the continuous-app loop (event → job → saga). opts: { payload?, idempotencyKey?, concurrencyKey?, priority? }." }, + { name: "defer(...)", type: "⛔ unsupported", desc: "Defined in the action union (DeferAction) but NOT executable. The processor throws an unsupportedOperation error and routes the message to the dead-letter queue (DLQ). There is no deferred replay — do not author a trigger that relies on defer." } + ] +}) }} + +{{ comp callout { type: "warning", title: "defer is defined-but-unsupported — it fails loud" } }} +The defer action exists in the trigger action union, but the runtime processor does +not implement deferred replay. When a handler emits a defer action, the +processor throws an unsupportedOperation error and routes the message to the +dead-letter queue (DLQ) — it does not silently swallow it. Build ingress flows on +enqueueJob only; if you need delayed work, schedule it on the +workers plugin from the enqueued job, or use a +defineScheduledTrigger cron rather than deferring at the trigger edge. + +{{ /comp }} + +## Runtime — ingress, processor, retry + +The trigger runtime (`@netscript/plugin-triggers-core/runtime`) assembles the pieces +the API service and background processor use. You configure these when wiring a custom +host; the scaffold wires them for you. + +{{ comp.apiTable({ + caption: "Trigger runtime entry points (from @netscript/plugin-triggers-core/runtime)", + rows: [ + { name: "createTriggerIngress(options)", type: "→ TriggerIngressPort", desc: "Builds the webhook ingress edge. Options: { definitions, eventStore, processor, verifier, selectVerifier?, resolveSecret?, logger?, now?, createEventId? }. Verifies + persists, then hands off to the processor; responds 202 Accepted." }, + { name: "createTriggerProcessor(options)", type: "→ TriggerProcessor", desc: "Builds the processor that runs handlers and dispatches actions. Options: { idempotency, dlq, dispatchAction?, logger?, now?, random? }. Applies the retry policy and routes exhausted/unsupported events to the DLQ." }, + { name: "defaultRetryPolicy()", type: "→ TriggerRetryPolicy", desc: "The default { maxAttempts, initialDelayMs, maxDelayMs, backoffMultiplier, jitter } policy applied before DLQ handoff. Override per definition via the trigger's retry field." } + ] +}) }} + +{{ comp callout { type: "note", title: "Where the background work runs" } }} +The triggers API service (port 8093) is the HTTP edge — it hosts +createTriggerIngress behind a raw Hono router. The actual trigger +processing — cron, file-watch, and DLQ handling — runs in a separate background +processor entrypoint at plugins/triggers/src/runtime/trigger-processor.ts. Aspire wires +both as resources (triggers-api and trigger-processor) when you aspire start. +{{ /comp }} + +## Endpoints & port + +The triggers API service runs on **`:8093`**. The webhook router resolves the inbound +`:triggerId` segment against your registered `defineWebhook` definitions — so a +definition with `path: 'payments/status'` is reachable at +`POST :8093/api/v1/webhooks/payments/status`. + +{{ comp.apiTable({ + caption: "Triggers API — runtime surface (port 8093, Hono)", + rows: [ + { name: "POST /api/v1/webhooks/:triggerId", type: "Hono route", desc: "Generic dispatch — :triggerId matches the path of any registered defineWebhook definition. Verifies, persists, responds 202 Accepted." }, + { name: "POST /api/v1/webhooks/inbound/generic", type: "Hono route", desc: "The scaffold's generic inbound webhook. Posting JSON here resolves trigger id 'inbound/generic' and runs its actions." }, + { name: "GET /api/v1/events?limit=10", type: "Hono route", desc: "Recent ingress events recorded by the event store." }, + { name: "GET /health", type: "Hono route", desc: "Liveness check for the triggers API service." } + ] +}) }} + +## Author + call a trigger + +The simple case is one webhook that fans an inbound request out to a single job. The +advanced case validates the payload first, then enqueues. The simple case is adapted +from the scaffold's `plugins/triggers/generic-webhook.ts` sample and compiles as-is; the +advanced tab is an illustrative pattern, not scaffold code — the real +`plugins/triggers/webhook-validate-data.ts` is an accept-and-drop sample +(`() => Promise.resolve([])`) that imports no zod and enqueues no job. Every handler +returns an array of `enqueueJob` actions — the only supported trigger action. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — generic-webhook.ts", + lang: "ts", + code: "// plugins/triggers/generic-webhook.ts\nimport { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders';\nimport type { JobDefinition } from '@netscript/plugin-workers-core';\n\n// A reference to the workers job this webhook will enqueue.\nconst healthCheckJob = {\n id: 'workers-plugin-health-check' as JobDefinition<'workers-plugin-health-check'>['id'],\n name: 'Workers Health Check',\n topic: 'default',\n} satisfies JobDefinition<'workers-plugin-health-check'>;\n\n// Inbound POST -> enqueue a workers job. The HTTP call returns immediately;\n// the job runs durably on the workers plugin (:8091).\nexport const genericInboundWebhook = defineWebhook(\n () => Promise.resolve([\n enqueueJob(healthCheckJob, { payload: { verbose: false }, priority: 50 }),\n ]),\n {\n id: 'generic-inbound-webhook',\n path: 'inbound/generic',\n verifier: 'memory',\n description: 'Open webhook that enqueues the workers plugin health-check job.',\n tags: ['webhook', 'runtime-task', 'health-check'],\n },\n);\n\nexport default genericInboundWebhook;" + }, + { + label: "Advanced — validate + enqueue", + lang: "ts", + code: "// plugins/triggers/webhook-validate-data.ts (pattern)\nimport { defineWebhook, enqueueJob } from '@netscript/plugin-triggers-core/builders';\nimport type { JobDefinition } from '@netscript/plugin-workers-core';\nimport { z } from 'zod';\n\n// Validate the inbound body before doing any work.\nconst InboundSchema = z.object({\n userId: z.string().min(1),\n source: z.string().default('webhook'),\n});\n\nconst createUserSettingsJob = {\n id: 'create-user-settings' as JobDefinition<'create-user-settings'>['id'],\n name: 'Create User Settings',\n topic: 'default',\n} satisfies JobDefinition<'create-user-settings'>;\n\nexport const validatedInboundWebhook = defineWebhook(\n (event) => {\n // Parse + validate. On a bad shape, accept-and-drop (return []),\n // or throw to reject the request.\n const parsed = InboundSchema.safeParse(event.payload.body ?? {});\n if (!parsed.success) return Promise.resolve([]);\n\n // Hand the validated payload to a workers job.\n return Promise.resolve([\n enqueueJob(createUserSettingsJob, {\n payload: { userId: parsed.data.userId },\n priority: 80,\n }),\n ]);\n },\n {\n id: 'validated-inbound-webhook',\n path: 'validate/data',\n verifier: 'memory',\n description: 'Validates the body with zod, then enqueues create-user-settings.',\n tags: ['webhook', 'validated'],\n },\n);\n\nexport default validatedInboundWebhook;" + }, + { + label: "Call it — curl", + lang: "bash", + code: "# Triggers API runs on :8093. POST to the webhook's resolved path.\ncurl -X POST http://localhost:8093/api/v1/webhooks/inbound/generic \\\n -H 'content-type: application/json' \\\n -d '{\"verbose\": false}'\n\n# Watch the resulting ingress events:\ncurl 'http://localhost:8093/api/v1/events?limit=10'\n\n# The enqueued job lands on the workers plugin (:8091):\ncurl http://localhost:8091/api/v1/workers/executions?limit=10" + } +] }) }} + +## How it wires to the rest of the app + +This is the last rung of the continuous-app thread. The `generic-inbound-webhook` +enqueues the workers health-check job; the `create-user-settings` job (authored in the +[background jobs](/background-processing/workers/) tutorial) publishes a +`UserSettingsCreated` saga message; the saga from the +[durable workflow](/durable-workflows/sagas/) tutorial handles it. One inbound POST +drives the whole chain — and every link is real scaffold code that compiles. + +## Production notes + +{{ comp callout { type: "warning", title: "Production pitfalls" } }} +
    +
  • Never ship the 'memory' verifier publicly. It accepts any POST so +you can iterate fast. Before exposing a webhook, set verifier: 'hmac-sha256' and a +secretEnv so only a sender holding the secret can enqueue work.
    • +
  • The signing secret lives in env, not the definition. secretEnv +names the env var; the runtime resolves it (resolveSecret) at ingress. Keep the secret +out of source and out of the definition object.
    • +
  • Return fast, work later. A handler should verify, validate, and +enqueueJob — not do the work inline. Webhook senders time out and retry; durable +processing belongs on the workers plugin.
    • +
  • De-dupe sender retries. Pass idempotencyKey (the event's key, or a +stable field) to enqueueJob so a retried POST collapses to one job.
    • +
  • File watches need stability on network shares. Set +stabilityThreshold so a half-written CSV does not fire mid-copy; use +forcePolling on the underlying watcher for network filesystems that miss native events.
    • +
  • Do not reach for defer. It is defined-but-unsupported, throws, and +routes to the DLQ. Schedule delayed work from the enqueued workers job, or use a +defineScheduledTrigger instead.
    • +
+{{ /comp }} + +## Reference + +{{ comp.xref({ key: "ref:triggers" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — triggers core reference", + body: "The full generated API surface for the triggers unit: builders (defineWebhook / defineFileWatch / defineScheduledTrigger), runtime (ingress + processor), adapters (HMAC verifier), and the Hono router types.", + href: "/reference/triggers/", + icon: "≡" + }, + { + title: "Look up — watchers reference", + body: "createWatcher, FileWatcher, strategies (native/polling/hybrid), and the GlobFilter / StabilityFilter / DedupFilter pipeline.", + href: "/reference/watchers/", + icon: "≡" + }, + { + title: "Do — Background jobs", + body: "Where the enqueued work runs: defineJobHandler, worker runtimes, and the enqueue path triggers hand off to.", + href: "/background-processing/workers/", + icon: "◆" + }, + { + title: "Understand — Durable sagas", + body: "The next link in the chain: how an enqueued job's saga message advances a durable workflow.", + href: "/durable-workflows/sagas/", + icon: "◎" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Durable sagas", href: "/durable-workflows/sagas/" }, next: { label: "Durable streams", href: "/durable-workflows/streams/" } }) }} diff --git a/docs/site/identity-access/auth.md b/docs/site/identity-access/auth.md new file mode 100644 index 000000000..deec9d1f6 --- /dev/null +++ b/docs/site/identity-access/auth.md @@ -0,0 +1,351 @@ +--- +layout: layouts/base.vto +title: Authentication +templateEngine: [vento, md] +prev: { label: "Fresh UI & design", href: "/web-layer/fresh-ui/" } +next: null +--- + +# Authentication + +NetScript ships authentication as a first-class official plugin: `netscript plugin install @netscript/plugin-auth` +adds an `auth-api` oRPC service that boots on port **8094** alongside your workers, sagas, and +triggers. The plugin is **pure-backend** — it owns the session lifecycle, OAuth/OIDC redirect +flow, and a stable five-endpoint REST/RPC surface, but it does **not** ship UI. You compose +exactly **one active backend** (single-active-backend), chosen by an environment variable, and +the service exposes the same contract no matter which backend is wired in. + +{{ comp.badge({ status: "alpha" }) }} + +{{ comp.diagram({ + src: "/assets/diagrams/auth-flow.svg", + alt: "A browser sign-in request enters the auth-api service, which resolves the single active backend from an env var; the kv-oauth backend redirects to the OAuth/OIDC provider, handles the callback, mints a normalized session in Deno KV, sets the __Host-ns_session cookie, and maps the session to a NetScript Principal that downstream services trust.", + caption: "Auth flow: browser → auth-api → resolved backend → provider redirect/callback → session store → __Host-ns_session cookie → Principal." +}) }} + +The design follows the framework's contracts-first doctrine: a core seam package +(`@netscript/plugin-auth-core`) defines the `AuthBackendPort`, three interchangeable adapter +packages implement it, and the `@netscript/plugin-auth` plugin composes the selected one into a +running service. Of the three adapters, only the KV-OAuth backend implements the full +**interactive** sign-in/callback flow today; the WorkOS and better-auth adapters are +non-interactive by design. + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for the auth plugin when you need session-based authentication backed by an OAuth +or OIDC provider — a typed /api/v1/auth/* surface your services and Fresh +UI can call to sign users in, resolve the current session, and sign out. The default +kv-oauth backend gives you a complete interactive redirect flow with KV-stored +sessions and an opaque HMAC-signed session token. To wire it into a workspace step by step, see +Add authentication; to understand the pure-backend +port model and why only one backend is active, see +The authentication model. +{{ /comp }} + +## What it is + +Auth is a **port-and-adapter seam, not a monolith**. The core package +`@netscript/plugin-auth-core` is contract-only: it defines the `AuthBackendPort` (and the four +sub-ports it composes — provider registry, session store, token crypto, and principal mapper), +the normalized `AuthSession` and `Principal` shapes, the `AUTH_SESSION_STATES`, the auth oRPC +contract, the config schemas, and the stream-event schemas. It contains **no provider SDK code**. +Each backend adapter (`@netscript/auth-kv-oauth`, `@netscript/auth-workos`, +`@netscript/auth-better-auth`) implements `AuthBackendPort` against a concrete identity provider, +and the `@netscript/plugin-auth` plugin selects exactly one of them at boot via +`NETSCRIPT_AUTH_BACKEND` and serves it as the `auth-api` service. Because every backend produces +the same `Principal`, downstream services authorize identically regardless of which provider is +wired in. The full rationale — why auth is pure-backend, what the single-active boundary buys you, +and how `InteractiveFlowPort` gates the redirect flow — is in +{{ comp.xref({ key: "explain:auth-model", text: "The authentication model" }) }}. + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Do — Add authentication", + body: "Task recipe: add the auth plugin, set NETSCRIPT_AUTH_BACKEND, run the auth.prisma migration, and wire kv-oauth with a provider preset.", + href: "/how-to/add-authentication/", + icon: "◆" + }, + { + title: "Understand — The authentication model", + body: "Why auth is pure-backend: the AuthBackendPort seam, the single-active-backend boundary, InteractiveFlowPort, and the three adapters' capability matrices.", + href: "/explanation/auth-model/", + icon: "✦" + }, + { + title: "Compose — Service authn / authz", + body: "Gate a service's own routes with the provider-agnostic /auth middleware seam — verify a credential or trust a header, complementary to this plugin.", + href: "/services-sdk/services/", + icon: "→" + } +] }) }} + +## Add it in two shapes + +The fastest path is the **provider preset**: add the plugin, set `NETSCRIPT_AUTH_BACKEND` +(it already defaults to `kv-oauth`), and hand a preset like `providers.google({...})` to +`createKvOAuthBackend`. The **advanced** shape uses the same factory but defines a custom OIDC +provider explicitly with `defineOAuthProvider` when your identity provider is not one of the +fourteen built-in presets. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — provider preset", + lang: "ts", + code: "// Add the plugin from the workspace root:\n// netscript plugin install @netscript/plugin-auth\n// Default backend is already kv-oauth (NETSCRIPT_AUTH_BACKEND=kv-oauth).\n\nimport { createKvOAuthBackend, getRequiredEnv, providers } from '@netscript/auth-kv-oauth';\n\n// One of 14 presets: github, google, gitlab, discord, slack, spotify,\n// facebook, twitter, auth0, okta, awsCognito, azureAd, logto, clerk.\nexport const backend = await createKvOAuthBackend({\n provider: providers.google({\n clientId: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_ID'),\n clientSecret: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_SECRET'),\n redirectUri: getRequiredEnv('NETSCRIPT_AUTH_REDIRECT_URI'),\n }),\n});\n\n// backend.name === 'kv-oauth' — the only adapter with backend.interactive." + }, + { + label: "Advanced — explicit provider", + lang: "ts", + code: "// Same factory, but define the OIDC provider yourself when your IdP\n// is not one of the built-in presets.\nimport { createKvOAuthBackend, defineOAuthProvider, getRequiredEnv } from '@netscript/auth-kv-oauth';\n\nconst provider = defineOAuthProvider({\n id: 'my-idp',\n clientId: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_ID'),\n clientSecret: getRequiredEnv('NETSCRIPT_AUTH_CLIENT_SECRET'),\n authorizationEndpoint: getRequiredEnv('NETSCRIPT_AUTH_AUTHORIZATION_ENDPOINT'),\n tokenEndpoint: getRequiredEnv('NETSCRIPT_AUTH_TOKEN_ENDPOINT'),\n userInfoEndpoint: getRequiredEnv('NETSCRIPT_AUTH_USERINFO_ENDPOINT'),\n redirectUri: getRequiredEnv('NETSCRIPT_AUTH_REDIRECT_URI'),\n scopes: ['openid', 'profile', 'email'],\n});\n\nexport const backend = await createKvOAuthBackend({ provider });" + } +] }) }} + +{{ comp callout { type: "note", title: "Alpha package pins" } }} +The scaffold pins jsr:@netscript/plugin-auth-core{{ releaseSpecifier }} and siblings +at the exact aligned NetScript version. Add auth through +netscript plugin install @netscript/plugin-auth, which wires the workspace correctly for +the alpha instead of hand-editing imports. +{{ /comp }} + +## Sign in, resolve the session, sign out + +Once the plugin is wired, your front end or a typed client drives the five-endpoint surface. A +sign-in begins the redirect, the provider returns to your callback, the service mints a session +and sets the cookie, and every subsequent request resolves the current session from that cookie. +The snippet below is a copy-ready client flow against the REST surface. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Browser / typed client flow", + lang: "ts", + code: "// app/sign-in.ts — drive the auth-api REST surface from the browser\nconst AUTH = 'http://localhost:8094/api/v1/auth';\n\n// 1. Begin sign-in. The kv-oauth backend responds with a provider redirect;\n// follow it to authenticate with the IdP.\nlocation.href = `${AUTH}/signin`;\n\n// 2. The IdP redirects back to /api/v1/auth/callback, which mints the\n// session and sets the __Host-ns_session cookie automatically.\n\n// 3. Resolve the current session on any later request (cookie is sent\n// automatically; the response is { authenticated, user, session }).\nconst me = await fetch(`${AUTH}/me`, { credentials: 'include' })\n .then((r) => r.json());\nif (me.authenticated) {\n console.log('signed in as', me.user.subject, '— state', me.session.state);\n}\n\n// 4. Sign out — revokes the session and clears the cookie.\nawait fetch(`${AUTH}/signout`, { method: 'POST', credentials: 'include' });" + }, + { + label: "Server-side backend port", + lang: "ts", + code: "// server/auth-guard.ts — use the resolved backend directly in app code\nimport { AuthBackendOperationUnsupportedError } from '@netscript/plugin-auth-core/ports';\nimport { backend } from './backend.ts';\n\n// Resolve the current session from an incoming Request via the session store.\nconst session = await backend.sessions.getSession({ request: incoming });\nif (session?.state === 'active') {\n // Map the normalized session to a NetScript Principal for authorization.\n const { principal } = backend.principalMapper.mapSessionToPrincipal(session);\n console.log(principal.subject, principal.scopes, principal.roles);\n}\n\n// Interactive sign-in is optional capability — guard before you call it.\nif (!backend.interactive) {\n throw new AuthBackendOperationUnsupportedError(\n backend.name,\n 'interactive.signIn',\n 'The active backend authenticates sessions only.',\n );\n}\nconst redirect = await backend.interactive.signIn(incoming);" + } +] }) }} + +## Key types first — the core auth contract + +Every backend normalizes to the same two shapes the rest of NetScript depends on: an +`AuthSession` (what the store persists) and a `Principal` (what services authorize against). The +`AuthSession` is mapped to a `Principal` by the backend's `principalMapper`, and the +`authenticate()` method on every backend returns an `AuthnResult`. These are the contract-only +types from `@netscript/plugin-auth-core` — confirmed against the live export surface. + +{{ comp.apiTable({ + caption: "Principal (@netscript/plugin-auth-core) — the identity services authorize against", + rows: [ + { name: "subject", type: "string", desc: "Stable subject identifier — a user id, service id, or API-key id." }, + { name: "scopes", type: "readonly string[]", desc: "Granted scopes for per-operation RPC/REST permission checks." }, + { name: "roles", type: "readonly string[]", desc: "Granted roles for role-based access checks." }, + { name: "scheme", type: "'api-key' | 'bearer' | 'trusted-header' | 'custom'", desc: "How the principal was established. Auth-plugin backends use 'custom' with the claim bag below." }, + { name: "claims", type: "Readonly>", desc: "Opaque verified claims — organization/tenant id, session id, provider permissions, or normalized WorkOS/better-auth metadata." } + ] +}) }} + +{{ comp.apiTable({ + caption: "AuthSession (@netscript/plugin-auth-core) — the normalized session the store persists", + rows: [ + { name: "id", type: "string", desc: "Stable session id; the value an opaque session token resolves to." }, + { name: "userId / subject", type: "string", desc: "User id and stable subject the session belongs to." }, + { name: "state", type: "AuthSessionState", desc: "Lifecycle state: 'active' | 'expired' | 'revoked' (from AUTH_SESSION_STATES)." }, + { name: "scopes / roles", type: "readonly string[]", desc: "Scopes and roles carried into the mapped Principal." }, + { name: "claims", type: "Readonly>", desc: "Verified provider claims preserved on the session." }, + { name: "issuedAt / expiresAt", type: "string (ISO)", desc: "Issue and expiry timestamps; refreshedAt / revokedAt are set on those transitions." }, + { name: "accountId / providerId", type: "string?", desc: "Optional linked provider account and provider id." }, + { name: "traceparent / tracestate", type: "string?", desc: "Optional W3C trace context carried for audit correlation." } + ] +}) }} + +{{ comp.apiTable({ + caption: "AuthnResult (@netscript/plugin-auth-core) — what backend.authenticate() returns", + rows: [ + { name: "{ ok: true, principal }", type: "success", desc: "A resolved Principal; optional responseHeaders and setCookies the service flushes to the client." }, + { name: "{ ok: false, reason }", type: "rejection", desc: "A typed rejection reason — fail-loud, never a silent anonymous principal." } + ] +}) }} + +## The service surface + +The plugin's service is named `auth-api` and is built with `@netscript/service`'s oRPC builder +(`createService(...).withRPC()`), not raw Hono. It mounts a public REST surface at +`/api/v1/auth/*` and the equivalent oRPC surface at `/api/rpc/v1/auth/*`, plus standard +`/health/live` and `/health/ready` probes. + +{{ comp.apiTable({ + caption: "auth-api endpoints (REST at /api/v1/auth/*, oRPC at /api/rpc/v1/auth/*)", + rows: [ + { name: "signin", type: "POST", desc: "Begins the interactive sign-in. Requires backend.interactive; on WorkOS / better-auth it returns AUTH_PROVIDER_ERROR (502) because those backends are non-interactive." }, + { name: "callback", type: "POST", desc: "Completes the OAuth/OIDC redirect, mints the session, sets the session cookie. Interactive-only — same non-interactive caveat as signin." }, + { name: "signout", type: "POST", desc: "Revokes the current session and clears the cookie." }, + { name: "session", type: "GET", desc: "Resolves the current AuthSession from the cookie (active | expired | revoked), refreshing on read when policy allows." }, + { name: "me", type: "GET", desc: "Returns { authenticated: true, user, session } when a valid active session exists, or { authenticated: false } (HTTP 200) when there is none." } + ] +}) }} + +{{ comp callout { type: "note", title: "Backend selection is one env var" } }} +The service picks one active backend from NETSCRIPT_AUTH_BACKEND +(or appsettings auth.backend): valid values are kv-oauth | +workos | better-auth, defaulting to kv-oauth. +This is the single-active-backend boundary — there is no multi-active routing, +cross-backend account linking, or global logout across backends in v1. Switching providers means +changing the env var and the backend's credentials, not running two backends at once. + +{{ /comp }} + +## The three backends + +All three implement the same `AuthBackendPort`, so the `auth-api` contract is identical across +them. They differ in one decisive capability: whether they expose an interactive +sign-in/callback flow. Each ships a single factory that returns an `AuthBackendPort`. + +{{ comp.apiTable({ + caption: "Backend adapters: factory, capability, and what each one needs", + rows: [ + { name: "@netscript/auth-kv-oauth", type: "createKvOAuthBackend(options)", desc: "Interactive (default). Full OAuth/OIDC redirect flow, Deno KV session store, real createSession / refreshSession / revokeSession, refresh-on-read (refreshMode / refreshSkewMs), opaque session token, __Host-ns_session cookie, AES-256-GCM token-at-rest. Needs a provider preset or defineOAuthProvider config." }, + { name: "@netscript/auth-workos", type: "createWorkosBackend({ workos, cookiePassword })", desc: "Non-interactive. WorkOS AuthKit sealed sessions, stateless verification. Needs a WorkOS SDK client and a cookie password. signin / callback return AUTH_PROVIDER_ERROR; session mutations throw AuthBackendOperationUnsupportedError." }, + { name: "@netscript/auth-better-auth", type: "createBetterAuthBackend({ auth, sessionTokenSecret })", desc: "Non-interactive. Validates externally-issued sessions via auth.api.getSession over a Prisma store (createNetscriptBetterAuth wires the better-auth Prisma adapter). Needs a better-auth instance and a token secret." } + ] +}) }} + +{{ comp callout { type: "important", title: "Only kv-oauth is interactive" } }} +The signin and callback endpoints require a backend that implements the +optional InteractiveFlowPort (signIn / handleCallback / +getSessionId / signOut). Today that is only +kv-oauth. On workos and better-auth those two +endpoints return a typed AUTH_PROVIDER_ERROR ("does not expose an interactive flow"), +and direct session mutations throw AuthBackendOperationUnsupportedError — a +deliberate, fail-loud capability boundary rather than a silent no-op. Use those adapters when +sign-in is handled by WorkOS or an external better-auth deployment and NetScript only verifies the +resulting session. +{{ /comp }} + +## How auth integrates with services + +The auth **plugin** (this page) signs human users in and resolves their sessions. A +NetScript **service** gates its own routes with the separate, provider-agnostic +`@netscript/service/auth` seam — `.withAuthn()` resolves a `Principal` and `.withAuthz()` makes +an authorization decision from it. Both layers speak the same `Principal` type, so they compose: +the auth plugin establishes identity, and a service's `.withAuthn({ authenticator })` can trust a +backend's authenticator to turn a request into that `Principal`. By default a service protects +`/api` and leaves `/health` anonymous. The full builder seam — `createStaticCredentialAuthenticator`, +`createTrustedHeaderAuthenticator`, `createScopeAuthorizer`, and the preset +`defineService(router, { auth: { authn, authz } })` form — is documented on the services hub. + +{{ comp callout { type: "note", title: "Plugin vs. service-auth seam" } }} +The auth plugin owns interactive sign-in, OAuth callbacks, and session cookies. +The service-auth seam is for +machine-to-machine and gateway-fronted gating — static credentials, trusted upstream headers, +scope checks — without an interactive identity provider. They are complementary and can run +together: sign users in with the plugin, then gate individual service routes with +.withAuthn() / .withAuthz(). See +{{ comp.xref({ key: "cap:services", text: "Services & contracts" }) }}. +{{ /comp }} + +## Database & runtime events + +The plugin contributes a package-provided `auth.prisma` schema with four +better-auth-shaped models — `User` → `auth_users`, `Session` → `auth_sessions`, +`Account` → `auth_accounts`, `Verification` → `auth_verifications`. These tables back the +better-auth adapter; `kv-oauth` keeps its sessions in Deno KV, and WorkOS is stateless. As with +every plugin schema, you bring the tables to life by running the database workflow **after Aspire +is up** — see [Database migrations](/how-to/database-migration/). + +The plugin also emits five durable `auth.*` runtime events through the durable-streams runtime — +the `AUTH_STREAM_EVENT_TYPES`: `auth.signin.started`, `auth.signin.failed`, +`auth.token.refreshed`, `auth.session.revoked`, and `auth.oidc.completed`. The session projection +is described by the `authStreamSchema` entity stream. For the streams runtime itself see +{{ comp.xref({ key: "cap:streams", text: "Durable streams" }) }}. + +{{ comp callout { type: "warning", title: "Scope: events are best-effort, no audit surface" } }} +The auth.* events are best-effort: they are no-ops unless the +durable-streams service is wired (DURABLE_STREAMS_URL / +services__streams__http__0 set), so they are not a guaranteed audit +trail. Only oidc.completed, token.refreshed, and +session.revoked write the session projection; signin.started / +signin.failed are diagnostic-only. There is no dedicated auth telemetry or +audit-observability surface yet — do not build dashboards expecting one. For general +tracing and structured logs see Telemetry & logging. +{{ /comp }} + +## Endpoints & ports + +{{ comp.apiTable({ + caption: "Authentication runtime surface", + rows: [ + { name: ":8094", type: "port", desc: "auth-api default port (AUTH_API_DEFAULT_PORT). Family: workers :8091, sagas :8092, triggers :8093, auth :8094." }, + { name: "auth-api", type: "service name", desc: "AUTH_API_SERVICE_NAME — the service contribution the auth plugin (AUTH_PLUGIN_ID 'auth') adds." }, + { name: "/api/v1/auth/*", type: "REST", desc: "Public REST surface: signin, callback, signout, session, me." }, + { name: "/api/rpc/v1/auth/*", type: "oRPC", desc: "The oRPC surface for the same five operations, for typed NetScript clients." }, + { name: "/health/live", type: "HTTP", desc: "Liveness probe; /health/ready for readiness. OpenAPI + docs served via .withOpenAPI()/.withDocs()." }, + { name: "NETSCRIPT_AUTH_BACKEND", type: "env", desc: "Selects the single active backend: kv-oauth (default) | workos | better-auth." } + ] +}) }} + +## Production notes + +{{ comp callout { type: "warning", title: "Footguns before you ship" } }} +
    +
  • Only kv-oauth signs users in. If +NETSCRIPT_AUTH_BACKEND is workos or better-auth, the +signin / callback endpoints return AUTH_PROVIDER_ERROR by +design — those adapters only verify externally-issued sessions. Pick the backend that matches who +owns the sign-in.
    • +
  • The session cookie is __Host-ns_session. The +__Host- prefix requires Path=/, no Domain, and +HTTPS — a misconfigured domain or a plain-HTTP origin makes the cookie refuse to +set. Use a real TLS origin in production.
    • +
  • Switching providers is an env-var + credentials change, not a runtime toggle. +There is no multi-active routing, cross-backend account linking, or global logout across backends +in v1 (single-active-backend).
    • +
  • Auth audit is opt-in, and not a compliance-grade ledger. A real, structured, +redacted auth audit surface ships via createAuthTelemetry, but it only records when a +subjectHashSalt is configured (resolved from NETSCRIPT_AUTH_AUDIT_SALT) — +with no salt it runs as a no-op. The raw auth.* stream events are themselves best-effort +and no-op until the durable-streams service is wired. Treat the audit surface as a strong, queryable +trail for operational and security review, not a tamper-evident, write-once compliance log. See +{{ comp.xref({ key: "explain:observability" }) }}.
    • +
  • Run the auth.prisma migration after Aspire is up. The +better-auth backend's tables (and any DB-backed flow) need your database provisioned first +(Postgres is the recommended engine; or mysql / mssql / sqlite via +--db — better-auth persists through Prisma, so it is not Postgres-specific) — the same +Aspire-first ordering every plugin schema follows.
    • +
+{{ /comp }} + +## Reference + +The auth runtime is a `@netscript/service`; the auth plugin, core contract package, and backend +adapters now have dedicated generated reference pages. + +{{ comp.xref({ key: "ref:auth" }) }} +{{ comp.xref({ key: "ref:service" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Do — Add authentication", + body: "Task recipe: add the auth plugin, set NETSCRIPT_AUTH_BACKEND, run the auth.prisma migration, and wire kv-oauth with a provider preset.", + href: "/how-to/add-authentication/", + icon: "◆" + }, + { + title: "Understand — The authentication model", + body: "Why auth is pure-backend: the AuthBackendPort seam, the single-active-backend boundary, InteractiveFlowPort, and the three adapters' capability matrices.", + href: "/explanation/auth-model/", + icon: "✦" + }, + { + title: "Look up — @netscript/service reference", + body: "The oRPC service builder behind auth-api: createService().withRPC(), the contract surface, the /auth authn-authz seam, and the runtime that serves /api/v1/auth/*.", + href: "/reference/service/", + icon: "≡" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Fresh UI & design", href: "/web-layer/fresh-ui/" }, next: null }) }} + + diff --git a/docs/site/observability/index.md b/docs/site/observability/index.md index 2c00c9e9a..1717a59d0 100644 --- a/docs/site/observability/index.md +++ b/docs/site/observability/index.md @@ -12,7 +12,7 @@ inspect runtime behavior, or connect package-level telemetry to the running syst {{ comp.cardsGrid({ columns: 3, cards: [ { eyebrow: "Overview & Concepts", title: "Telemetry model", body: "Trace context and logging across services, workers, sagas, and orchestration.", href: "/explanation/observability/", icon: "O" }, - { eyebrow: "Quickstart", title: "Turn on tracing", body: "Run aspire start, trigger a job, and watch the runtime's automatic spans land in the dashboard — the lowest-effort path to a first trace.", href: "/capabilities/telemetry/", icon: "Q" }, + { eyebrow: "Quickstart", title: "Turn on tracing", body: "Run aspire start, trigger a job, and watch the runtime's automatic spans land in the dashboard — the lowest-effort path to a first trace.", href: "/observability/telemetry/", icon: "Q" }, { eyebrow: "How-To", title: "Add OpenTelemetry", body: "Wire OTel into a workspace.", href: "/how-to/add-opentelemetry/", icon: "H" }, { eyebrow: "API Reference", title: "telemetry", body: "Generated telemetry package symbols.", href: "/reference/telemetry/", icon: "R" }, { eyebrow: "API Reference", title: "logger", body: "Generated logger package symbols.", href: "/reference/logger/", icon: "R" } diff --git a/docs/site/observability/telemetry.md b/docs/site/observability/telemetry.md new file mode 100644 index 000000000..196d58173 --- /dev/null +++ b/docs/site/observability/telemetry.md @@ -0,0 +1,436 @@ +--- +layout: layouts/base.vto +title: Telemetry & logging +templateEngine: [vento, md] +prev: { label: "KV, queues & cron", href: "/data-persistence/kv-queues-cron/" } +next: { label: "Fresh UI & design", href: "/web-layer/fresh-ui/" } +--- + +# Telemetry & logging + +NetScript treats observability as a built-in, not a bolt-on. Every service and plugin +runtime is wired for **OpenTelemetry** — services serve their RPC handlers with +trace-context propagation, the worker runtime wraps job dispatch, execution, scheduling, +subprocess hand-off, and task execution in real OTel spans, and structured logs flow through +the framework `logger`. The viewing surface is the **Aspire dashboard at +`https://localhost:18888`**, which collects OTLP traces, metrics, and structured logs from +every resource in the app graph (services, plugin APIs, background processors) the moment you +run `aspire start`. You do not stand up Jaeger, Grafana, or a log shipper to get started — the +AppHost provisions the OTLP collector and the dashboard for you. + +{{ comp.diagram({ + src: "/assets/diagrams/otel-traceparent.svg", + alt: "A W3C traceparent header propagates from a service request through the worker runtime into a subprocess and across a saga boundary, keeping every span under one trace id.", + caption: "Trace-context propagation: a single traceparent (W3C) flows service → worker → subprocess → saga, so spans from every runtime join one distributed trace in Aspire." +}) }} + +This page is the capability hub: what telemetry exists, how to emit it, where to view it, +and the one place that is still a scaffold stub. For the full generated API of +each unit, follow the reference links: the telemetry primitives live at +[`/reference/telemetry/`](/reference/telemetry/) and the structured logger at +[`/reference/logger/`](/reference/logger/). + +{{ comp callout { type: "important", title: "Aspire first, then telemetry" } }} +The dashboard, the OTLP collector, and the per-resource trace/log views all come up with the +orchestrator — they are not separate processes you start by hand. Run +cd aspire && aspire start before any netscript db +command (Aspire provisions Postgres and Redis first), then open the dashboard URL printed in +the console (https://localhost:18888, with a one-time auth token). Until Aspire is +running there is no :18888 surface to view traces or logs on. See +Database & migration for the full startup order. +{{ /comp }} + +## What it is + +Telemetry in NetScript is three OpenTelemetry signals — **traces**, **metrics**, and +**logs** — wired through one package, `@netscript/telemetry`, and one viewing surface, the +Aspire dashboard. Traces are the headline: a request enters a service, the worker runtime +dispatches and executes a job, a subprocess runs a polyglot task, and a saga advances — and +because a single **W3C `traceparent`** header propagates across every one of those process +boundaries, the whole fan-out collapses into one trace tree. Metrics (worker counts, SSE +connection stats) and structured logs ride the same OTLP export. The framework owns transport +(OTLP/HTTP to the collector) and viewing (the dashboard); it now owns most of emission too — +the worker, scheduler, queue, saga, and SSE runtimes are instrumented for you. The full +mental model — what is framework-real versus a scaffold stub — is in +[Observability](/explanation/observability/). + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Do — Add OpenTelemetry", + body: "Task-oriented recipe: lean on the automatic job spans, add custom spans via @netscript/telemetry withChildSpan, propagate traceparent across services and subprocesses, and point the OTLP export (:4318) at the dashboard.", + href: "/how-to/add-opentelemetry/", + icon: "◆" + }, + { + title: "Understand — Observability", + body: "The mental model: how spans, structured logs, health endpoints, and Aspire traces fit together — and the precise framework-vs-scaffold map of what is real (traceJobExecution, task.execute) versus the tracked createJobTools(ctx) stub.", + href: "/explanation/observability/", + icon: "◎" + }, + { + title: "Look up — telemetry reference", + body: "The full generated deno doc API for @netscript/telemetry — tracer facade, context/traceparent helpers, the NetScript instrumentation helpers, and the oRPC tracing plugin.", + href: "/reference/telemetry/", + icon: "≡" + } +] }) }} + +## What you get out of the box + +Telemetry in NetScript spans three layers — emission (in framework runtimes and your handler +code), transport (OTLP over HTTP to the collector), and viewing (the Aspire dashboard). The +framework owns transport and viewing, and it now owns most of emission too: the worker runtime +traces the entire job lifecycle automatically. The only thing you still wire by hand is custom +spans *inside* a job handler — and even there the `@netscript/telemetry` helpers do the work. + +{{ comp.featureGrid({ items: [ + { + title: "Structured logging", + body: "The @netscript/logger unit gives every service and plugin a level-aware, structured logger. Config is declared in netscript.config.ts (logging: { level: 'info', format: 'text' }) and threaded through the runtime — text or JSON, filterable in the dashboard.", + icon: "▤" + }, + { + title: "OpenTelemetry traces (real)", + body: "Service request paths propagate W3C trace context, and the worker runtime emits real spans for job dispatch, execution, scheduling, subprocess hand-off, and task.execute. @opentelemetry/api (^1.9) is in the catalog and wired into the request and job paths so spans flow to the collector.", + icon: "◈" + }, + { + title: "OTLP → Aspire dashboard", + body: "The generated AppHost configures an OTLP endpoint (http://localhost:4318) and the Aspire dashboard (https://localhost:18888) so traces, metrics, and logs from every resource land in one place — no collector to deploy.", + icon: "◎" + }, + { + title: "Browser console logs (default)", + body: "Generated app resources call withBrowserLogs() automatically (the AppHost pins Aspire.Hosting.Browsers), so a Fresh/Vite app's browser console output — including a client-side error an island throws — is forwarded into the dashboard next to server logs and traces. No opt-in, nothing to wire.", + icon: "▷" + }, + { + title: "Per-resource health", + body: "Each plugin API exposes a liveness probe — workers :8091 GET /health, sagas :8092 GET /health/live, triggers :8093 GET /health, auth :8094 — surfaced as resource state in the dashboard.", + icon: "✚" + } +] }) }} + +{{ comp callout { type: "note", title: "Browser logs land in the dashboard too" } }} +Because generated app resources emit withBrowserLogs() by default, the class of failure +that is otherwise invisible from the server — a client-side TypeError a durable-streams +island throws in the browser, say — shows up in the Aspire dashboard's Console logs +for that app resource, correlated with the server-side spans of the same request. You do not start a +separate browser-log collector; the AppHost wires it when it scaffolds the app. +{{ /comp }} + +## Enable tracing & see a span + +Tracing turns on from the **environment** — `@netscript/telemetry/config` resolves a +`TelemetryConfig` from the standard `OTEL_*` variables, and the Aspire AppHost already sets +them for you. The fastest path is to run `aspire start`, trigger any job, and the +runtime's automatic spans show up in the dashboard with no code change. The tab below adds a +**custom** span on top of that automatic trace. + +{{ comp callout { type: "note", title: "Local vs deployed: same instrumentation, different collector" } }} +Locally, Aspire provisions the OTLP collector and the dashboard and injects the +OTEL_* variables into every resource, so telemetry works the moment you run +aspire start — nothing to configure. Note the dashboard and OTLP endpoints bind to +ephemeral ports (the generated profile uses localhost:0), so use the URL +aspire start prints rather than assuming :18888/:4318. Running +aspire start --isolated (an upstream Aspire CLI flag) also randomizes the dashboard, OTLP, +and resource-service ports and isolates secrets, so several apphosts can boot in parallel without +colliding. +Deployed, there is no Aspire and no dashboard: you point +OTEL_EXPORTER_OTLP_ENDPOINT at your own collector or a hosted backend (Grafana Tempo, +Honeycomb, Jaeger, …). The instrumentation code is identical — only the endpoint/protocol env and the +sampler change. That single OTLP seam is the whole porting story; see Production notes below. +{{ /comp }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "1 — Turn it on (env-driven)", + lang: "ts", + code: "// netscript.config.ts — declare the structured-logging contract for the workspace.\n// Tracing itself is enabled by OTEL_* env vars (the Aspire AppHost sets these); the\n// config package reads them — there is no defineTelemetry() call to make.\nimport { defineConfig } from '@netscript/config';\n\nexport default defineConfig({\n name: 'my-app',\n version: '1.0.0',\n logging: { level: 'info', format: 'text' }, // level: debug|info|warn|error\n plugins: ['./plugins/workers/mod.ts'],\n});\n\n// Anywhere in app code you can inspect what telemetry resolved (log-safe, redacted):\nimport { describeTelemetryConfig, isTelemetryEnabled } from '@netscript/telemetry/config';\nif (isTelemetryEnabled()) console.log(describeTelemetryConfig());\n// → { enabled: true, endpoint: 'http://localhost:4318', protocol, serviceName, sampler }" + }, + { + label: "2 — Add a custom span", + lang: "ts", + code: "// plugins/workers/jobs/health-check.ts — a real child span inside a handler.\n// The dispatcher already wraps this job in traceJobExecution automatically;\n// withChildSpan opens a child span under that parent. recordJobProgress's 3rd arg\n// is a UNIT label (e.g. 'steps'), not a free-text message.\nimport { createSuccessResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { withChildSpan, recordJobProgress } from '@netscript/telemetry/instrumentation';\n\nconst handler = defineJobHandler(async (ctx) => {\n recordJobProgress(1, 2, 'steps'); // → job.progress event on the trace\n\n const envCheck = await withChildSpan('check.environment', async (span) => {\n span.setAttribute('check.name', 'environment');\n return { ok: true };\n });\n\n recordJobProgress(2, 2, 'steps');\n return createSuccessResult({ healthy: envCheck.ok });\n});\n\nexport default Object.assign(handler, { id: 'workers-plugin-health-check' as const });" + } +] }) }} + +{{ comp callout { type: "tip", title: "Lowest-effort observability path" } }} +Run aspire start, trigger a job, and open :18888 → Traces. +You will see the dispatch span, the execution span, progress and step events, and — for +subprocess or polyglot tasks — a continued trace across the process boundary. None of that +requires editing the scaffold. Reach for custom instrumentation only when you want spans +inside your own handler logic. +{{ /comp }} + +## Key types: the telemetry init/config surface + +Configuration is **read from the environment**, not constructed — `@netscript/telemetry/config` +resolves and caches a `TelemetryConfig` from the `OTEL_*` variables. These are the confirmed +fields and the helpers that read them. + +{{ comp.apiTable({ + caption: "TelemetryConfig (@netscript/telemetry/config — resolved from OTEL_* env vars)", + rows: [ + { name: "enabled", type: "boolean", desc: "Whether OpenTelemetry instrumentation is active (driven by OTEL_DENO / endpoint presence). Mirror it with isTelemetryEnabled()." }, + { name: "endpoint", type: "string | undefined", desc: "OTLP exporter endpoint URL (e.g. http://localhost:4318). Read directly via getOtlpEndpoint()." }, + { name: "protocol", type: "string", desc: "OTLP exporter protocol (e.g. http/protobuf), from OTEL_EXPORTER_OTLP_PROTOCOL." }, + { name: "serviceName", type: "string", desc: "Service name reported to backends, from OTEL_SERVICE_NAME (or the default). Read via getServiceName()." }, + { name: "serviceVersion", type: "string", desc: "Service version reported to backends." }, + { name: "resourceAttributes", type: "Record", desc: "Resource attributes parsed from OTEL_RESOURCE_ATTRIBUTES." }, + { name: "sampler", type: "string", desc: "Trace sampler name, from OTEL_TRACES_SAMPLER (e.g. parentbased_always_on, traceidratio)." }, + { name: "debug", type: "boolean", desc: "Whether debug-level telemetry logging is on (OTEL_LOG_LEVEL)." } + ] +}) }} + +{{ comp.apiTable({ + caption: "Config helpers (@netscript/telemetry/config)", + rows: [ + { name: "getTelemetryConfig()", type: "→ TelemetryConfig", desc: "Resolve telemetry config from the OTEL_* environment variables." }, + { name: "getConfig()", type: "→ TelemetryConfig", desc: "Return the process-cached config (resetConfig() clears the cache, mainly for tests)." }, + { name: "isTelemetryEnabled()", type: "→ boolean", desc: "Whether instrumentation is enabled for this process." }, + { name: "getOtlpEndpoint()", type: "→ string | undefined", desc: "The configured OTLP endpoint, when present." }, + { name: "getServiceName()", type: "→ string", desc: "The configured service name, or the default." }, + { name: "describeTelemetryConfig()", type: "→ TelemetryConfigDescription", desc: "A redacted, log-safe summary of the resolved config — safe to print in diagnostics." }, + { name: "OTEL_ENV_VARS", type: "const record", desc: "The OTEL_* variable NAMES NetScript reads: OTEL_DENO, OTEL_EXPORTER_OTLP_ENDPOINT/_PROTOCOL, OTEL_SERVICE_NAME, OTEL_RESOURCE_ATTRIBUTES, OTEL_TRACES_SAMPLER, OTEL_LOG_LEVEL, the BSP/BLRP schedule delays, and OTEL_METRIC_EXPORT_INTERVAL." } + ] +}) }} + +## Span & instrumentation helpers + +Two subpaths cover hand-written instrumentation. `@netscript/telemetry/tracer` is the +low-level facade over `@opentelemetry/api` — get a tracer, open a span, run a callback inside +one. `@netscript/telemetry/instrumentation` is the NetScript-domain layer the runtimes +themselves use — job, scheduler, queue, and SSE spans with the right attributes baked in. +The `/context` subpath holds the **`traceparent`** propagation helpers that make the +cross-boundary trace in the diagram above possible. + +{{ comp.apiTable({ + caption: "Tracer facade (@netscript/telemetry/tracer)", + rows: [ + { name: "getTracer(name, version)", type: "→ Tracer", desc: "Cached tracer for an instrumentation name/version. Domain shortcuts exist: getJobTracer/getQueueTracer/getWorkerTracer/getSchedulerTracer/getSagaTracer/getSSETracer/getKVTracer." }, + { name: "withSpan(tracer, name, fn, options)", type: "→ Promise", desc: "Run an async callback inside a span and close it on completion. withSpanSync is the synchronous form." }, + { name: "createSpan(tracer, name, options)", type: "→ Span", desc: "Create a span from a tracer + CreateSpanOptions (kind, attributes, parentContext, links). You end() it yourself." }, + { name: "setSpanAttributes / setSpanError / setSpanOk", type: "(span, …) → void", desc: "Bulk-set attributes, mark a span failed (with an optional Error), or mark it OK. addSpanEvent(span, name, attrs?) adds an event." }, + { name: "getActiveSpan() / getActiveContext()", type: "→ Span? / Context", desc: "Read the span / OTel context active on the current async path. isTracingEnabled() gates work." }, + { name: "SpanKind / SpanStatusCode / TracerNames", type: "const", desc: "Span-kind (INTERNAL/SERVER/CLIENT/PRODUCER/CONSUMER), status (UNSET/OK/ERROR), and the standard domain tracer-name strings." } + ] +}) }} + +{{ comp.apiTable({ + caption: "NetScript instrumentation helpers (@netscript/telemetry/instrumentation)", + rows: [ + { name: "traceJobExecution(options, fn)", type: "→ Promise", desc: "Wrap a job run in a span with job.started/completed/failed events. Emitted automatically by the dispatcher; the supported entry point for tracing a job by hand." }, + { name: "withChildSpan(name, fn, attributes?)", type: "→ Promise", desc: "Open a child span under the active span and run fn inside it. The go-to for custom spans in a handler." }, + { name: "recordJobProgress(current, total, unit)", type: "→ void", desc: "Emit a job.progress event. The 3rd arg is a UNIT label (e.g. 'steps'), NOT a description. addJobStepEvent(stepName, attrs?) records job.step.* events." }, + { name: "startJobDispatchSpan / traceJobDispatch", type: "span / → Promise", desc: "Open the dispatch span and return PropagationHeaders to carry traceparent to the executor. createJobSubprocessEnv injects traceparent/tracestate into a subprocess env." }, + { name: "scheduler & cron spans", type: "span / event", desc: "createSchedulerStartSpan, createScheduleJobSpan, startSchedulerTickSpan, traceSchedulerTick, recordCronJobRun — cron/schedule dispatch traced end to end." }, + { name: "traceQueue(queue, options) / TracedQueue", type: "→ TracedQueue", desc: "Wrap a MessageQueue so enqueue/consume carry trace context. SSE helpers (startSSEConnection, createSSEEventSpan, traceSSEEvent) trace Server-Sent-Event streams." } + ] +}) }} + +{{ comp.apiTable({ + caption: "Traceparent / context propagation (@netscript/telemetry/context)", + rows: [ + { name: "formatTraceparent(spanContext)", type: "→ string", desc: "Serialize a span context to a W3C traceparent header value. parseTraceparent(value) parses one back to a ParsedTraceparent | null." }, + { name: "injectContext(headers, ctx?) / extractContext(headers)", type: "→ headers / → Context", desc: "Inject the active trace context into outbound PropagationHeaders, or extract a remote one from inbound headers — the core of cross-service propagation." }, + { name: "withContext(ctx, fn) / withContextAsync(ctx, fn)", type: "→ T / → Promise", desc: "Run a callback with a given OTel context active so spans created inside attach to the right parent." }, + { name: "createMessageHeaders / resolveParentContextFromHeaders", type: "→ headers / → Context", desc: "Build propagation headers for a queue message, and resolve the parent context back out on the consumer side." }, + { name: "getTraceId(ctx?) / getSpanId(ctx?)", type: "→ string?", desc: "Read the current trace/span ids — handy for correlating structured log lines with a trace." } + ] +}) }} + +## Service & RPC tracing toggle + +Services opt into trace propagation through the oRPC layer rather than by hand. The +`@netscript/telemetry/orpc` subpath ships an oRPC **tracing plugin** that opens a SERVER span +per RPC and continues any inbound `traceparent`, plus an error-handling plugin that classifies +failures. On the service builder this is the `.withRPC({ traceContext: true })` toggle (see +[Services](/services-sdk/services/)); the plugin +factory below is the underlying surface if you mount oRPC yourself. + +{{ comp.apiTable({ + caption: "oRPC tracing surface (@netscript/telemetry/orpc)", + rows: [ + { name: "createTracingPlugin(options?)", type: "→ TracingPlugin", desc: "oRPC plugin that opens a SERVER span per call and continues an inbound traceparent. Options are TracingPluginOptions." }, + { name: "createErrorHandlingPlugin(options?)", type: "→ ErrorHandlingPlugin", desc: "Classifies errors as client | server | transient and records them on the span; takes an optional ErrorClassifier and ErrorLogger." }, + { name: "createTraceContext()", type: "→ TraceContext", desc: "Build the per-call trace context the plugin threads through. addEvent / setAttributes / getTraceId / getSpanId operate on the active span." } + ] +}) }} + +{{ comp callout { type: "note", title: "Database & worker tracing toggles live next door" } }} +The Prisma query-tracing and worker-runtime tracing toggles are documented on their own hubs +to avoid duplication. Database query spans are wired through the +database client; the worker runtime's automatic job +spans are described in background jobs. This hub +owns the cross-cutting telemetry surface — config, the tracer facade, +traceparent propagation, and the oRPC plugin — that those runtimes build on. See +{{ comp.xref({ key: "cap:database", text: "Database & Prisma" }) }} for the per-query toggle. +{{ /comp }} + +## What is traced automatically + +Before you write a single instrumentation call, the worker runtime already produces a usable +trace tree in Aspire. The `@netscript/telemetry` instrumentation is wired into the dispatcher, +the executor, and the scheduler, so the moment a job runs you get spans with attributes, +durations, status, and lifecycle events — no scaffold changes required. + +{{ comp.apiTable({ + caption: "Automatic worker traces (emitted by the runtime — see /reference/telemetry/)", + rows: [ + { name: "traceJobExecution", type: "span", desc: "Wraps each job's execution with attributes, duration, status, and job.started / job.completed / job.failed / job.exception events. Emitted by the dispatcher." }, + { name: "recordJobProgress", type: "event", desc: "job.progress events carrying current / total / unit — the runtime records real progress as the job advances." }, + { name: "addJobStepEvent", type: "event", desc: "job.step.* events for each step the runtime walks through." }, + { name: "scheduler spans", type: "span", desc: "createSchedulerStartSpan / createScheduleJobSpan / startSchedulerTickSpan / recordCronJobRun — cron and schedule dispatch are traced end to end." }, + { name: "subprocess traceparent", type: "context", desc: "The dispatcher injects W3C traceparent / tracestate into the subprocess env so out-of-process job runs continue the same trace (initJobTracing / runTracedJob / createJobSubprocessEnv)." }, + { name: "task.execute", type: "span", desc: "The multi-runtime task executor wraps each task run in a task.execute span — polyglot and TS tasks alike show up in the trace tree." } + ] +}) }} + +## Instrument a handler + +The two tabs below show the two emission paths a developer touches: structured logging +through the framework logger, and **custom** spans inside a job handler. For custom spans, +call the `@netscript/telemetry` helpers (`traceJobExecution`, `withChildSpan`, +`recordJobProgress`) directly — they are the real, supported surface. Read the +callout under the tabs to understand why you reach for the telemetry package rather than the +scaffold's `createJobTools(ctx)` trace helpers. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Structured logging", + lang: "ts", + code: "// netscript.config.ts — declare the logging contract for the whole workspace.\nimport { defineConfig } from '@netscript/config';\n\nexport default defineConfig({\n name: 'my-app',\n version: '1.0.0',\n // level: 'debug' | 'info' | 'warn' | 'error'; format: 'text' | 'json'\n logging: { level: 'info', format: 'text' },\n plugins: ['./plugins/workers/mod.ts', './plugins/sagas/mod.ts'],\n});\n\n// Inside a job handler, the logger comes from the job tools (console-backed today,\n// and surfaced under the resource's Console logs view in the Aspire dashboard).\n// import { createJobTools } from './job-tools.ts';\nconst emit = (log) => {\n log.info('user provisioned', { userId: 'u_123', source: 'scaffold' });\n log.warn('rate limit approaching', { remaining: 4 });\n};" + }, + { + label: "Custom spans via @netscript/telemetry", + lang: "ts", + code: "// plugins/workers/jobs/health-check.ts — real custom spans inside a handler.\n// The dispatcher already wraps this job in traceJobExecution automatically;\n// withChildSpan + recordJobProgress let you add detail under that parent span.\nimport { createSuccessResult, createFailureResult, defineJobHandler } from '@netscript/plugin-workers-core';\nimport { withChildSpan, recordJobProgress } from '@netscript/telemetry/instrumentation';\n\nconst handler = defineJobHandler(async (ctx) => {\n const { log } = ctx;\n log.info('Starting workers plugin health check');\n\n // Real progress event — 3rd arg is a UNIT label, not a message.\n recordJobProgress(1, 2, 'steps');\n\n // withChildSpan opens a real child span under the job-execution span.\n const envCheck = await withChildSpan('check.environment', async (span) => {\n span.setAttribute('check.name', 'environment');\n return { ok: true };\n });\n\n recordJobProgress(2, 2, 'steps');\n if (!envCheck.ok) return createFailureResult('environment check failed');\n return createSuccessResult({ status: 'healthy' });\n});\n\nexport default Object.assign(handler, { id: 'workers-plugin-health-check' as const });" + } +] }) }} + +{{ comp callout { type: "warning", title: "Known gap: the scaffold createJobTools(ctx) trace/progress helpers are no-op stubs" } }} +Job dispatch and execution are instrumented with real OTel spans — traces +show up in Aspire automatically (traceJobExecution, recordJobProgress, +scheduler spans, and subprocess traceparent propagation are all live in the runtime). The one +remaining gap is narrow and specific: the trace.{addEvent,withChildSpan,recordProgress} +and progress(...) helpers returned by the scaffold's createJobTools(ctx) +are currently no-op stubs in the generated copy. log writes to the +console; those particular trace/progress helpers do nothing. This is a known, tracked +limitation with a fix planned (debt workers-scaffold-job-tools-noop), not a +permanent design choice. Workaround: for custom spans and progress, call the +@netscript/telemetry helpers directly (as in the tab above) rather than the scaffold +createJobTools trace surface — those are real today. See +Observability for the full framework-vs-scaffold map. + +{{ /comp }} + +## Endpoints & ports + +Telemetry has no single service of its own — it is emitted by every runtime and aggregated +by Aspire. These are the real addresses you interact with, validated by the CLI E2E suite. + +{{ comp.apiTable({ + caption: "Observability surfaces (link to /reference/telemetry/ and /reference/logger/ for the generated APIs)", + rows: [ + { name: "https://localhost:18888", type: "dashboard", desc: "Aspire dashboard — traces, structured logs, metrics, and resource state for the whole app graph. Auth token printed by `aspire start`." }, + { name: "http://localhost:4318", type: "OTLP/HTTP", desc: "OTLP ingest endpoint the AppHost configures (aspire.config.json https profile). Runtimes export spans and logs here; the dashboard reads them back. This is the seam you point at a hosted backend." }, + { name: "GET :8091/health", type: "liveness", desc: "Workers API health probe — reported as resource health in the dashboard." }, + { name: "GET :8092/health/live", type: "liveness", desc: "Sagas API liveness route." }, + { name: "GET :8093/health", type: "liveness", desc: "Triggers API health probe (Hono service)." }, + { name: ":8094", type: "service", desc: "Auth API (auth-api) — its request spans propagate trace context like any other service." } + ] +}) }} + +## View it in Aspire + +With `aspire start` up, the dashboard at `https://localhost:18888` gives you four views over the +same telemetry stream — there is no separate tool to configure. + +{{ comp.apiTable({ + caption: "Aspire dashboard views (https://localhost:18888)", + rows: [ + { name: "Resources", type: "graph", desc: "Live state of every app-graph resource: postgres, redis, workers-api, workers, sagas-api, sagas, triggers-api, triggers, auth-api. Health probes drive the status colour." }, + { name: "Console logs", type: "stream", desc: "Per-resource stdout/stderr — the framework logger's text/JSON output lands here in real time." }, + { name: "Structured logs", type: "OTLP", desc: "Structured log records exported over OTLP, filterable by resource, level, and attributes." }, + { name: "Traces", type: "OTLP", desc: "Distributed traces collected from the OTLP endpoint (http://localhost:4318) — job dispatch/execution/scheduler/task spans and cross-service request spans as trace context propagates." } + ] +}) }} + +{{ comp callout { type: "tip", title: "Use this when…" } }} +Reach for the dashboard whenever you need to answer "what happened and where": +which resource is unhealthy, what a service logged for a given request, why a job failed, or how +a call fanned out across services and subprocesses. Because Aspire wires OTLP for you, the +lowest-effort observability path is to run aspire start and open +:18888 — no extra dependency, no collector to deploy. When you outgrow the local +dashboard, the same OTLP export (http://localhost:4318) is the seam you point at a +hosted backend. +{{ /comp }} + +## Production notes + +{{ comp callout { type: "warning", title: "Footguns before you ship telemetry" } }} +

❌ Trace 100% of traffic in production. The local Aspire default samples +everything — fine for dev, ruinous at scale.
+✅ Set a ratio sampler via env. Sampling is an env decision, not a code one: set +OTEL_TRACES_SAMPLER (surfaced as TelemetryConfig.sampler) to +traceidratio with a rate. Exception: keep always-on sampling for low-volume +services or a targeted debugging window.

+

❌ Fork the runtime to change telemetry backends. Editing instrumentation to +point at Grafana/Honeycomb is churn you will re-do every upgrade.
+✅ Point the exporter. The OTLP export (OTEL_EXPORTER_OTLP_ENDPOINT, +default http://localhost:4318) is the single seam — swap endpoint/protocol via env and +the instrumentation code does not change.

+

❌ Put unbounded values in span attributes. User ids, raw URLs, or full payloads +in span.setAttribute explode index cardinality on the backend.
+✅ Use stable, low-cardinality keys. Correlate to a specific record with +getTraceId() in a structured log line instead of widening the span.

+

❌ Pass a message as recordJobProgress's 3rd argument. Free text +there inflates cardinality and misreads the API.
+✅ Pass a stable unit label. 'steps' / 'items' — the +argument is a unit, not a description.

+

❌ Ship code that relies on the scaffold createJobTools(ctx) trace/progress +helpers. Those are no-op stubs today (debt workers-scaffold-job-tools-noop).
+✅ Call the @netscript/telemetry helpers directly for real spans and +progress, as in the handler tab above.

+{{ /comp }} + +## Reference + +This hub is intentionally thin — the full generated API lives in the reference. + +{{ comp.xref({ key: "ref:telemetry" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Do — Add OpenTelemetry", + body: "Task-oriented recipe: lean on the automatic job spans, add custom spans via @netscript/telemetry withChildSpan, propagate traceparent across services and subprocesses, and point the OTLP export (:4318) at the dashboard.", + href: "/how-to/add-opentelemetry/", + icon: "◆" + }, + { + title: "Understand — Observability", + body: "The mental model: how spans, structured logs, health endpoints, and Aspire traces fit together — and the precise framework-vs-scaffold map of what is real (traceJobExecution, task.execute) versus the tracked createJobTools(ctx) stub.", + href: "/explanation/observability/", + icon: "◎" + }, + { + title: "Look up — telemetry reference", + body: "The full generated deno doc API for @netscript/telemetry — tracer facade, context/traceparent helpers, the instrumentation helpers, scheduler spans, and the oRPC tracing plugin. The authority for signatures.", + href: "/reference/telemetry/", + icon: "≡" + }, + { + title: "Look up — logger reference", + body: "The full generated deno doc API for @netscript/logger — log levels, formats, and the structured-logging surface declared in netscript.config.ts.", + href: "/reference/logger/", + icon: "≡" + } +] }) }} + +{{ comp.nextPrev({ + prev: { label: "KV, queues & cron", href: "/data-persistence/kv-queues-cron/" }, + next: { label: "Fresh UI & design", href: "/web-layer/fresh-ui/" } +}) }} diff --git a/docs/site/orchestration-runtime/cli-scaffold.md b/docs/site/orchestration-runtime/cli-scaffold.md new file mode 100644 index 000000000..d87da8354 --- /dev/null +++ b/docs/site/orchestration-runtime/cli-scaffold.md @@ -0,0 +1,9 @@ +--- +layout: layouts/base.vto +title: CLI and Scaffold +templateEngine: [vento, md] +--- + +# CLI & Scaffold + +_Story page forthcoming._ diff --git a/docs/site/orchestration-runtime/runtime-config.md b/docs/site/orchestration-runtime/runtime-config.md new file mode 100644 index 000000000..da3e01a5e --- /dev/null +++ b/docs/site/orchestration-runtime/runtime-config.md @@ -0,0 +1,283 @@ +--- +layout: layouts/base.vto +title: Runtime configuration +templateEngine: [vento, md] +prev: { label: "Polyglot tasks", href: "/background-processing/polyglot-tasks/" } +next: { label: "Capabilities", href: "/capabilities/" } +--- + +# Runtime configuration + +NetScript splits configuration into two layers: a **typed project config** you author +once in `netscript.config.ts` and load at startup (`@netscript/config`), and a +**hot-reloadable runtime override layer** that operators roll out without redeploying +(`@netscript/runtime-config`). Together they give you one validated, type-safe view of how +the app is wired — defaults baked into the schema, values pulled from a config file and the +environment, and last-mile operational overrides applied at runtime. + +{{ comp.diagram({ src: "/assets/diagrams/runtime-config-resolution.svg", alt: "Configuration resolution chain: schema defaults flow into the loaded config file, environment variables override typed values via resolveEnv, and hot-reloadable runtime override files apply last to produce a frozen resolved config.", caption: "Resolution chain — schema defaults → config file (defineConfig / loadConfig) → environment (resolveEnv / getEnv) → runtime override files (loadRuntimeConfig) → the validated config the framework reads." }) }} + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for project config (@netscript/config) when you need to +declare how the app is built and wired — services, ports, databases, plugins, +paths — once, with full TypeScript inference and Zod validation. Reach for +runtime overrides (@netscript/runtime-config) when an +operator needs to toggle a feature flag, disable a job/saga/trigger, or add a task +without a redeploy. For one-off typed reads of a single environment +variable, use getEnv. See also +architecture. +{{ /comp }} + +## What it is + +Project config is the **static, schema-validated topology** of a NetScript app. You author +it with `defineConfig` (or `defineConfigAsync` when the shape depends on mode/command), +NetScript validates it against a Zod schema, and `loadConfig` / `initConfig` resolve the +authored file into a fully typed `NetScriptConfig`. Environment variables feed into this +layer through `resolveEnv` / `getEnv`, which coerce and default typed values per call. + +Runtime config is a **separate, dynamic override layer**. It lives under a `runtime/` +directory of versioned JSON files (jobs, sagas, triggers, features, tasks) selected by a +`current` version pointer. `loadRuntimeConfig` reads it into a `RuntimeConfig` snapshot; +`watchRuntimeConfig` reloads on change. Missing files produce **empty defaults** so startup +never blocks on overrides that have not been rolled out yet. The two packages are +deliberately distinct: `@netscript/config` owns the build-time contract, and +`@netscript/runtime-config` owns the hot-reloadable operational layer. + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Architecture", + body: "How project config, the workspace layout, and the plugin system fit together at startup.", + href: "/explanation/architecture/", + icon: "◎" + }, + { + title: "Do — Choose a queue provider", + body: "A config-driven decision recipe: pick and wire a queue backend through project config.", + href: "/how-to/choose-a-queue-provider/", + icon: "◆" + } +] }) }} + +## Minimal example + +Author the project config once, then load and inspect it at startup. The `runtimeConfig` +section of the project config controls where the **generated** runtime schema/output lives; +the live overrides are read separately by `@netscript/runtime-config`. + +```ts +// netscript.config.ts +import { defineConfig } from '@netscript/config'; + +export default defineConfig({ + name: 'orders', + version: '1.0.0', + databases: { + active: 'postgres', + config: [{ provider: 'postgres', schema: 'database/postgres/schema' }], + }, + services: { + api: { port: 3000 }, + }, + plugins: ['@netscript/plugin-workers'], +}); +``` + +```ts +// src/bootstrap.ts — load project config + apply runtime overrides +import { initConfig, getConfig } from '@netscript/config'; +import { isFeatureEnabled, loadRuntimeConfig } from '@netscript/runtime-config'; + +// 1. Project config: validated once, then read synchronously anywhere via getConfig(). +await initConfig(); +const config = getConfig(); + +// 2. Runtime overrides: hot-reloadable JSON the operator controls (empty if unset). +const runtime = await loadRuntimeConfig(); +const rolloutOn = isFeatureEnabled(runtime, 'worker-rollout', false); + +console.log(config.name, config.services?.api?.port, rolloutOn); +``` + +## Key types first + +The project config you author is `NetScriptConfigInput`; what `loadConfig` / `initConfig` +return is the fully validated `NetScriptConfig`. These are the load-bearing fields. + +{{ comp.apiTable({ + caption: "NetScriptConfig — the validated project config (returned by loadConfig / initConfig / getConfig)", + rows: [ + { name: "name", type: "string (required)", desc: "Project name." }, + { name: "version", type: "string (required)", desc: "Project version." }, + { name: "paths", type: "PathsConfig (required)", desc: "Workspace path conventions (services, packages, apps, workers, sagas, triggers, plugins, contracts, database, tasks, deploy)." }, + { name: "databases", type: "DatabasesConfig (required)", desc: "{ active?: provider, config: DatabaseConfig[] } — configured databases and the active selector." }, + { name: "plugins", type: "string[] (required)", desc: "Enabled plugin package names or specifiers." }, + { name: "services", type: "Record?", desc: "Service configuration by service name." }, + { name: "apps", type: "Record?", desc: "Frontend application configuration by app name." }, + { name: "logging", type: "LoggingConfig?", desc: "{ level, format, timestamps, colors? } logging behavior." }, + { name: "aspire", type: "AspireConfig?", desc: "{ appHost, dashboardPort } Aspire orchestration settings." }, + { name: "sagas / triggers / gateway / sdk / deploy", type: "section objects?", desc: "Optional per-capability config sections." }, + { name: "runtimeConfig", type: "RuntimeConfigSection?", desc: "Runtime schema/config OUTPUT settings (where generated runtime schema is written) — not the live overrides." } + ] +}) }} + +The runtime override snapshot is a separate type — `RuntimeConfig` — returned by +`loadRuntimeConfig()`. Each field is an array loaded from one topic directory. + +{{ comp.apiTable({ + caption: "RuntimeConfig — the hot-reloadable override snapshot (returned by loadRuntimeConfig)", + rows: [ + { name: "jobs", type: "JobOverride[]", desc: "Job overrides from runtime/jobs/*.json (enabled, schedule, timeout, maxRetries, timezone, concurrency)." }, + { name: "sagas", type: "SagaOverride[]", desc: "Saga overrides from runtime/sagas/*.json (enabled, timeout, maxRetries, compensationTimeout)." }, + { name: "triggers", type: "TriggerOverride[]", desc: "Trigger overrides from runtime/triggers/*.json (enabled, paths)." }, + { name: "features", type: "FeatureFlag[]", desc: "Feature flags from runtime/features/*.json (id, enabled, description?, rolloutPercentage?)." }, + { name: "tasks", type: "RuntimeTask[]", desc: "Runtime task definitions from runtime/tasks/*.json (id, name, runtime, entrypoint, enabled?, timeout?, schedule?)." } + ] +}) }} + +## Resolution & precedence + +There is no single monolithic merge function — resolution happens across the two packages, +each with its own precedence. The order below is the conceptual chain the diagram depicts. + +{{ comp.apiTable({ + caption: "How a value is resolved (lowest precedence first)", + rows: [ + { name: "1. Schema defaults", type: "@netscript/config", desc: "Zod schema fills omitted fields when defineConfig validates the authored input into NetScriptConfig." }, + { name: "2. Config file", type: "loadConfig / initConfig", desc: "File search order: options.configFile → netscript.config.ts → .js → .mjs. cwd resolves from options.cwd → NETSCRIPT_PROJECT_ROOT → Deno.cwd()." }, + { name: "3. Environment", type: "resolveEnv / getEnv", desc: "Per-variable: read the env var (def.env overrides the name) and coerce by type (string|number|boolean|json); if absent, use default; if required and absent with no default, throw." }, + { name: "4. Runtime overrides", type: "loadRuntimeConfig", desc: "Hot-reloadable JSON under the runtime dir, keyed by the 'current' version pointer; applied per-id (job/saga/trigger/feature/task). Missing files → empty defaults." } + ] +}) }} + +{{ comp.apiTable({ + caption: "@netscript/config — primary functions", + rows: [ + { name: "defineConfig(input)", type: "(NetScriptConfigInput) => NetScriptConfig", desc: "Type-safe, validated config definition for netscript.config.ts. The 80% path." }, + { name: "defineConfigAsync(fn)", type: "((env: ConfigEnv) => …) => () => Promise", desc: "Async/env-aware config when the shape depends on { mode, command }." }, + { name: "loadConfig(options?)", type: "(LoadConfigOptions?) => Promise", desc: "Find, import, and validate the config file. Options: { cwd?, configFile? }." }, + { name: "initConfig(options?)", type: "(LoadConfigOptions?) => Promise", desc: "Load once and cache; call at startup. Then read synchronously via getConfig()." }, + { name: "getConfig()", type: "() => NetScriptConfig", desc: "Synchronous access to the cached config; throws if initConfig() has not run." }, + { name: "isConfigLoaded() / clearConfigCache()", type: "() => boolean / () => void", desc: "Inspect or reset the config cache (clearConfigCache is for tests/reload)." }, + { name: "resolveEnv(schema) / getEnv(name, opts?)", type: "typed env readers", desc: "Coerce env vars (string|number|boolean|json) with defaults; getEnv reads one variable." }, + { name: "getMode() / isDev() / isProd() / isTest() / hasEnv(name)", type: "env helpers", desc: "Current mode (DENO_ENV/NODE_ENV) and presence checks." }, + { name: "discoverWorkspace(root?) / findWorkspaceRoot(dir) / findMember(ws, name)", type: "workspace", desc: "Classify and locate Deno workspace members." }, + { name: "inspectConfig(target)", type: "(Partial | string) => InspectionReport", desc: "JSON-stable diagnostic report for CLI rendering." } + ] +}) }} + +{{ comp.apiTable({ + caption: "@netscript/runtime-config — primary functions", + rows: [ + { name: "loadRuntimeConfig()", type: "() => Promise", desc: "Load overrides from the runtime dir via the 'current' pointer; empty defaults when files are missing." }, + { name: "watchRuntimeConfig(onChange, opts?)", type: "(cb, { signal?, prefix? }) => void", desc: "Watch the runtime config files and invoke onChange after debounced reloads." }, + { name: "isFeatureEnabled(config, flagId, defaultValue)", type: "(RuntimeConfig, string, boolean) => boolean", desc: "Read a feature flag, falling back when the flag is absent." }, + { name: "getJobOverride / getSagaOverride / getTriggerOverride", type: "(RuntimeConfig, id) => …Override | undefined", desc: "Look up a single override by id." }, + { name: "getRuntimeTask(config, taskId)", type: "(RuntimeConfig, string) => RuntimeTask | undefined", desc: "Get a runtime task definition by id." }, + { name: "summarizeRuntimeConfig(config, prefix?)", type: "(RuntimeConfig, string?) => RuntimeConfigSummary", desc: "Structured summary of active overrides (disabled jobs/sagas/triggers/features, message lines) for caller-owned presentation." }, + { name: "RUNTIME_CONFIG_TOPICS", type: "('jobs'|'sagas'|'triggers'|'features'|'tasks')[]", desc: "The topic names backed by versioned JSON files." } + ] +}) }} + +## Runtime override layout + +The runtime override directory is resolved from the environment, then read through the +`current` pointer that names the active version of each topic file. + +{{ comp.fileTree({ tree: { + name: "runtime", + comment: "NETSCRIPT_RUNTIME_CONFIG_DIR (else NETSCRIPT_TASKS_DIR's parent, else ./runtime)", + children: [ + { name: "current", comment: "version pointer — JSON { version, jobs, sagas, … } or plain 'x.y.z'" }, + { name: "features", children: [ { name: "v1.0.0.json", comment: "FeatureFlag[]" } ] }, + { name: "jobs", children: [ { name: "v1.0.0.json", comment: "JobOverride[]" } ] }, + { name: "sagas", children: [ { name: "v1.0.0.json", comment: "SagaOverride[]" } ] }, + { name: "triggers", children: [ { name: "v1.0.0.json", comment: "TriggerOverride[]" } ] }, + { name: "tasks", children: [ { name: "v1.0.0.json", comment: "RuntimeTask[]" } ] } + ] +} }) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Read a feature flag at startup", + lang: "ts", + code: "// src/feature-gate.ts\nimport { isFeatureEnabled, loadRuntimeConfig } from '@netscript/runtime-config';\n\nconst runtime = await loadRuntimeConfig();\n\n// Third arg is the fallback when the flag file/entry is absent.\nif (isFeatureEnabled(runtime, 'worker-rollout', false)) {\n // run the gated path\n}" + }, + { + label: "Hot-reload on change", + lang: "ts", + code: "// src/runtime-watch.ts\nimport { summarizeRuntimeConfig, watchRuntimeConfig } from '@netscript/runtime-config';\n\nconst controller = new AbortController();\n\nwatchRuntimeConfig(async (config) => {\n const summary = summarizeRuntimeConfig(config);\n // re-apply overrides; summary.messages is presentation-ready\n console.log(summary.messages.join('\\n'));\n}, { signal: controller.signal });" + }, + { + label: "Typed env reads", + lang: "ts", + code: "// src/env.ts\nimport { getEnv, resolveEnv } from '@netscript/config';\n\nconst port = getEnv('PORT', { type: 'number', default: 3000 });\n\nconst env = resolveEnv({\n DATABASE_URL: { env: 'DB_URL', required: true },\n DEBUG: { type: 'boolean', default: false },\n});" + } +] }) }} + +## Production notes + +{{ comp callout { type: "warning", title: "Two packages, two precedence rules — not one merge" } }} +There is no single function that merges defaults, file, env, and runtime +overrides into one object. @netscript/config validates and loads the +project config; per-variable env coercion is explicit via resolveEnv / +getEnv; and @netscript/runtime-config is a separate +hot-reloadable layer your code reads with loadRuntimeConfig and applies per id. +Don't expect a runtime override file to silently change your NetScriptConfig — +you read and apply the override explicitly. +{{ /comp }} + +{{ comp callout { type: "important", title: "Runtime dir resolution & missing files" } }} +The runtime override directory is resolved as +NETSCRIPT_RUNTIME_CONFIG_DIR → else the parent of +NETSCRIPT_TASKS_DIR → else ./runtime. A missing directory, +pointer, or topic file is not an error — loadRuntimeConfig +returns empty defaults so startup proceeds while operators roll overrides out +independently. Make sure your deployment sets the env var (or the working directory) you +expect, or you may silently read empty overrides. +{{ /comp }} + +{{ comp callout { type: "note", title: "getConfig() must follow initConfig()" } }} +getConfig() returns the cached config and throws if +initConfig() has not run. Call initConfig() once at startup, then +read synchronously anywhere. In tests, use clearConfigCache() to force a +reload between cases. +{{ /comp }} + +## Reference → + +This hub is intentionally thin — the full generated API surface for both packages lives in +the reference. + +{{ comp.xref({ key: "ref:runtime-config" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — @netscript/runtime-config", + body: "loadRuntimeConfig, watchRuntimeConfig, the override types, and the topic helpers.", + href: "/reference/runtime-config/", + icon: "≡" + }, + { + title: "Look up — @netscript/config", + body: "defineConfig, loadConfig/initConfig, resolveEnv/getEnv, the NetScriptConfig schema, and workspace helpers.", + href: "/reference/config/", + icon: "≡" + }, + { + title: "Understand — Architecture", + body: "Where project config, the workspace, and the plugin system sit in the framework.", + href: "/explanation/architecture/", + icon: "◎" + }, + { + title: "Browse — Capabilities", + body: "All NetScript capabilities: services, jobs, sagas, triggers, database, and more.", + href: "/capabilities/", + icon: "◆" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Polyglot tasks", href: "/background-processing/polyglot-tasks/" }, next: { label: "Capabilities", href: "/capabilities/" } }) }} diff --git a/docs/site/services-sdk/sdk.md b/docs/site/services-sdk/sdk.md new file mode 100644 index 000000000..fcc3edcd0 --- /dev/null +++ b/docs/site/services-sdk/sdk.md @@ -0,0 +1,228 @@ +--- +layout: layouts/base.vto +title: Typed SDK & client +templateEngine: [vento, md] +prev: { label: "Fresh meta-framework", href: "/web-layer/" } +next: { label: "Polyglot tasks", href: "/background-processing/polyglot-tasks/" } +--- + +# Typed SDK & client + +The `@netscript/sdk` is the **typed client and data layer** for NetScript: it turns an +[`@orpc/contract`](/explanation/contracts/) into a discovered, type-safe service client, +wraps each contract action in a **cache-first query factory** (KV-backed SWR), and bridges +those into [TanStack Query](https://tanstack.com/query) for islands. The contract object is +the single source of truth — the *same* object the [service](/services-sdk/services/) +implements is the one the client imports, so caller and server **cannot drift**. +{{ comp.badge({ status: "alpha" }) }} + +{{ comp.diagram({ + src: "/assets/diagrams/sdk-data-flow.svg", + alt: "A contract object flows into a generated typed client, then a cache-first query factory backed by Deno KV, then into a Fresh island; the same contract object is imported by the oRPC service so the two ends cannot drift.", + caption: "One contract → typed client → KV cache-first query → island. The service imports the same contract, so a renamed field is a compile error on both sides." +}) }} + +## What it is + +The SDK is layered. **L1** is the typed client: `createServiceClient()` resolves a +service URL from [Aspire service discovery](/explanation/aspire/) and returns a callable +object whose method signatures are inferred from the contract. **L2** wraps each contract +action in a query factory (`createQueryFactories`) that runs through a shared KV-backed cache +provider and exposes `queryOptions`/`mutationOptions`/`clientKey`/`key`/`getCachedEntry` per +action. **L3** is the `defineServices()` preset that builds clients, server query factories, +and frontend query utils from one contract map. You consume the layer you need and can drop +down without rewiring. See [Contracts](/explanation/contracts/) for the type-flow theory. + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for the SDK when an app or a service needs to call a typed oRPC service +and render its data with cache-first, server-hydrated queries. Use the bare +createServiceClient for service-to-service +RPC; add createQueryFactories when you want KV-backed SWR and TanStack option +helpers; reach for Fresh when an island must +hydrate that same query key on the client. +{{ /comp }} + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Build a service + client", + body: "Track D walks contract → typed client → query factory → definePage layer → island, end to end.", + href: "/tutorials/storefront/02-catalog-service/", + icon: "→" + }, + { + title: "Do — Discover a service", + body: "Task recipe: resolve a service URL from Aspire env and stand up a typed client for it.", + href: "/how-to/discover-services/", + icon: "◆" + } +] }) }} + +## Minimal example + +A single `lib/api-clients.ts` is the spine of the data layer: build the typed client from a +contract, then derive a query factory from `{ contract, client }`. Every consumer — server +loader or island — imports from here. + +```ts +// apps/playground/lib/api-clients.ts +import { createServiceClient } from '@netscript/sdk/client'; +import { createQueryFactories } from '@netscript/sdk/query'; +import { ordersContract } from '@contracts'; + +// L1 — typed client. `serviceName` resolves a URL via Aspire discovery. +export const ordersClient = createServiceClient({ + contract: ordersContract, + serviceName: 'orders', +}); + +// L2 — cache-first query factory bound to that client (KV-backed SWR). +export const api = createQueryFactories({ + orders: { contract: ordersContract, client: ordersClient }, +}); + +// Direct typed call: `.list()` is fully inferred from the contract. +const recent = await ordersClient.list({ limit: 10 }); +``` + +{{ comp callout { type: "note", title: "Plugin services add a routerName" } }} +A first-party plugin API (workers, sagas, triggers) mounts its router under a named segment, so +its client needs routerName in addition to serviceName: +createServiceClient<typeof workersContract>({ contract, serviceName: 'workers-api', routerName: 'workers' }). +Plain workspace services omit it. +{{ /comp }} + +## Key types first + +The query factory is the heart of the data layer. Each contract action becomes an +**`ActionMethod`** carrying the full cache-first surface — keys, prefetch, cached reads, and +TanStack option helpers. These are the methods you call from a `definePage` loader and an +island; reading them first explains the rest of the page. + +{{ comp.apiTable({ + caption: "ActionMethod — per-action query helpers (api.orders.list.*)", + rows: [ + { name: "queryOptions(props, options?)", type: "(input, ActionQueryOptions?) => QueryOptions", desc: "TanStack queryOptions with a typed queryKey and queryFn derived from the contract. The primary helper a loader/island passes to useQuery." }, + { name: "mutationOptions(options?)", type: "(ActionMutationOptions?) => MutationOptions", desc: "TanStack mutationOptions with a typed mutationKey and mutationFn for writes (e.g. updateStatus)." }, + { name: "key(props)", type: "(input) => readonly [string, action, string]", desc: "Canonical SERVER cache key [resource, action, serializedInput] used by the KV cache." }, + { name: "clientKey(props?)", type: "(input?) => readonly unknown[]", desc: "CLIENT-side TanStack query key, prefix-matchable for invalidateQueries. clientKey() vs key() is the server↔client split." }, + { name: "getCachedEntry(props)", type: "(input) => Promise | null>", desc: "Read cached data WITH its cache timestamp (cachedAt) — the SWR primitive a layer loader uses to decide stale-reload." }, + { name: "getCachedData(props)", type: "(input) => Promise", desc: "Read cached data only, without fetching." }, + { name: "prefetch(props, options?)", type: "(input, QueryParams?) => void", desc: "Warm the cache in the background (fire-and-forget)." }, + { name: "invalidate()", type: "() => Promise", desc: "Invalidate all cached queries for this action." } + ] +}) }} + +## Constructing the client & factory + +The two L1/L2 constructors and their options. `createServiceClient` is the only call that +touches discovery; `createQueryFactories` is pure wiring over `{ contract, client }`. + +{{ comp.apiTable({ + caption: "createServiceClient(options) — CreateServiceClientOptions", + rows: [ + { name: "contract", type: "TContract — required", desc: "The @orpc/contract object. Drives both client typing AND HTTP method inference." }, + { name: "serviceName", type: "string — required", desc: "Service name registered in Aspire / NetScript config. Resolved to a URL via discovery (services__{name}__{protocol}__{index}; browser uses the VITE_ mirror)." }, + { name: "routerName", type: "string?", desc: "Router-name segment for URL path construction. Required for plugin API services, omitted for plain services." }, + { name: "protocol", type: "'http' | 'https'?", desc: "Resolved protocol for service discovery." }, + { name: "apiPath / apiVersion", type: "string?", desc: "Base RPC path and API version segment overrides." }, + { name: "propagateTraceContext", type: "boolean?", desc: "Auto-propagate W3C traceparent/tracestate headers on each call." } + ] +}) }} + +The L3 alternative builds all three layers from one map: +`defineServices({ orders: { contract, serviceName: 'orders' } })` returns +`{ clients, queryFactories, queryUtils }` — the same L2 values, just composed in one call. + +## Service discovery & query client + +`serviceName` is resolved at call time from Aspire-injected environment variables. The +`@netscript/sdk/discovery` subpath exposes the resolvers directly when you need a raw URL, and +`@netscript/sdk/query-client` provides the browser-side TanStack glue. + +{{ comp.apiTable({ + caption: "Discovery & query-client helpers", + rows: [ + { name: "getServiceUrl(name, protocol, index)", type: "@netscript/sdk/discovery", desc: "Resolve a service URL from Aspire browser-VITE or server services__* env. This is how serviceName resolves." }, + { name: "getServiceInfo(name) / isServiceAvailable(name) / getAllServices()", type: "@netscript/sdk/discovery", desc: "Inspect a service's endpoints, check availability, or list all server-side Aspire service names (topology)." }, + { name: "createNetScriptQueryClient(options)", type: "@netscript/sdk/query-client", desc: "A TanStack QueryClient with server-first defaults: staleTime 30s, gcTime 300s, refetchOnWindowFocus false, retry 1." }, + { name: "createServiceQueryUtils()", type: "@netscript/sdk/query-client", desc: "Bridge a typed SDK client into oRPC/TanStack frontend query utilities." }, + { name: "bridgeInvalidation(resource, action?)", type: "@netscript/sdk/query-client", desc: "Build a client-side invalidation filter ({ queryKey }) for queryClient.invalidateQueries()." }, + { name: "toClientKeyPrefix(resource, action?)", type: "@netscript/sdk/query-client", desc: "Map a server resource/action to a prefix-matchable client query key, e.g. ['orders','list']." }, + { name: "cacheQuery.setCachedData(key, data, ttl)", type: "@netscript/sdk/cache", desc: "Server-only: fire-and-forget pre-warm of an entity into the KV cache. Importing /cache auto-registers the shared provider." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Server loader (definePage layer)", + lang: "ts", + code: "// routes/(dashboard)/orders/(_loaders)/orders-list.ts\nimport { api } from '@/lib/api-clients.ts';\n\n// Cache-first: read the KV entry (with cachedAt) for SWR; fall back to a fetch.\nexport const loadOrders = async () => {\n const entry = await api.orders.list.getCachedEntry({ limit: 20 });\n if (entry) return entry; // serve cached; SDK reloads stale in the background\n return { data: await api.orders.list.queryOptions({ limit: 20 }).queryFn(), cachedAt: Date.now() };\n};" + }, + { + label: "Island (TanStack hydration)", + lang: "tsx", + code: "// orders/(_islands)/OrdersQueryIsland.tsx\nimport { useQuery, useQueryClient } from '@netscript/fresh/query';\nimport { api } from '@/lib/api-clients.ts';\n\n// Same contract action → same query key as the server loader, so the island\n// hydrates from server state instead of refetching on mount.\nconst OrdersList = () => {\n const qc = useQueryClient();\n const orders = useQuery(api.orders.list.queryOptions({ limit: 20 }));\n // invalidate by prefix after a write:\n const refresh = () => qc.invalidateQueries({ queryKey: api.orders.list.clientKey() });\n return null; // render orders.data\n};" + }, + { + label: "Safe error narrowing", + lang: "ts", + code: "// services/orders/src/routers/v1.ts — service-to-service call\nimport { safe, isDefinedError } from '@netscript/sdk/client';\nimport { usersClient } from '@/lib/api-clients.ts';\n\nconst [error, user, isDefined] = await safe(usersClient.getById({ id }));\nif (error) {\n // narrow to a typed, contract-declared error\n if (isDefinedError(error)) return { code: error.code, status: error.status };\n throw error;\n}" + } +] }) }} + +## Production notes + +{{ comp callout { type: "important", title: "/cache and /discovery are server-only" } }} +@netscript/sdk/cache touches Deno KV and @netscript/sdk/discovery +reads services__* env on the server — never import them into island/browser code. +Browsers resolve service URLs through the VITE_-mirrored keys +that Aspire injects, and the query factory's cache reads happen on the server loader; the +island receives hydrated state and a clientKey() for invalidation only. +{{ /comp }} + +{{ comp callout { type: "warning", title: "clientKey() vs key() — don't cross the streams" } }} +key(props) is the server KV cache key +([resource, action, serializedInput]); clientKey(props?) is the +client TanStack key (prefix-matchable). Invalidate on the client with +clientKey() / toClientKeyPrefix(...) — passing a server +key() to invalidateQueries will silently match nothing. NetScript is in +alpha; subpath barrels are stable but signatures may still move. +{{ /comp }} + +## Reference → + +This hub is intentionally thin — the full generated API (client, query, query-client, cache, +collections, discovery, streams, telemetry, ports) lives in the reference. + +{{ comp.xref({ key: "ref:sdk" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — @netscript/sdk reference", + body: "Generated API for every subpath: createServiceClient, createQueryFactories, defineServices, discovery, and the cache engine.", + href: "/reference/sdk/", + icon: "≡" + }, + { + title: "Understand — Contracts", + body: "The oRPC contract → implement → handler → typed client → query → island type flow that makes drift a compile error.", + href: "/explanation/contracts/", + icon: "◎" + }, + { + title: "Build — Services & contracts", + body: "The server side of the contract: defineService, the fluent builder, OpenAPI/Scalar, and the auth seam.", + href: "/services-sdk/services/", + icon: "◆" + }, + { + title: "Render — Fresh meta-framework", + body: "How a definePage layer + island consume these query factories with cache-first hydration.", + href: "/web-layer/", + icon: "→" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Fresh meta-framework", href: "/web-layer/" }, next: { label: "Polyglot tasks", href: "/background-processing/polyglot-tasks/" } }) }} diff --git a/docs/site/services-sdk/services.md b/docs/site/services-sdk/services.md new file mode 100644 index 000000000..994747859 --- /dev/null +++ b/docs/site/services-sdk/services.md @@ -0,0 +1,452 @@ +--- +layout: layouts/base.vto +title: Services & contracts +templateEngine: [vento, md] +prev: { label: "Capabilities", href: "/capabilities/" } +next: { label: "Background jobs", href: "/background-processing/workers/" } +--- + +# Services & contracts + +A NetScript **service** is a typed HTTP runtime that *implements* an +[`@orpc/contract`](/explanation/contracts/) definition. You author the contract once +(route + zod input/output), `implement()` it, bind `.handler()`s, and serve the resulting +router on a Hono + oRPC runtime. The contract object is the single source of truth: the +same object a typed client imports is the one the server implements, so caller and server +**cannot drift**. The example `users` service answers on port **3001** with both an +OpenAPI surface (`/api/v1/users/*`) and a typed oRPC endpoint (`/api/rpc/*`). + +{{ comp.diagram({ + src: "/assets/diagrams/request-lifecycle.svg", + alt: "A browser request flows into the service router, through middleware to the matched contract handler, optionally to the database, and back as a typed response.", + caption: "Request lifecycle: browser → service router → middleware → contract handler → database → typed response." +}) }} + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for a service when you need a synchronous, request/response API with +a versioned, type-locked contract shared between front end and back end — a CRUD surface, +an internal RPC boundary, or a public REST/OpenAPI endpoint. For fire-and-forget +work use a background job; for +message-driven, long-running orchestration use a +durable saga; for inbound HTTP that kicks +off work use a trigger; for +signed-in identity and sessions compose the +auth plugin. +{{ /comp }} + +## What it is + +A service is the **Layer 3 preset** (`defineService`) over a three-layer package. Layer 1 is +small primitives — health, error, RPC, OpenAPI, and Scalar docs handlers you can mount in any +Hono app. Layer 2 is `createService()`, a fluent builder that materializes a mountable +`ServiceApp` (via `build()`) or starts a Deno listener (via `serve()`). Layer 3 is +`defineService()`, the curated one-call preset used by generated service entrypoints. Every +layer takes the oRPC router as its input, and the same contract the router implements is the +one a typed client imports — so a renamed field is a compile error on both sides. The full +type story is in [Contracts](/explanation/contracts/). + +## Learn → / Do → + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Build a service", + body: "Track A 02: author a contract, stand up the users service on :3001, call it from a typed client.", + href: "/tutorials/storefront/02-catalog-service/", + icon: "→" + }, + { + title: "Do — Expose OpenAPI & Scalar", + body: "Recipe: turn on the generated OpenAPI spec and the Scalar docs UI for an existing service.", + href: "/how-to/expose-openapi-scalar/", + icon: "◆" + }, + { + title: "Do — Graceful shutdown", + body: "Recipe: drain in-flight requests and run teardown hooks on SIGINT/SIGTERM before the process exits.", + href: "/how-to/graceful-shutdown/", + icon: "◆" + } +] }) }} + +## The contract is the source of truth + +Before there is a service there is a contract. A contract is plain `@orpc/contract` +routes whose inputs and outputs are zod schemas, collected into an object and passed to +`implement()` from `@orpc/server`. `implement()` returns a `.handler()`-bindable object +the service router consumes. The example workspace versions its contracts under +`contracts/versions/v1/` and re-exports them as `@/contracts` — so a contract +bump is an explicit, reviewable version directory, never an accidental break. + +```ts +// contracts/versions/v1/users.contract.ts +import { z } from 'zod'; +import { oc } from '@orpc/contract'; +import { implement } from '@orpc/server'; + +export const UsersContractV1 = { + health: oc.route({ method: 'GET' }).input(z.object({}).optional()).output(UsersHealthSchemaV1), + list: oc.route({ method: 'POST' }).input(UsersListInputSchemaV1).output(UsersListResponseSchemaV1), + updateStatus: oc.route({ method: 'POST' }).input(UsersUpdateStatusInputSchemaV1).output(UsersUpdateStatusResponseSchemaV1), +}; + +// `implement()` produces the .handler()-bindable object the service router consumes. +export const UsersV1 = implement(UsersContractV1); +``` + +The chain end-to-end — contract → `implement()` → `.handler()` → typed client → query → +island — is laid out in [Contracts](/explanation/contracts/). The key property: the +client imports the *same* contract object, so a renamed field or a changed output shape is +a **compile error in both the handler and the caller** before it can ship. + +## Headline API: two ways to construct a service + +NetScript ships **two** service-construction APIs, and the example project uses both. +Workspace services use the one-call `defineService(router, options)` — the right default +for the 80% case. Plugin API services (workers, sagas, auth) use the fluent +`createService(router, options).with*().serve()` builder when they need to add CORS, +OpenAPI, a database client, auth middleware, or custom context step by step. Both stand up +the same Hono + oRPC runtime; `defineService` is a curated preset over the same builder. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Simple — defineService (one-shot)", + lang: "ts", + code: "// services/users/src/main.ts\nimport { defineService } from '@netscript/service';\nimport { router } from './router.ts';\n\n// One call wires CORS, request logging, OpenAPI, RPC, and health endpoints.\nawait defineService(router, {\n name: 'users',\n version: '1.0.0',\n port: parseInt(Deno.env.get('PORT') || '3001'),\n openapi: { title: 'Users API', description: 'users service' },\n debug: true,\n});" + }, + { + label: "Advanced — createService().serve() (fluent)", + lang: "ts", + code: "// plugins/workers/services/src/main.ts — step-by-step composition\nimport { createService } from '@netscript/service';\nimport { router } from './router.ts';\n\nawait createService(router, { name: 'workers', version: '1.0.0', port: 8091 })\n .withCors()\n .withLogger()\n .withOpenAPI({ title: 'Workers API' })\n .withDocs()\n .withDatabase(dbClient)\n .withContext(() => ({ workers: runtime }))\n .withRPC({ traceContext: true })\n .withHealth()\n .withServiceInfo()\n .onStartup(async () => {/* seed, warm caches */})\n .serve();" + } +] }) }} + +### `defineService(router, options)` — the preset options + +`DefineServiceOptions` extends the base `ServiceConfig` (`name`, `version`, `port`) and adds +the preset-only keys below. These are the **complete** option keys confirmed against the +package surface — nothing is omitted. + +{{ comp.apiTable({ + caption: "DefineServiceOptions (extends ServiceConfig)", + rows: [ + { name: "name", type: "string (required)", desc: "Service name used for logging, telemetry, and health-check labels." }, + { name: "version", type: "string?", desc: "Service version (e.g. '1.0.0'); surfaced on /health and the OpenAPI spec." }, + { name: "port", type: "number?", desc: "Default listener port if serve() is not passed an explicit port. The generated entrypoint reads Deno.env.get('PORT') || '3001'." }, + { name: "db", type: "DbContext?", desc: "Database context injected as context.db. Accepts a single Prisma client (with $queryRaw) or a multi-db record like { netscript, mdb, prosco, prev }; the first value exposing $queryRaw is auto-wired as the /health and /health/ready probe client." }, + { name: "openapi", type: "{ title?; description? }?", desc: "Turns on the generated OpenAPI spec endpoint and the Scalar docs UI with this title/description." }, + { name: "debug", type: "boolean?", desc: "Enables verbose oRPC logging. Defaults to the NETSCRIPT_DEBUG env var." }, + { name: "auth", type: "{ authn: AuthnOptions; authz?: AuthzOptions }?", desc: "Installs the authentication (and optional authorization) gate on guarded paths — the preset form of .withAuthn()/.withAuthz()." }, + { name: "tls", type: "ServiceTlsOptions?", desc: "Opt-in TLS: { cert, key } as PEM strings. When set, the listener serves HTTPS and negotiates HTTP/2 via ALPN automatically. Forwarded to serve() as .serve({ tls }). See TLS & HTTP/2 below." } + ] +}) }} + +{{ comp callout { type: "note", title: "Handlers bind to the contract" } }} +A router aggregates versioned handlers — export const router = { v1: { users: { ...UsersV1, health } } } — +and each handler is bound from the implemented contract: +v1.users.list.handler(async ({ input }) => { /* return records */ }). In the +scaffold's Step 5 the users handlers return seeded +in-memory records (no database yet) — the front-end ↔ contract proof. Wiring +Prisma comes later; see Database. +{{ /comp }} + +## Endpoints & ports + +A `defineService` runtime exposes its OpenAPI routes, a typed oRPC endpoint mounted under +`/api/rpc/*`, and a health check. The example `users` service is reachable once +`aspire start` is up (Aspire provisions Postgres/Redis first) or when you run it directly +with `deno task --cwd services/users dev`. + +{{ comp.apiTable({ + caption: "Users service surface (port 3001)", + rows: [ + { name: "/api/v1/users/*", type: "HTTP/OpenAPI", desc: "REST surface generated from the contract (list, updateStatus, health)." }, + { name: "/api/rpc/*", type: "oRPC", desc: "Typed RPC endpoint a generated client calls — same contract object, no drift. This is the served default path (not /rpc)." }, + { name: "/health", type: "HTTP", desc: "Liveness/readiness check; anonymous by default (excluded from authn)." }, + { name: ":3001", type: "port", desc: "Default service port; read from Deno.env.get('PORT') || '3001'." } + ] +}) }} + +{{ comp callout { type: "important", title: "Aspire first, then hit the endpoint" } }} +Services that touch the database need orchestration up first: +cd aspire && aspire start brings up Postgres and Redis (dashboard at +https://localhost:18888) before any +netscript db command. The seeded-records example service runs without the DB, +but the real workflow expects Aspire running. See Aspire. +{{ /comp }} + +## OpenAPI spec & Scalar docs + +Passing `openapi` to `defineService` (or calling `.withOpenAPI().withDocs()` on the builder) +turns on a REST surface generated from the same contract, a machine-readable OpenAPI JSON +spec, and the **Scalar** interactive docs UI. Under the hood three Layer-1 primitives do the +work, and you can mount them directly in any host Hono app when you need finer control: +`createOpenAPISpec` serves the spec JSON, `createScalarDocs` serves the Scalar HTML page, and +`createOpenAPIHandler` serves the REST routes themselves (it adds the `ZodSmartCoercionPlugin` +so query-string values coerce to their schema types automatically). + +{{ comp.apiTable({ + caption: "OpenAPI / Scalar primitives (@netscript/service)", + rows: [ + { name: "createOpenAPISpec(router, config)", type: "→ ServiceHandler", desc: "Serves the OpenAPI JSON spec. config is OpenAPIConfig (see below). Mount at e.g. /api/openapi.json." }, + { name: "createScalarDocs(options)", type: "→ ServiceHandler", desc: "Serves the Scalar docs UI HTML that loads the spec. options is ScalarDocsOptions (specUrl, title?, theme?)." }, + { name: "createScalarJs()", type: "→ ServiceHandler", desc: "Serves the bundled Scalar JS so the docs UI works offline without CDN access." }, + { name: "createOpenAPIHandler(router, config?)", type: "→ FetchHandler", desc: "The OpenAPI REST request handler (with ZodSmartCoercionPlugin). config is RPCHandlerConfig; mount under /api/*." } + ] +}) }} + +{{ comp.apiTable({ + caption: "OpenAPIConfig (createOpenAPISpec)", + rows: [ + { name: "title", type: "string (required)", desc: "API title shown in the spec and docs." }, + { name: "version", type: "string (required)", desc: "API version, e.g. '1.0.0'." }, + { name: "description", type: "string?", desc: "Longer API description." }, + { name: "servers", type: "Array<{ url; description? }>?", desc: "Server URLs advertised in the spec (e.g. staging vs. production base URLs)." } + ] +}) }} + +{{ comp.apiTable({ + caption: "ScalarDocsOptions (createScalarDocs)", + rows: [ + { name: "specUrl", type: "string (required)", desc: "URL the docs UI fetches the OpenAPI spec from, e.g. '/api/openapi.json'." }, + { name: "title", type: "string?", desc: "Docs page title." }, + { name: "theme", type: "'default' | 'kepler' | 'moon' | 'purple' | 'saturn'", desc: "Scalar UI theme." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Preset — defineService({ openapi })", + lang: "ts", + code: "// services/users/src/main.ts\nimport { defineService } from '@netscript/service';\nimport { router } from './router.ts';\n\n// Turns on the OpenAPI spec + Scalar docs UI in one option.\nawait defineService(router, {\n name: 'users',\n version: '1.0.0',\n port: 3001,\n openapi: {\n title: 'Users API',\n description: 'User management service',\n },\n});" + }, + { + label: "Primitives — mount in a host app", + lang: "ts", + code: "// host/openapi-routes.ts — wire the primitives onto an existing Hono app\nimport {\n createOpenAPISpec,\n createScalarDocs,\n createScalarJs,\n} from '@netscript/service';\nimport { router } from './router.ts';\n\napp.get('/api/openapi.json', createOpenAPISpec(router, {\n title: 'Users API',\n version: '1.0.0',\n description: 'User management service',\n}));\n\napp.get('/api/docs', createScalarDocs({\n specUrl: '/api/openapi.json',\n title: 'Users API',\n theme: 'kepler',\n}));\n\n// Serve Scalar offline (no CDN dependency)\napp.get('/api/docs/scalar.js', createScalarJs());" + } +] }) }} + +## Health, readiness & liveness + +The preset wires `/health` automatically; the underlying primitives let you compose probes +by hand or mount them in a host app. `createHealthHandler` runs every registered +`HealthCheck` in parallel and returns an aggregate `HealthResponse` +(`status` ∈ `healthy | degraded | unhealthy`). `createReadinessHandler` takes an array of +async boolean checks and is the one to point an orchestrator's *readiness* probe at — it +fails until dependencies (DB, caches) are reachable. `createLivenessHandler` is a bare +"is the process up" probe that returns 200 with no dependency checks, for an orchestrator's +*liveness* probe. The `healthChecks` namespace ships pre-built checks for the common +dependencies so you don't hand-roll them. + +{{ comp.apiTable({ + caption: "Health primitives & checks (@netscript/service)", + rows: [ + { name: "createHealthHandler(options?)", type: "→ ServiceHandler", desc: "Runs all checks in parallel, returns an aggregate HealthResponse. options: HealthHandlerOptions { checks?, version?, includeDetails? (default true) }." }, + { name: "createReadinessHandler(checks)", type: "→ ServiceHandler", desc: "Readiness probe: checks is Array<() => Promise>; reports not-ready until every check resolves true. Mount at /health/ready." }, + { name: "createLivenessHandler()", type: "→ ServiceHandler", desc: "Liveness probe: 200 OK while the process runs, no dependency checks. Mount at /health/live." }, + { name: "healthChecks.database(db)", type: "→ HealthCheck", desc: "Pre-built DB probe; runs a $queryRaw `SELECT 1`-style ping against the Prisma client." }, + { name: "healthChecks.kv()", type: "→ HealthCheck", desc: "Pre-built check for the KV store." }, + { name: "healthChecks.service(name, baseUrl)", type: "→ HealthCheck", desc: "Probes another service's health endpoint by name + base URL." }, + { name: "healthChecks.custom(name, fn)", type: "→ HealthCheck", desc: "Wraps your own async () => Promise as a named check." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Compose probes by hand", + lang: "ts", + code: "// host/health-routes.ts — readiness + liveness + aggregate health\nimport {\n createHealthHandler,\n createLivenessHandler,\n createReadinessHandler,\n healthChecks,\n} from '@netscript/service';\nimport { db } from '@database';\n\n// Aggregate health (parallel checks + details)\napp.get('/health', createHealthHandler({\n version: '1.0.0',\n checks: [healthChecks.database(db)],\n}));\n\n// Liveness: is the process up?\napp.get('/health/live', createLivenessHandler());\n\n// Readiness: are dependencies reachable?\napp.get('/health/ready', createReadinessHandler([\n async () => { await db.$queryRaw`SELECT 1`; return true; },\n]));" + } +] }) }} + +{{ comp callout { type: "note", title: "Liveness vs. readiness" } }} +Point an orchestrator's liveness probe at /health/live (restart +the container only when the process is wedged) and its readiness probe at +/health/ready (stop routing traffic until the DB and caches answer). Wiring both +to the same dependency-checking endpoint is the classic footgun: a transient DB blip then +triggers a restart instead of a brief traffic pause. +{{ /comp }} + +## Graceful shutdown + +`serve()` installs SIGINT/SIGTERM (or SIGBREAK on Windows) handlers and drains in-flight +requests before exiting. Register teardown work with `.onShutdown(hook)` on the builder — a +`ShutdownHook` receives a `ShutdownContext` (`reason`, optional `signal`) and runs during the +drain. The drain is bounded by `drainTimeoutMs` (default `30_000`); when it elapses the +service stops anyway and the resulting `ShutdownReport` records `timedOut: true` plus a +per-hook `ShutdownHookOutcome` for each registered hook. Calling `running.stop()` triggers +the same drain manually (reason `'manual'`). + +{{ comp.apiTable({ + caption: "Shutdown types (@netscript/service)", + rows: [ + { name: "ShutdownHook", type: "(context: ShutdownContext) => Promise | void", desc: "Async teardown callback registered via .onShutdown(). Run in registration order during the drain." }, + { name: "ShutdownContext", type: "{ reason: ShutdownReason; signal?: Deno.Signal }", desc: "Passed to each hook. signal is set only when reason is 'signal'." }, + { name: "ShutdownReason", type: "'signal' | 'manual' | 'startup-failure'", desc: "Why the drain started: an OS signal, a manual stop() call, or a failed startup hook." }, + { name: "ShutdownHookOutcome", type: "{ ok: boolean; error?: string }", desc: "Per-hook result; error holds a normalized message when a hook throws or rejects." }, + { name: "ShutdownReport", type: "{ reason; timedOut: boolean; hooks: readonly ShutdownHookOutcome[] }", desc: "Final result of a completed shutdown — the reason, whether the timeout elapsed, and every hook outcome in execution order." } + ] +}) }} + +{{ comp.apiTable({ + caption: "ServeOptions (serve()) — shutdown-relevant keys", + rows: [ + { name: "port", type: "number?", desc: "Preferred listener port; use 0 for an ephemeral port." }, + { name: "signal", type: "AbortSignal?", desc: "External signal that stops the listener when aborted (drives the drain)." }, + { name: "drainTimeoutMs", type: "number?", desc: "Max time to wait for in-flight requests and shutdown hooks before forcing exit. Defaults to 30_000." }, + { name: "handleSignals", type: "boolean?", desc: "Install SIGINT/SIGTERM (or SIGBREAK) handlers. Defaults to true." }, + { name: "tls", type: "ServiceTlsOptions?", desc: "Opt-in TLS ({ cert, key } PEM strings). Present → HTTPS + HTTP/2 via ALPN; absent → plain HTTP/1.1. See TLS & HTTP/2 below." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "onShutdown teardown", + lang: "ts", + code: "// services/users/src/main.ts — drain + teardown on signal or manual stop\nimport { createService } from '@netscript/service';\nimport { router } from './router.ts';\nimport { db } from '@database';\n\nconst running = await createService(router, { name: 'users', version: '1.0.0' })\n .withRPC()\n .withHealth()\n .onShutdown(async ({ reason, signal }) => {\n // reason: 'signal' | 'manual' | 'startup-failure'\n audit.record({ event: 'shutdown', reason, signal });\n await db.$disconnect();\n })\n .serve({\n port: 3001,\n drainTimeoutMs: 10_000, // wait up to 10s for in-flight work\n handleSignals: true, // SIGINT/SIGTERM/SIGBREAK\n });\n\n// Later, in tests or a supervisor: trigger the same drain manually.\nawait running.stop();" + } +] }) }} + +## TLS & HTTP/2 (opt-in) + +By default a service listens over plain HTTP. Pass a `tls` option and the *same* listener serves +HTTPS and negotiates **HTTP/2 automatically via ALPN** — HTTP/1.1 stays available as a fallback, and +there is no separate `http2` flag. TLS is purely additive: a service that omits `tls` is unchanged. + +`tls` is a `ServiceTlsOptions` object whose `cert` and `key` are **PEM-encoded strings** — the +certificate chain and the private key as *contents*, not file paths. It is accepted by both +`defineService(router, { …, tls })` and the fluent builder's `.serve({ tls })`. + +```ts +// services/users/src/main.ts — serve HTTPS + HTTP/2 with an inline cert/key +import { defineService } from '@netscript/service'; +import { router } from './router.ts'; + +await defineService(router, { + name: 'users', + port: 3000, + tls: { + cert: await Deno.readTextFile('cert.pem'), // PEM contents + key: await Deno.readTextFile('key.pem'), // PEM contents + }, +}); +// Fluent equivalent: createService(router, { name }).withHealth().serve({ tls: { cert, key } }) +``` + +If you omit `tls`, the listener falls back to environment configuration: set **both** +`NETSCRIPT_TLS_CERT_FILE` and `NETSCRIPT_TLS_KEY_FILE` and the service serves HTTPS from those files. +Unlike the inline `cert`/`key` (PEM contents), these env vars are **file paths** — and both are +required; one alone is ignored. + +{{ comp.apiTable({ + caption: "ServiceTlsOptions (@netscript/service) + env fallback", + rows: [ + { name: "cert", type: "string (required)", desc: "PEM-encoded certificate chain — the contents, not a path." }, + { name: "key", type: "string (required)", desc: "PEM-encoded private key — the contents, not a path." }, + { name: "NETSCRIPT_TLS_CERT_FILE", type: "env (path)", desc: "Fallback when tls is omitted: file path to the PEM certificate chain. Both env vars must be set together." }, + { name: "NETSCRIPT_TLS_KEY_FILE", type: "env (path)", desc: "Fallback when tls is omitted: file path to the PEM private key. Both env vars must be set together." } + ] +}) }} + +## Service-layer authn / authz middleware + +The service builder ships a **provider-agnostic** authentication and authorization seam in +`@netscript/service/auth`. It is a thin Hono-middleware layer over the request pipeline — +deliberately **distinct from the [auth plugin](/identity-access/auth/)**, which composes a +sign-in/session backend (kv-oauth, WorkOS, better-auth). Use this seam when a service needs +to gate its own routes — verify a credential, trust an upstream identity header, or check a +scope — without taking on an interactive identity provider. + +Two stages wrap the request: `.withAuthn()` resolves a `Principal` (authentication) via an +`AuthenticatorPort`, and `.withAuthz()` makes an `AuthzDecision` from that principal +(authorization) via an `AuthorizerPort`. By default the `/api` surface is protected and +`/health` is anonymous — both configurable through `AuthnOptions.protect` / +`AuthnOptions.allowAnonymous`. The preset form is `defineService(router, { auth: { authn, authz } })`. + +{{ comp.apiTable({ + caption: "@netscript/service/auth surface", + rows: [ + { name: "createStaticCredentialAuthenticator", type: "authenticator", desc: "Matches a configured static credential (e.g. API key / shared secret) → Principal." }, + { name: "createTrustedHeaderAuthenticator", type: "authenticator", desc: "Trusts an identity asserted by an upstream proxy header (e.g. behind a gateway)." }, + { name: "createScopeAuthorizer", type: "authorizer", desc: "Allows/denies a request by checking the Principal's scopes against required scopes." }, + { name: ".withAuthn({ authenticator, protect?, allowAnonymous? })", type: "builder method", desc: "Installs the authentication gate (AuthnOptions). protect defaults to ['/api']; allowAnonymous defaults to ['/health']." }, + { name: ".withAuthz({ authorizer, denyByDefault? })", type: "builder method", desc: "Installs the authorization gate (AuthzOptions) from the Principal. denyByDefault fails closed and defaults to true." } + ] +}) }} + +{{ comp.tabbedCode({ tabs: [ + { + label: "Fluent — withAuthn / withAuthz", + lang: "ts", + code: "// Gate a service with a static credential + scope check\nimport { createService } from '@netscript/service';\nimport {\n createStaticCredentialAuthenticator,\n createScopeAuthorizer,\n} from '@netscript/service/auth';\nimport { router } from './router.ts';\n\nconst authenticator = createStaticCredentialAuthenticator({\n credentials: {\n [Deno.env.get('SERVICE_API_KEY') ?? '']: {\n subject: 'service:ci',\n scopes: ['users:write'],\n },\n },\n});\nconst authorizer = createScopeAuthorizer({\n rules: [{ match: () => true, requireScopes: ['users:write'] }],\n denyByDefault: true,\n});\n\nawait createService(router, { name: 'users', version: '1.0.0', port: 3001 })\n .withRPC()\n .withAuthn({ authenticator })\n .withAuthz({ authorizer })\n .withHealth() // /health stays anonymous\n .serve();" + }, + { + label: "Trusted-header (behind a gateway)", + lang: "ts", + code: "// Trust identity asserted by an upstream proxy/gateway\nimport { createTrustedHeaderAuthenticator } from '@netscript/service/auth';\n\n// The Principal is assembled internally from these header NAMES — point at the\n// headers your gateway sets; you do not map values yourself.\nconst authenticator = createTrustedHeaderAuthenticator({\n subjectHeader: 'x-forwarded-user',\n scopesHeader: 'x-forwarded-scopes', // optional; space/comma-separated\n});\n\n// then: .withAuthn({ authenticator })" + } +] }) }} + +{{ comp callout { type: "note", title: "Service-auth seam vs. the auth plugin" } }} +This middleware is for machine-to-machine and gateway-fronted gating: static +credentials, trusted upstream headers, scope checks. It does not do interactive +sign-in, OAuth callbacks, or session cookies. For signed-in human users +(sign-in / callback / session / me), compose the +auth plugin backend instead — the two layers +are complementary and can run together. +{{ /comp }} + +## Production notes + +{{ comp callout { type: "warning", title: "Footguns before you ship" } }} +
    +
  • Don't wire liveness to dependency checks. /health/live must +stay dependency-free; only /health/ready should fail on an unreachable DB. Crossing +them turns a transient DB blip into a container restart loop.
    • +
  • Set drainTimeoutMs below your platform's kill grace. The drain +defaults to 30_000; if your orchestrator sends SIGKILL sooner, in-flight requests +are cut off — pick a timeout under the platform's termination grace period.
    • +
  • The served RPC path is /api/rpc/*, not /rpc. +Point generated clients at the same path the runtime mounts.
    • +
  • denyByDefault is true. With .withAuthz() +installed, any request that reaches no matching rule is rejected — add an explicit allow rule for +public-but-authenticated routes rather than relaxing the default.
    • +
  • Aspire first for DB-backed services. aspire start must bring up +Postgres/Redis before any netscript db command or the readiness probe will fail.
    • +
+{{ /comp }} + +## Reference + +This hub is intentionally thin — the full generated API lives in the reference. + +{{ comp.xref({ key: "ref:service" }) }} + +{{ comp.featureGrid({ items: [ + { + title: "Look up — @netscript/service", + body: "The full generated API: defineService, createService, the fluent builder, the OpenAPI/Scalar and health primitives, the shutdown types, and the /auth middleware seam.", + href: "/reference/service/", + icon: "≡" + }, + { + title: "Look up — @netscript/contracts", + body: "Contract primitives plus the /crud, /query, and /transform helpers that generate CRUD contracts, paginated queries, and response transformers.", + href: "/reference/contracts/", + icon: "≡" + }, + { + title: "Understand — Contracts", + body: "The oRPC contract → implement → handler → typed client → query → island type flow, explained.", + href: "/explanation/contracts/", + icon: "◎" + }, + { + title: "Learn — Build a service", + body: "Guided tutorial: contract → users service on :3001 → typed client → island, built from scratch.", + href: "/tutorials/storefront/02-catalog-service/", + icon: "→" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Capabilities", href: "/capabilities/" }, next: { label: "Background jobs", href: "/background-processing/workers/" } }) }} diff --git a/docs/site/web-layer/fresh-ui.md b/docs/site/web-layer/fresh-ui.md new file mode 100644 index 000000000..ec9802d44 --- /dev/null +++ b/docs/site/web-layer/fresh-ui.md @@ -0,0 +1,264 @@ +--- +layout: layouts/base.vto +title: Fresh UI & design +templateEngine: [vento, md] +prev: { label: "Telemetry & logging", href: "/observability/telemetry/" } +next: { label: "Authentication", href: "/identity-access/auth/" } +--- + +# Fresh UI & design + +NetScript gives you a **front end with the same contracts-first rigor as the back end** — and it +does so in two distinct layers that are easy to conflate. There is **`@netscript/fresh`, a real +meta-framework** (a published package of Fresh runtime extensions, route/island builders, a +TanStack-Query bridge, forms, and a streams client), and there is the **scaffolded dashboard app** +(`apps/dashboard`) that the meta-framework *powers*. The package is the reusable engine; the app is +one fully-wired showcase built on top of it. Both ship in every workspace, but they live at +different addresses and you reach for them at different moments. + +{{ comp callout { type: "tip", title: "Use this when" } }} +Reach for this layer when you need a typed, server-rendered web UI over your +services — an internal admin panel, an operations console, or a customer-facing app — that cannot +drift from your oRPC contracts. Use @netscript/fresh (the +meta-framework) to build pages/islands in any Fresh app; use apps/dashboard +(the scaffolded app) as the worked reference to copy patterns from. For the data boundary the UI +renders see Services & contracts; to add or regenerate UI +pieces see Customize the Fresh UI. +{{ /comp }} + +## Two layers, one design grammar + +{{ comp.apiTable({ + caption: "The meta-framework vs. the scaffolded app", + rows: [ + { name: "@netscript/fresh", type: "meta-framework (package)", desc: "A published Deno package: Fresh runtime extensions, the definePage/island builders, a TanStack-Query bridge, forms, defer primitives, a streams client, and route contracts. Reusable across any Fresh app you build. Reference: /reference/fresh/." }, + { name: "apps/dashboard", type: "scaffolded app", desc: "One concrete Fresh 2.x application generated into your repo, built ON @netscript/fresh and wired to your users service. The worked showcase you copy patterns from — not a finished product UI." }, + { name: "@netscript/fresh-ui", type: "component registry", desc: "The copy-source component library + design tokens installed into apps/dashboard/components/ui/ via netscript ui:init / ui:add. You own the copied files. Reference: /reference/fresh-ui/." } + ] +}) }} + +{{ comp callout { type: "important", title: "Don't conflate the package with the app" } }} +@netscript/fresh is a meta-framework you import — it exposes targeted +subpaths (/server, /builders, /route, /query, +/form, /defer, /streams, /interactive, +/vite, /testing, /error) and can power any Fresh app, +not just the scaffold. apps/dashboard is the app that consumes it. When +this page says "the dashboard," it means the scaffolded app; when it says +"@netscript/fresh," it means the reusable engine. +{{ /comp }} + +## The meta-framework: `@netscript/fresh` + +`@netscript/fresh` is a published package (`Fresh runtime extensions, builders, forms, defer +primitives, and route contracts for NetScript`) built on `@fresh/core` 2.x + Preact. You consume it +through **targeted subpath exports** rather than one fat barrel — import only the surface you need. +The full generated API for every subpath lives at the +[`@netscript/fresh` reference](/reference/fresh/); the table below is the map. + +{{ comp.apiTable({ + caption: "@netscript/fresh subpath exports (the meta-framework surface)", + rows: [ + { name: "@netscript/fresh/server", type: "bootstrap", desc: "defineFreshApp(...) — the baseline app bootstrap (static files, file-system routes) so every NetScript Fresh app boots identically." }, + { name: "@netscript/fresh/builders", type: "page builder", desc: "The definePage() / route-reference builders that bind a typed route, declare server-loaded layers, and wire telemetry." }, + { name: "@netscript/fresh/route", type: "route contracts", desc: "Route-reference primitives (createRouteReference) for typed, generated route patterns consumed via router.ts." }, + { name: "@netscript/fresh/query", type: "TanStack bridge", desc: "QueryIsland, useQuery, useMutation, useQueryClient — the TanStack-Query hydration bridge over your contract-derived query factories." }, + { name: "@netscript/fresh/form", type: "forms", desc: "Typed form primitives for server-validated, contract-aware form handling." }, + { name: "@netscript/fresh/defer", type: "streaming defer", desc: "Defer primitives for streaming/deferred server rendering of slower layers." }, + { name: "@netscript/fresh/streams", type: "streams client", desc: "The Fresh-side durable-stream client for consuming HTTP/SSE durable streams in the browser (the producer runtime lives in @netscript/plugin-streams-core)." }, + { name: "@netscript/fresh/interactive", type: "interactivity", desc: "Interactive island runtime helpers used by the hydrated client islands." }, + { name: "@netscript/fresh/vite", type: "build", desc: "The Vite integration the dev/build pipeline uses." }, + { name: "@netscript/fresh/testing", type: "testing", desc: "Test helpers for exercising pages, islands, and route contracts." }, + { name: "@netscript/fresh/error", type: "diagnostics", desc: "Typed error/diagnostics surface for the Fresh runtime." } + ] +}) }} + +{{ comp callout { type: "note", title: "Distinct from the back-end stream producer" } }} +The Fresh-side @netscript/fresh/streams client consumes durable streams over +HTTP/SSE in the browser. The producer runtime — the thing that writes stream state +— is real and lives server-side in @netscript/plugin-streams-core +(createDurableStream), served as an Aspire service on +:4437. See Streams for the +producer/consumer split. +{{ /comp }} + +## The scaffolded app: `apps/dashboard` + +Every NetScript workspace ships `apps/dashboard` — a [Fresh](https://fresh.deno.dev/) 2.x +application built with Preact, Tailwind CSS v4, and Vite. It is a workspace member registered in the +root `deno.json` and orchestrated by Aspire alongside your services. It boots through +`defineFreshApp` from `@netscript/fresh/server`, the meta-framework's baseline bootstrap, so every +NetScript Fresh app starts the same way. It is **not a placeholder**: the scaffold wires it directly +to your oRPC contracts so the dashboard renders typed, server-prefetched data from the same `users` +service the back end implements. + +{{ comp.apiTable({ + caption: "apps/dashboard layout (verbatim from a fresh scaffold)", + rows: [ + { name: "main.ts", type: "entry", desc: "App entry: export const app = defineFreshApp({ name: 'dashboard' }). Reads PORT (default 8010) and logs a startup banner." }, + { name: "router.ts", type: "routing", desc: "Stable route entrypoint. Re-exports generated routePatterns + routes and builds typed appRoutes via createRouteReference." }, + { name: "routes/", type: "pages", desc: "File-system routes: index.tsx, dashboard.tsx, health.tsx, examples/service/, examples/telemetry/, the (design) system pages, plus _app.tsx / _layout.tsx shells." }, + { name: "islands/", type: "interactivity", desc: "Client-hydrated Preact islands (e.g. ThemeToggle, SidebarToggle, Toast under islands/ui/)." }, + { name: "components/ui/", type: "design system", desc: "The copy-source component library you own (@netscript/fresh-ui): button, card, data-table, form-field, badge, and more (tsx + matching CSS in assets/ui/)." }, + { name: "lib/", type: "service wiring", desc: "example-service.ts builds a typed oRPC client + query factories from your contract; cn.ts, public-types.ts." }, + { name: "assets/", type: "styling", desc: "design.css, tokens.css/json, theme-bridge.css, and per-component CSS — the Tailwind v4 + design-token layer." }, + { name: ".generated/", type: "generated", desc: "manifest.ts + routes.ts produced by the Fresh route generator; consume via router.ts, never directly." } + ] +}) }} + +{{ comp callout { type: "note", title: "Two ports, one app" } }} +The dashboard's own dev server reads PORT and defaults to 8010 +(http://localhost:8010, health at /health). That is distinct from the +Aspire dashboard at https://localhost:18888, +which orchestrates the whole resource graph. Bring the platform up first with +cd aspire && aspire start (Postgres/Redis + every service), or run the app +standalone with deno task --cwd apps/dashboard dev. +{{ /comp }} + +## Contract-driven by default + +The dashboard does not hand-roll `fetch` calls. `apps/dashboard/lib/example-service.ts` imports your +`UsersContractV1` and turns it into a typed client and TanStack-Query factories through the NetScript +SDK — the same contract object the `users` service implements, so the UI cannot drift from the API. + +```ts +// apps/dashboard/lib/example-service.ts +import { createServiceClient } from '@netscript/sdk/client'; +import { createQueryFactories } from '@netscript/sdk/query'; +import { bridgeInvalidation } from '@netscript/sdk/query-client'; +import { UsersContractV1 } from '@plugin-smoke/contracts'; + +export const exampleServiceName = 'users'; +export const exampleServiceClient = createServiceClient({ + contract: UsersContractV1, + serviceName: exampleServiceName, + routerName: 'users', +}); + +// queryOptions / mutationOptions derived straight from the contract. +export const exampleServiceQueries = createQueryFactories({ + service: { contract: UsersContractV1, client: exampleServiceClient }, +}).service; +``` + +A **route** is declared with the meta-framework's `definePage()` builder (`@netscript/fresh/builders`) +— it binds a typed route reference, declares server-loaded layers, and wires telemetry — while an +**island** consumes the query factories above through the `@netscript/fresh/query` bridge for +hydration, optimistic mutations, and cache invalidation on the client. + +{{ comp.tabbedCode({ tabs: [ + { + label: "Route — routes/examples/service/index.tsx", + lang: "tsx", + code: "// definePage() declares a typed page: route + server-loaded layers + telemetry.\nimport { appRoutes } from '@app/router.ts';\nimport { definePage } from '@app/utils.ts';\nimport { ServiceExampleLabPanel } from './(_components)/lab-panel.tsx';\nimport { loadServiceShowcaseData } from './(_shared)/service-showcase.ts';\n\nexport const serviceExamplePage = definePage()\n .withRoute(appRoutes.serviceExample)\n .withPolicy('balanced')\n .withTelemetry({ enabled: true, spanName: 'scaffold.examples.users' })\n .withMeta(() => ({ title: 'users example', description: 'Backed by the users service.' }))\n .withLayer('lab', ServiceExampleLabPanel, { loader: loadServiceShowcaseData })\n .build();\n\nexport const { default: page } = serviceExamplePage;\nexport { page as default };" + }, + { + label: "Island — (_islands)/ServiceShowcaseLab.tsx", + lang: "tsx", + code: "// Client-hydrated island: typed query + optimistic mutation over the contract.\nimport { QueryIsland, useMutation, useQuery, useQueryClient } from '@netscript/fresh/query';\nimport { exampleServiceQueries, exampleServiceListInvalidation } from '@app/lib/example-service.ts';\n\nconst Lab = (props: { input: { status?: string } }) => {\n const queryClient = useQueryClient();\n const { data, refetch, isRefetching } = useQuery({\n ...exampleServiceQueries.list.queryOptions(props.input),\n staleTime: 15_000,\n });\n const advance = useMutation({\n ...exampleServiceQueries.updateStatus.mutationOptions(),\n onSettled: () => queryClient.invalidateQueries(exampleServiceListInvalidation),\n });\n return ;\n};\n\nexport default (props: { input: { status?: string } }) => (\n \n);" + } +] }) }} + +{{ comp callout { type: "important", title: "Islands are the only client code" } }} +Fresh ships zero JavaScript by default. Anything under islands/ is the only +code hydrated in the browser; routes and components render on the server. The users +showcase loads its list on the server, dehydrates the query cache, and the +ServiceShowcaseLab island re-hydrates it for interactive refetch and optimistic status +updates — the contract-to-pixel proof end to end. +{{ /comp }} + +## The design system & owned components + +The scaffold installs the NetScript Fresh UI foundation (`@netscript/fresh-ui`) into +`apps/dashboard/components/ui/` and its CSS into `assets/ui/`, driven by design tokens in +`assets/tokens.json` / `tokens.css`. Because the components are copied into your repo, editing the +UI is editing your own files — there is no framework component you cannot open. The bundled +`(design)` route group renders a live token, component, and composition gallery so you can see every +primitive in your project. + +What ships falls into four groups, each installable by registry id (`ui:add `) or as a named +collection: + +{{ comp.apiTable({ + caption: "What the registry ships", + rows: [ + { name: "Interactive namespaces", type: "@netscript/fresh-ui/interactive", desc: "Stateful, accessible compound primitives: Accordion, Dialog, Drawer, Popover, Sheet, Tabs, Tooltip, and the headless Combobox seam (useCombobox) that powers the command palette and autocompletes." }, + { name: "L0 primitives", type: "@netscript/fresh-ui/primitives", desc: "Small platform-contract helpers: Show (wrapper-free conditional render), VisuallyHidden and its SrOnly alias, and Icon — a first-class stroke-SVG icon primitive also re-exported from the package root." }, + { name: "AI / workspace primitives", type: "ui:add", desc: "The agent-surface set: avatar, citation-chip, code-block, model-selector, tool-call-card, chart-block, donut, prompt-input (auto-grows to fit its content), message (renderInline + TypingIndicator), and dropzone (drag, clipboard-paste, and multi-file ingest) — plus the foundation set (button, card, badge, data-table, forms, feedback, dashboard blocks)." }, + { name: "Command & search utilities", type: "ui:add", desc: "command-palette (.ns-cmdk — the ⌘K modal that wraps the Combobox) and search (.ns-search — a compact nav affordance with a ⌘K hint that opens it)." } + ] +}) }} + +You manage that library with two CLI commands (run from the workspace root): + +{{ comp.apiTable({ + caption: "Fresh UI CLI commands", + rows: [ + { name: "netscript ui:init", type: "install foundation", desc: "Installs the Fresh UI foundation set into an app workspace. The scaffold runs the equivalent for you; run it once when adding UI to an app that lacks it." }, + { name: "netscript ui:add ", type: "add an item", desc: "Copies one registry item or a named collection into components/ui/, wires its CSS, and merges any required deno.json imports. Example: netscript ui:add data-table." } + ] +}) }} + +{{ comp callout { type: "warning", title: "Copy-source ownership tradeoff" } }} +Because ui:add copies code into your repo, you own it: customize freely, but you do +not get automatic upstream updates. Re-running ui:add … --force +re-copies the source — and overwrites your local edits — so treat foundation files as yours and keep +heavy customization in your own components. Full options live behind +netscript ui:init --help / netscript ui:add --help and in the +@netscript/fresh-ui reference. +{{ /comp }} + +## Scope + +The scaffolded dashboard is a working, contract-wired **showcase** — not a finished product UI. It +ships the `users` example (server-prefetched list + optimistic status mutation), a telemetry +example, a CRUD example, a health page, and the design gallery. Treat these as the canonical patterns +to copy from, not features to ship as-is; build your real screens by following the same +`definePage` + island + query-factory shape against `@netscript/fresh`. The meta-framework is the +durable surface — the app is one application of it. + +## Endpoints & ports + +{{ comp.apiTable({ + caption: "Fresh UI runtime surface", + rows: [ + { name: ":8010", type: "port", desc: "Dashboard dev server (Deno.env.get('PORT') || '8010'). Standalone: deno task --cwd apps/dashboard dev." }, + { name: "/health", type: "HTTP", desc: "App health route logged at startup (http://localhost:8010/health)." }, + { name: ":18888", type: "Aspire", desc: "Aspire dashboard that orchestrates the dashboard alongside services/plugins; token printed by aspire start." }, + { name: ":3001", type: "upstream", desc: "The users service the dashboard's typed client calls — same UsersContractV1, no drift." }, + { name: ":4437", type: "streams", desc: "The durable-streams Aspire service the @netscript/fresh/streams client consumes over HTTP/SSE (producer runtime in @netscript/plugin-streams-core)." } + ] +}) }} + +## Where to go next + +This hub is intentionally thin — the full generated APIs live in the reference. Pick the lane that +matches what you're doing. + +{{ comp.featureGrid({ items: [ + { + title: "Learn — Build a service", + body: "Guided tutorial: contract → users service on :3001 → typed client → the island that renders it in the dashboard.", + href: "/tutorials/storefront/02-catalog-service/", + icon: "→" + }, + { + title: "Do — Customize the Fresh UI", + body: "Task recipe: ui:init / ui:add, the copy-source ownership model, and editing the design tokens.", + href: "/how-to/customize-fresh-ui/", + icon: "◆" + }, + { + title: "Look up — @netscript/fresh reference", + body: "The meta-framework's full generated API: defineFreshApp, definePage, the query/route/form/defer/streams builders, and the server runtime.", + href: "/reference/fresh/", + icon: "≡" + }, + { + title: "Look up — @netscript/fresh-ui reference", + body: "The component registry, foundation set, design tokens, and the ui:init / ui:add surface.", + href: "/reference/fresh-ui/", + icon: "≡" + } +] }) }} + +{{ comp.nextPrev({ prev: { label: "Telemetry & logging", href: "/observability/telemetry/" }, next: { label: "Authentication", href: "/identity-access/auth/" } }) }}