An upstream-less ai-gateway source idles instead of throwing (#649)

hypaware-core/plugins-workspace/ai-gateway/src/config.js

@@ -13,9 +13,12 @@ export const FALLBACK_LISTEN = '127.0.0.1:0'
 
 /**
  * Validate and normalize the ai-gateway config slice. Returns the
- * compiled shape used by the source/listener. Validation is strict:
- * missing or malformed `upstreams` is rejected loudly because the
- * gateway has nothing useful to do without at least one upstream.
+ * compiled shape used by the source/listener. Missing or malformed
+ * `upstreams` compiles to an empty list rather than an error: adapter
+ * plugins contribute the rest of the routing table as presets after this
+ * runs, and a config that wants the gateway plugin only for its dataset and
+ * materializer (`@hypaware/hermes`) legitimately names no upstream at all.
+ * The source decides what an empty table means, not this function.
  *
  * @param {unknown} raw
  * @returns {AiGatewayConfig}

hypaware-core/plugins-workspace/ai-gateway/src/source.js

@@ -49,18 +49,47 @@ export function createStartSource(state) {
       entrypoints: createEntrypointActivity(),
     }
 
+    // `undefined` when the compiled routing table is empty: the source idles
+    // instead of binding a listener that could route nothing. See
+    // {@link launchListener}.
     let proxy = await launchListener(ctx, state, liveState)
 
+    // The config `status()` reports on, not the one this source booted with.
+    // `reload()` hands the daemon's new context to the listener but the
+    // closure above keeps the boot-time `ctx` forever, so reading it would
+    // publish a stale `details.upstreams` after every reload. Core's
+    // `gateway_idle_no_upstreams` diagnostic reads exactly that field to
+    // tell "hermes-only, correctly idle" from "an upstream was dropped", so
+    // it has to describe the config in force now.
+    let activeCtx = ctx
+
     return {
       async status() {
+        const configured = readConfiguredUpstreams(activeCtx)
         /** @type {SourceStatus} */
         const status = {
           state: 'ready',
           rowsWritten: liveState.rowsWritten,
           details: {
-            host: proxy.host,
-            port: proxy.port,
-            upstreams: readConfiguredUpstreamNames(ctx),
+            // Omitted while idle, which is already how `gatewaySourceDetails`
+            // (core `daemon/status.js`) reads "no reachable gateway here" off
+            // the status file for a bind that never happened.
+            ...(proxy ? { host: proxy.host, port: proxy.port } : { listening: false }),
+            // Raw configured names, pre-compile, deliberately: an entry the
+            // compiler dropped (a `url =` where `base_url` was meant) still
+            // appears here, which is what lets core see the difference
+            // between a gateway with nothing to proxy and a gateway whose
+            // upstream fell out of the routing table.
+            upstreams: configured.names,
+            // The names cannot carry the whole signal, because `name` is one
+            // of the two keys whose absence drops an entry: an upstream
+            // written with a `provider` and a `base_url` but no `name` leaves
+            // `upstreams: []`, indistinguishable from hermes-only. The count
+            // is the wider question ("did this config ask for any upstream at
+            // all?") and it is the one core's `gateway_idle_no_upstreams`
+            // diagnostic gates on; the names only decide how the warning
+            // reads.
+            upstreams_configured: configured.count,
             registered_presets: Array.from(state.presets.keys()),
             projectors: state.projectors.map((p) => p.name),
             // @ref LLP 0066#ephemeral: surface the live opt-out count so an
@@ -80,6 +109,7 @@ export function createStartSource(state) {
             recent_entrypoints: liveState.entrypoints.snapshot(),
           },
         }
+        if (!proxy) status.message = 'idle: no upstreams configured, nothing to proxy'
         if (liveState.lastError) status.lastError = liveState.lastError
         return status
       },
@@ -89,13 +119,14 @@ export function createStartSource(state) {
         // new config. Connections in flight finish through the
         // recorder's drain (called inside stop()) so their rows are not
         // lost across the reload.
-        await proxy.stop()
+        await proxy?.stop()
         state.listen = undefined
         proxy = await launchListener(nextCtx, state, liveState)
+        activeCtx = nextCtx
       },
 
       async stop() {
-        await proxy.stop()
+        await proxy?.stop()
         state.listen = undefined
       },
     }
@@ -108,13 +139,63 @@ export function createStartSource(state) {
  * `AiGatewayCapability.localEndpoint()` returns the bound URL; clears
  * it on stop/reload.
  *
+ * Returns `undefined` when the compiled routing table is empty, leaving the
+ * source idle with no listener at all.
+ *
+ * The gateway plugin does two separable jobs, and a config can legitimately
+ * want only one. At activation it contributes the `ai_gateway_messages`
+ * dataset and the shared `ai_gateway.projected_exchange` materializer; at
+ * source start it runs the proxy. `@hypaware/hermes` wants the first alone:
+ * it reads Hermes's own `state.db` and is "never modified, configured, or
+ * proxied" (LLP 0119), yet the materializer is a hard `requires.plugins`
+ * dependency (LLP 0120), so its picker row composes the gateway plugin while
+ * contributing no upstream. A hermes-only picker run therefore produces
+ * `upstreams: []` with no adapter presets either, and failing the source
+ * start there would take a correct install down over a dataset-only
+ * dependency. Idling instead leaves `state.listen` unset, so
+ * `localEndpoint()` keeps throwing rather than handing an attach a URL
+ * nothing is listening on.
+ *
+ * @ref LLP 0120#consequences [constrained-by]: hermes composes the gateway plugin for the materializer alone, so an upstream-less gateway is a valid config rather than a misconfiguration
+ *
  * @param {PluginActivationContext} ctx
  * @param {GatewayState} state
  * @param {{ rowsWritten: number, exchangeBytes: number, lastError: string | undefined, listenFallbackFrom: string | undefined, entrypoints: ReturnType<typeof createEntrypointActivity> }} liveState
- * @returns {Promise<StartedProxy>}
+ * @returns {Promise<StartedProxy | undefined>}
  */
 async function launchListener(ctx, state, liveState) {
   const config = compileConfig(ctx.config)
+  // Hoisted out of `bind` below (which runs twice on the EADDRINUSE fallback
+  // path) because the answer decides whether we bind at all. Pure over
+  // `config.upstreams` and `state.presets`, neither of which moves between
+  // the two binds.
+  const upstreams = mergeUpstreams(config.upstreams, state)
+  if (upstreams.length === 0) {
+    liveState.listenFallbackFrom = undefined
+    // Two configs reach an empty routing table and they are not the same
+    // event. A hermes-only install asked for no upstream at all: idle is the
+    // outcome it wanted, and `info` is the right volume. A config that listed
+    // upstreams and still compiled to none lost every one of them to
+    // `compileUpstreams` (a missing or misspelled `base_url` is dropped
+    // silently), so the operator is going to get ECONNREFUSED from a gateway
+    // that reports itself started. Name the entries that vanished, at `warn`.
+    const configured = readConfiguredUpstreams(ctx)
+    if (configured.count > 0) {
+      ctx.log.warn('aigw.idle_no_upstreams', {
+        [Attr.PLUGIN]: PLUGIN_NAME,
+        registered_presets: state.presets.size,
+        configured_upstreams: configured.count,
+        configured_upstream_names: configured.names,
+        reason: 'every configured upstream was dropped at compile: check base_url on each entry',
+      })
+    } else {
+      ctx.log.info('aigw.idle_no_upstreams', {
+        [Attr.PLUGIN]: PLUGIN_NAME,
+        registered_presets: state.presets.size,
+      })
+    }
+    return undefined
+  }
   const recorder = createRecorder({ redactHeaders: config.redactHeaders })
   const projector = createAiGatewayMessageProjector({
     gatewayId: config.gatewayId,
@@ -188,7 +269,7 @@ async function launchListener(ctx, state, liveState) {
   /** @param {string} listen */
   const bind = (listen) => startProxy({
     listen,
-    upstreams: mergeUpstreams(config.upstreams, state),
+    upstreams,
     startExchange: (init) => recorder.startExchange(init),
     onExchangeFinished,
     // Serve `/_hypaware/*` control requests locally over the gateway's
@@ -288,16 +369,21 @@ export function mergeUpstreams(configUpstreams, state) {
 }
 
 /**
- * Read the names of configured upstreams from the activation config.
- * Defensive: if config has been mutated to a degenerate shape, returns
- * an empty list so status() never throws.
+ * Both halves of "what did the config ask for?": how many upstream entries it
+ * listed at all, and the names among them. The count is the wider signal (an
+ * entry with no `name` still counts), so the idle log and `hyp status` can be
+ * loud about a config that listed upstreams and compiled to none even when the
+ * names are unusable.
+ *
+ * Defensive: if config has been mutated to a degenerate shape, returns a zero
+ * count and an empty list so `status()` never throws.
  *
  * @param {PluginActivationContext} ctx
- * @returns {string[]}
+ * @returns {{ count: number, names: string[] }}
  */
-function readConfiguredUpstreamNames(ctx) {
+function readConfiguredUpstreams(ctx) {
   const raw = /** @type {Record<string, unknown>} */ (ctx.config ?? {}).upstreams
-  if (!Array.isArray(raw)) return []
+  if (!Array.isArray(raw)) return { count: 0, names: [] }
   /** @type {string[]} */
   const out = []
   for (const entry of raw) {
@@ -306,7 +392,7 @@ function readConfiguredUpstreamNames(ctx) {
       if (typeof name === 'string' && name.length > 0) out.push(name)
     }
   }
-  return out
+  return { count: raw.length, names: out }
 }
 
 /**

src/core/daemon/status.js

@@ -112,13 +112,8 @@ const GATEWAY_PLUGIN_NAME = '@hypaware/ai-gateway'
  * @ref LLP 0086#endpoint-discovery [implements]: the daemon's live bound port is read from status.json sources[].details, not guessed
  */
 export function gatewaySourceDetails(sources) {
-  const list = Array.isArray(sources) ? sources : []
-  const source =
-    list.find((s) => s && s.plugin === GATEWAY_PLUGIN_NAME) ??
-    list.find((s) => s && s.name === 'ai-gateway')
-  const rawDetails = source && typeof source.details === 'object' ? source.details : undefined
-  if (!rawDetails) return undefined
-  const details = /** @type {Record<string, unknown>} */ (rawDetails)
+  const details = gatewaySourceRawDetails(sources)
+  if (!details) return undefined
   const port = details.port
   if (typeof port !== 'number' || !Number.isInteger(port) || port <= 0) return undefined
   const host = typeof details.host === 'string' && details.host.length > 0 ? details.host : '127.0.0.1'
@@ -131,6 +126,71 @@ export function gatewaySourceDetails(sources) {
   return { host, port, listenFallback, ...(listenFallbackFrom ? { listenFallbackFrom } : {}) }
 }
 
+/**
+ * The gateway source's `status()` details as the daemon captured them, before
+ * any "is it bound?" filtering. `gatewaySourceDetails` above answers "where do
+ * I send traffic?" and so returns nothing for a gateway that never bound; the
+ * idle checks below need the details of exactly that case.
+ *
+ * @param {SourceSnapshot[] | undefined} sources
+ * @returns {Record<string, unknown> | undefined}
+ */
+function gatewaySourceRawDetails(sources) {
+  const list = Array.isArray(sources) ? sources : []
+  const source =
+    list.find((s) => s && s.plugin === GATEWAY_PLUGIN_NAME) ??
+    list.find((s) => s && s.name === 'ai-gateway')
+  const rawDetails = source && typeof source.details === 'object' ? source.details : undefined
+  if (!rawDetails) return undefined
+  return /** @type {Record<string, unknown>} */ (rawDetails)
+}
+
+/**
+ * How many upstreams a *deliberately idle* gateway was nonetheless configured
+ * with, and which of them it can name, or `undefined` when the gateway is
+ * bound, absent, or idle for the reason it is allowed to be idle.
+ *
+ * An upstream-less gateway is a legitimate config (LLP 0120: hermes composes
+ * the plugin for its materializer alone and contributes no upstream), so the
+ * source idles instead of failing to start. That trade turns one class of
+ * misconfiguration silent: a config that *did* list upstreams and lost them
+ * all to `compileUpstreams` (an entry missing either required key is dropped
+ * without complaint, and `diagnoseV1Config`'s `gateway_missing_*_upstream`
+ * check does not fire for that shape, since it matches on `provider` too) also
+ * idles, reporting `started` and `healthy` while the user's client gets
+ * ECONNREFUSED.
+ *
+ * The *count* is what separates them, not the names. `compileUpstreams` drops
+ * an entry for a missing `name` exactly as silently as for a missing
+ * `base_url`, and a nameless entry contributes nothing to `details.upstreams`,
+ * so a config of `provider = "anthropic", base_url = "..."` looks identical to
+ * hermes-only through the names alone. `details.upstreams_configured` counts
+ * the entries the config listed whatever shape they were in, so hermes-only
+ * yields 0 and any dropped upstream yields at least 1. The names still ride
+ * along, pre-compile, because they make the warning concrete when they exist.
+ *
+ * A status file written before `upstreams_configured` existed carries names
+ * only; those still count for themselves, so an older daemon's dropped
+ * `base_url` stays visible.
+ *
+ * @param {SourceSnapshot[] | undefined} sources
+ * @returns {{ count: number, names: string[] } | undefined}
+ */
+function gatewayIdleWithConfiguredUpstreams(sources) {
+  const details = gatewaySourceRawDetails(sources)
+  if (!details || details.listening !== false) return undefined
+  const upstreams = details.upstreams
+  const names = Array.isArray(upstreams)
+    ? /** @type {string[]} */ (upstreams.filter((u) => typeof u === 'string' && u.length > 0))
+    : []
+  const rawCount = details.upstreams_configured
+  const count =
+    typeof rawCount === 'number' && Number.isInteger(rawCount) && rawCount >= 0
+      ? rawCount
+      : names.length
+  return count > 0 ? { count, names } : undefined
+}
+
 /**
  * How many recent client surfaces `hyp status` will report. The gateway keeps
  * its own, deliberately equal, cap on the writing side; this one exists
@@ -598,6 +658,42 @@ export async function collectHypAwareStatus(opts = {}) {
       repair: [`free ${from} and restart the daemon - attached clients re-point automatically`],
     })
   }
+  const idleGatewayUpstreams = daemon.running
+    ? gatewayIdleWithConfiguredUpstreams(daemonStatusFile?.sources)
+    : undefined
+  if (idleGatewayUpstreams) {
+    // The gateway bound nothing while the config listed upstreams it wanted
+    // proxied: every one was dropped at compile, so there is no listener and
+    // no error either. Before the source was allowed to idle this was a source
+    // start failure and `hyp status` said `[failed]`; the same install must not
+    // now read `[started]` / `healthy` with the reason living only in a log
+    // line. Non-degrading like `gateway_port_fallback`: an install that
+    // *wanted* no upstream (hermes-only) reports no configured upstreams here
+    // and never reaches this branch, so it stays healthy and silent.
+    // @ref LLP 0114#fallback-is-visible [implements]: an exception to "the gateway is listening" is readable from status.json steadily, not only from a boot-time log line
+    const { count, names } = idleGatewayUpstreams
+    // Count first, names in parentheses when there are any: `name` is itself
+    // one of the two keys that drops an entry, so the config that most needs
+    // this warning is exactly the one that can supply no name to print.
+    const named = names.length > 0 ? ` (${names.join(', ')})` : ''
+    diagnostics.push({
+      severity: 'warning',
+      kind: 'gateway_idle_no_upstreams',
+      message: `the gateway is running but listening on nothing: ${count} ${count === 1 ? 'upstream' : 'upstreams'}${named} ${count === 1 ? 'is' : 'are'} configured but none compiled to a route (each needs both a 'name' and a 'base_url') - clients will get connection refused`,
+      // Not `hyp config validate`: it prints `config ok` for this config and
+      // exits 0. `@hypaware/ai-gateway` registers no config section, so
+      // nothing validates upstream shape, and `diagnoseV1Config` matches an
+      // upstream by its `provider` field, so a nameless anthropic entry
+      // satisfies the one check that does look. A repair line that sends the
+      // user to a command which affirms the broken config is worse than no
+      // repair line, so point at the file and the two required keys instead.
+      // @ref LLP 0139#repair-must-be-runnable [constrained-by]: a repair has to be a step that changes something, so the inert validate command gives way to the edit that fixes it
+      repair: [
+        `add the missing 'name' / 'base_url' to each upstream in ${configPath} ('hyp config validate' does not check upstream shape)`,
+        `hyp daemon restart  # the daemon reads the file only at boot`,
+      ],
+    })
+  }
   /** @type {ClientAttachReport[]} */
   const clients = []
   const clientDescriptors = catalog?.clientDescriptors ?? new Map()

src/core/daemon/types.d.ts

@@ -99,6 +99,7 @@ export type StatusDiagnosticKind =
   | 'client_attach_stale'
   | 'client_attached_not_configured'
   | 'gateway_port_fallback'
+  | 'gateway_idle_no_upstreams'
   | 'recent_errors'
   | 'remote_config_rolled_back'
   | 'local_only_list_unreadable'