perf(data): stop query-debug diagnostics convoying the SQLite worker

.changeset/query-debug-diagnostics-convoy.md

@@ -0,0 +1,15 @@
+---
+'@xnetjs/sqlite': patch
+'@xnetjs/data': patch
+---
+
+Query-plan debug diagnostics no longer convoy the SQLite worker. With
+`xnet:query:debug` enabled, every query used to issue EXPLAIN QUERY PLAN +
+PRAGMA schema_version + one PRAGMA index_info per index as separate serial
+worker round-trips — hundreds per boot, delaying real query results by
+18-20s. `getIndexInfo` now dedupes concurrent callers onto one in-flight
+build and fetches all index metadata in a single batched
+`pragma_index_info` join (with a per-index fallback for runtimes without
+table-valued pragmas), and the storage adapter collects plan diagnostics
+once per unique compiled SQL shape per session instead of per execution
+(invalidated when adaptive indexes are created or dropped).

packages/data/src/store/sqlite-adapter.test.ts

@@ -2052,6 +2052,55 @@ describe('SQLiteNodeStorageAdapter', () => {
       }
     })
 
+    it('throttles plan diagnostics to one collection per compiled SQL shape (2026-07-05 convoy)', async () => {
+      // Pre-fix, EVERY debug-mode execution issued EXPLAIN QUERY PLAN +
+      // PRAGMA schema_version + the index inventory as separate round-trips
+      // on the single serial worker — hundreds per boot, delaying the very
+      // queries being measured by 18-20s. Diagnostics must be collected once
+      // per unique compiled SQL shape and served from the session memo after.
+      const throttledAdapter = new SQLiteNodeStorageAdapter(db, { queryDiagnostics: true })
+      const querySpy = vi.spyOn(db, 'query')
+      const explainCount = () =>
+        querySpy.mock.calls.filter(([sql]) => String(sql).startsWith('EXPLAIN QUERY PLAN')).length
+
+      const descriptor = {
+        schemaId: taskSchemaId,
+        includeDeleted: false,
+        where: { status: 'open' }
+      }
+
+      const first = await throttledAdapter.queryNodes(descriptor)
+      expect(first.plan.usedIndexNames).toBeDefined()
+      expect(first.plan.availableIndexCount).toBeGreaterThan(0)
+      expect(explainCount()).toBe(1)
+
+      // Same shape again — and the same shape with different bound values
+      // (values are `?` params, so the compiled SQL is identical): both are
+      // served from the memo, with diagnostics still present on every plan.
+      const second = await throttledAdapter.queryNodes(descriptor)
+      const third = await throttledAdapter.queryNodes({
+        ...descriptor,
+        where: { status: 'done' }
+      })
+      expect(second.plan.usedIndexNames).toBeDefined()
+      expect(third.plan.usedIndexNames).toBeDefined()
+      expect(explainCount()).toBe(1)
+
+      // Concurrent cold executions (the boot pattern) share one in-flight
+      // collection instead of each enqueueing their own.
+      const coldAdapter = new SQLiteNodeStorageAdapter(db, { queryDiagnostics: true })
+      const before = explainCount()
+      const results = await Promise.all(
+        Array.from({ length: 8 }, () => coldAdapter.queryNodes(descriptor))
+      )
+      for (const result of results) {
+        expect(result.plan.usedIndexNames).toBeDefined()
+      }
+      expect(explainCount()).toBe(before + 1)
+
+      querySpy.mockRestore()
+    })
+
     it('skips plan diagnostics by default', async () => {
       const defaultAdapter = new SQLiteNodeStorageAdapter(db)
 

packages/data/src/store/sqlite-adapter.ts

@@ -151,9 +151,13 @@ export interface SQLiteNodeStorageAdapterOptions {
   adaptiveIndexing?: SQLiteAdaptiveIndexingOptions
   queryVerification?: SQLiteQueryVerificationOptions
   /**
-   * Collect EXPLAIN QUERY PLAN + index inventory per query. Costs extra
-   * round trips per query, so it is off unless explicitly enabled or the
-   * `xnet:query:debug` localStorage flag is set.
+   * Collect EXPLAIN QUERY PLAN + index inventory for queries. Costs extra
+   * round trips, so it is off unless explicitly enabled or the
+   * `xnet:query:debug` localStorage flag is set. Diagnostics are collected
+   * once per unique compiled SQL shape per session (invalidated when the
+   * adapter itself runs DDL) — per-execution collection convoyed the serial
+   * worker and delayed the very queries being measured by 18-20s at boot
+   * (2026-07-05 capture).
    */
   queryDiagnostics?: boolean
   /**
@@ -316,6 +320,12 @@ const DEFAULT_QUERY_VERIFICATION: QueryVerificationConfig = {
 
 const QUERY_TELEMETRY_FLUSH_THRESHOLD = 50
 
+// Distinct compiled SQL shapes are usually few (dozens), but IN-list binds
+// mint one shape per list length, so a long debug session can keep growing.
+// Evict oldest-first past this — recomputing a plan later is cheap; holding
+// thousands of memo entries is not.
+const COMPILED_QUERY_DIAGNOSTICS_MEMO_LIMIT = 512
+
 interface PendingQueryTelemetry {
   schemaId: string
   descriptorJson: string
@@ -429,6 +439,15 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter {
 
   private storageCapabilitiesPromise?: Promise<NodeQueryStorageCapabilitiesMetadata>
 
+  /**
+   * Plan diagnostics memoized per compiled SQL shape (the string EXPLAINed),
+   * shared across executions AND concurrent callers. Cleared when this adapter
+   * creates/drops an adaptive index, since that changes plans. Debug mode must
+   * not distort what it measures: per-execution EXPLAIN + index-inventory
+   * round-trips convoyed the single serial worker at boot.
+   */
+  private compiledQueryDiagnosticsMemo = new Map<string, Promise<CompiledQueryDiagnostics>>()
+
   private spatialTablesState: SpatialTablesState = 'unknown'
 
   private fullTextSearchTablesState: FullTextSearchTablesState = 'unknown'
@@ -3282,7 +3301,38 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema
     return this.queryDiagnostics || this.isQueryDebugEnabled()
   }
 
-  private async collectCompiledQueryDiagnostics(
+  /**
+   * Throttled: one collection per unique compiled SQL shape per session.
+   * EXPLAIN plans depend on the statement, not the bound values, so keyset
+   * pages of the same query share one entry. Concurrent callers share the
+   * in-flight promise; failed collections are not memoized.
+   */
+  private collectCompiledQueryDiagnostics(
+    compiled: CompiledNodeQuery
+  ): Promise<CompiledQueryDiagnostics> {
+    const memoKey = compiled.sql
+    const memoized = this.compiledQueryDiagnosticsMemo.get(memoKey)
+    if (memoized) {
+      return memoized
+    }
+
+    const pending = this.computeCompiledQueryDiagnostics(compiled).then((diagnostics) => {
+      if (diagnostics.diagnosticsError) {
+        this.compiledQueryDiagnosticsMemo.delete(memoKey)
+      }
+      return diagnostics
+    })
+    if (this.compiledQueryDiagnosticsMemo.size >= COMPILED_QUERY_DIAGNOSTICS_MEMO_LIMIT) {
+      const oldest = this.compiledQueryDiagnosticsMemo.keys().next().value
+      if (oldest !== undefined) {
+        this.compiledQueryDiagnosticsMemo.delete(oldest)
+      }
+    }
+    this.compiledQueryDiagnosticsMemo.set(memoKey, pending)
+    return pending
+  }
+
+  private async computeCompiledQueryDiagnostics(
     compiled: CompiledNodeQuery
   ): Promise<CompiledQueryDiagnostics> {
     try {
@@ -3574,6 +3624,8 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema
     if (createdIndex) {
       await runAnalyze(this.db, 'node_property_scalars')
       await this.db.exec('PRAGMA optimize')
+      // New index → plans may change; memoized diagnostics are stale.
+      this.compiledQueryDiagnosticsMemo.clear()
     }
 
     return touchedIndexNames
@@ -3786,6 +3838,8 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema
   ): Promise<void> {
     await this.db.exec(`DROP INDEX IF EXISTS ${this.quoteSqlIdentifier(indexName)}`)
     await this.db.run('DELETE FROM query_index_candidates WHERE index_name = ?', [indexName])
+    // Dropped index → plans may change; memoized diagnostics are stale.
+    this.compiledQueryDiagnosticsMemo.clear()
     this.debugAdaptiveIndex('drop', { indexName, reason })
   }