Phase 2: job-scoped same-realm search cache during indexing

packages/host/app/routes/render.ts

@@ -214,6 +214,24 @@ export default class RenderRoute extends Route<Model> {
     let parsedOptions = parseRenderRouteOptions(options);
     let canonicalOptions = serializeRenderRouteOptions(parsedOptions);
     this.#setupTransitionHelper(id, nonce, canonicalOptions);
+    // Stamp the "consuming realm" — the realm that owns the card being
+    // rendered — onto a global the store-service's federated-search
+    // wrapper reads. The realm-server's job-scoped search cache pairs
+    // this with `x-boxel-job-id` to gate same-realm-only caching:
+    // cross-realm reads bypass the cache because peer realms can swap
+    // independently.
+    try {
+      let consumingRealm = this.realm.realmOf(new URL(id));
+      (
+        globalThis as unknown as { __boxelConsumingRealm?: string }
+      ).__boxelConsumingRealm = consumingRealm
+        ? String(consumingRealm)
+        : undefined;
+    } catch {
+      (
+        globalThis as unknown as { __boxelConsumingRealm?: string }
+      ).__boxelConsumingRealm = undefined;
+    }
     // CS-10872: render-stage breadcrumb. `model()` running means we
     // made it past route setup and are about to build the render
     // model. Each long-running stage below updates this slot so the

packages/host/app/services/store.ts

@@ -33,6 +33,8 @@ import {
   isSingleCardDocument,
   isLinkableCollectionDocument,
   resolveFileDefCodeRef,
+  X_BOXEL_CONSUMING_REALM_HEADER,
+  X_BOXEL_JOB_ID_HEADER,
   Deferred,
   delay,
   mergeRelationships,
@@ -130,6 +132,33 @@ function duringPrerenderHeaders(): Record<string, string> {
     .__boxelDuringPrerender;
   return flag ? { [DURING_PRERENDER_HEADER]: '1' } : {};
 }
+
+// While rendering inside a prerender tab the render route writes
+// `__boxelConsumingRealm` with the URL of the realm whose card is being
+// rendered. Attach it to outbound `_federated-search` requests so the
+// realm-server's job-scoped cache layer can gate same-realm-only
+// caching. Read each fetch (not cached at module scope) so a tab that
+// renders cards from multiple realms in sequence sends the correct
+// header per request. Returns an empty object when the global is not
+// set so non-prerender (live SPA) fetches behave exactly as before.
+function consumingRealmHeader(): Record<string, string> {
+  let r = (globalThis as unknown as { __boxelConsumingRealm?: string })
+    .__boxelConsumingRealm;
+  return r ? { [X_BOXEL_CONSUMING_REALM_HEADER]: r } : {};
+}
+
+// Companion to `consumingRealmHeader()`. The prerender server's
+// `prerenderVisitAttempt` injects `__boxelJobId` onto the page before
+// transitioning into the render route — see
+// `packages/realm-server/prerender/render-runner.ts`. Read it on each
+// fetch (not module-scope-cached) so a page reused across multiple
+// visits picks up the current visit's job id. Outside a prerender
+// tab the global is undefined and we send no header, so user / API
+// callers continue to bypass the realm-server's job-scoped cache.
+function jobIdHeader(): Record<string, string> {
+  let j = (globalThis as unknown as { __boxelJobId?: string }).__boxelJobId;
+  return j ? { [X_BOXEL_JOB_ID_HEADER]: j } : {};
+}
 const queryFieldSeedFromSearchSymbol = Symbol.for(
   'cardstack-query-field-seed-from-search',
 );
@@ -840,6 +869,8 @@ export default class StoreService extends Service implements StoreInterface {
           Accept: SupportedMimeType.CardJson,
           'Content-Type': 'application/json',
           ...duringPrerenderHeaders(),
+          ...consumingRealmHeader(),
+          ...jobIdHeader(),
         },
         body: JSON.stringify({ ...query, realms }),
       },
@@ -907,6 +938,8 @@ export default class StoreService extends Service implements StoreInterface {
           Accept: SupportedMimeType.CardJson,
           'Content-Type': 'application/json',
           ...duringPrerenderHeaders(),
+          ...consumingRealmHeader(),
+          ...jobIdHeader(),
         },
         body: JSON.stringify({ ...query, realms }),
       },

packages/realm-server/handlers/handle-search.ts

@@ -3,8 +3,10 @@ import {
   buildSearchErrorResponse,
   DURING_PRERENDER_HEADER,
   SupportedMimeType,
+  X_BOXEL_CONSUMING_REALM_HEADER,
   parseSearchQueryFromPayload,
   parseSearchQueryFromRequest,
+  sanitizeConsumingRealmHeader,
   SearchRequestError,
   searchRealms,
 } from '@cardstack/runtime-common';
@@ -17,8 +19,16 @@ import {
   getMultiRealmAuthorization,
   getSearchRequestPayload,
 } from '../middleware/multi-realm-authorization';
+import type { JobScopedSearchCache } from '../job-scoped-search-cache';
+import {
+  PRERENDER_JOB_ID_HEADER,
+  sanitizePrerenderJobId,
+} from '../prerender/prerender-constants';
 
-export default function handleSearch(): (ctxt: Koa.Context) => Promise<void> {
+export default function handleSearch(opts?: {
+  searchCache?: JobScopedSearchCache;
+}): (ctxt: Koa.Context) => Promise<void> {
+  let searchCache = opts?.searchCache;
   return async function (ctxt: Koa.Context) {
     let { realmList, realmByURL } = getMultiRealmAuthorization(ctxt);
 
@@ -43,11 +53,47 @@ export default function handleSearch(): (ctxt: Koa.Context) => Promise<void> {
     }
 
     let cacheOnlyDefinitions = ctxt.get(DURING_PRERENDER_HEADER).length > 0;
-    let combined = await searchRealms(
-      realmList.map((realmURL) => realmByURL.get(realmURL)),
-      cardsQuery,
-      cacheOnlyDefinitions ? { cacheOnlyDefinitions: true } : undefined,
-    );
+    let searchOpts = cacheOnlyDefinitions
+      ? { cacheOnlyDefinitions: true }
+      : undefined;
+    let runSearch = () =>
+      searchRealms(
+        realmList.map((realmURL) => realmByURL.get(realmURL)),
+        cardsQuery,
+        searchOpts,
+      );
+
+    // Job-scoped same-realm cache. Gated on all three:
+    //   (a) `x-boxel-job-id` is present and well-formed (only the
+    //       indexer worker stamps this; live user / API callers never
+    //       carry it and therefore always see fresh data),
+    //   (b) `x-boxel-consuming-realm` is present and well-formed (the
+    //       host's render route only sets it during prerender),
+    //   (c) the request's `realms` list is exactly `[consumingRealm]`
+    //       — cross-realm reads bypass the cache because a peer
+    //       realm can swap its `boxel_index` mid-batch and the cached
+    //       value would freeze a stale snapshot.
+    let jobId = searchCache
+      ? sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER))
+      : null;
+    let consumingRealm = searchCache
+      ? sanitizeConsumingRealmHeader(ctxt.get(X_BOXEL_CONSUMING_REALM_HEADER))
+      : null;
+    let cacheable =
+      searchCache &&
+      jobId &&
+      consumingRealm &&
+      realmList.length === 1 &&
+      realmList[0] === consumingRealm;
+
+    let combined = cacheable
+      ? await searchCache!.getOrPopulate({
+          jobId: jobId!,
+          query: cardsQuery,
+          opts: searchOpts,
+          populate: runSearch,
+        })
+      : await runSearch();
 
     await setContextResponse(
       ctxt,

packages/realm-server/job-scoped-search-cache.ts

@@ -0,0 +1,205 @@
+import {
+  normalizeQueryForSignature,
+  sortKeysDeep,
+  type LinkableCollectionDocument,
+  type Query,
+} from '@cardstack/runtime-common';
+
+// Default entry TTL. Picked to comfortably outlive a single indexing
+// batch (workers cap from-scratch jobs at 6 min, incremental jobs are
+// shorter) while bounding the worst case where a job ends without a
+// NOTIFY-driven eviction reaching this process — a leaked entry persists
+// at most this long. Cross-job collision is impossible because the cache
+// key includes `jobId`, so a stale leak only hurts memory, never
+// correctness.
+const DEFAULT_TTL_MS = 10 * 60 * 1000;
+
+// Hard cap on total entries across all jobs. When the cap is reached
+// the FIFO-oldest entry is evicted to make room. Cap exists to bound
+// worst-case memory: the `jobId` header is sanitized to a digits-only
+// shape but the cache otherwise accepts any well-formed
+// `(jobId, query, opts)` tuple from an authenticated caller, so a
+// reader who mints synthetic jobIds and varied queries could otherwise
+// grow the cache without bound for the full TTL window. Picked to
+// comfortably accommodate the busiest realistic workload (a from-
+// scratch reindex of a piranha-class realm fires hundreds of distinct
+// queries within one job) while keeping worst-case footprint bounded
+// to ~tens of MB.
+const DEFAULT_MAX_ENTRIES = 5000;
+
+type CachedEntry = {
+  result: LinkableCollectionDocument;
+  timer: ReturnType<typeof setTimeout>;
+  // Position in the FIFO eviction ring. Stored on the entry so a
+  // cache hit doesn't need a separate map lookup to know its slot.
+  fifoSeq: number;
+};
+
+// Same-realm read cache used during indexing. Each entry is keyed by
+// `(jobId, normalizedQuery, normalizedOpts)` and represents one
+// `_federated-search` populate computed during the lifetime of one
+// indexing job. Safe because within an indexing batch the writer
+// touches `boxel_index_working`, not `boxel_index` — so every read of
+// the same realm's `boxel_index` returns identical bytes until the
+// batch's `applyBatchUpdates` swap fires. The job-id boundary scopes
+// the cache to a single batch; a subsequent job hashes to different
+// keys and never reuses a stale value.
+//
+// The handler gates entry into this cache on three conditions all
+// holding: `x-boxel-job-id` present, `x-boxel-consuming-realm` present,
+// and the request's `realms` array is exactly `[consumingRealm]`.
+// Cross-realm reads bypass the cache because peer realms can swap
+// independently — a cached read against a foreign realm could freeze
+// a stale snapshot. Anonymous (no jobId) reads also bypass: those
+// callers are not inside the batch's snapshot-stable read window and
+// must always see live state.
+//
+// Entries store the *resolved* doc, not the in-flight promise.
+// Concurrent same-key callers each run their own `populate` (Phase 1's
+// in-flight dedup at `RealmIndexQueryEngine.searchCards` already
+// coalesces the heavy inner SQL+loadLinks walk for same-realm calls
+// arriving concurrently). The first to finish stores its result here;
+// later sequential callers within the same job see the cached doc and
+// short-circuit before re-entering `searchRealms`.
+//
+// Storing promises was tempting (it would also dedupe at this layer)
+// but creates a tail-latency stall: a slow first populate blocks every
+// later same-key caller past their render-timeout window, even when
+// they could otherwise have run their own search in parallel and made
+// progress. Resolved-only avoids that failure mode and keeps the
+// benefit of sequential dedup, which is the win this cache exists for.
+export class JobScopedSearchCache {
+  #byJob = new Map<string, Map<string, CachedEntry>>();
+  // FIFO ring keyed by an ever-incrementing sequence so eviction
+  // ordering survives the (jobId, innerKey) name space. The oldest
+  // surviving sequence number is `#evictionCursor`; advances as the
+  // entry it points at is evicted (either via cap or its own TTL).
+  #fifo = new Map<number, { jobId: string; innerKey: string }>();
+  #nextFifoSeq = 0;
+  readonly #ttlMs: number;
+  readonly #maxEntries: number;
+
+  constructor(opts?: { ttlMs?: number; maxEntries?: number }) {
+    this.#ttlMs = opts?.ttlMs ?? DEFAULT_TTL_MS;
+    this.#maxEntries = opts?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+  }
+
+  async getOrPopulate(args: {
+    jobId: string;
+    query: Query;
+    opts: unknown | undefined;
+    populate: () => Promise<LinkableCollectionDocument>;
+  }): Promise<LinkableCollectionDocument> {
+    let innerKey = buildInnerKey(args.query, args.opts);
+    let jobMap = this.#byJob.get(args.jobId);
+    let existing = jobMap?.get(innerKey);
+    if (existing) {
+      return existing.result;
+    }
+
+    let result = await args.populate();
+
+    // Late-arriving check: the populate may have just settled while a
+    // peer's populate (same key) also settled and stored its result
+    // first. Last-write-wins; either of the two resolved docs is
+    // equally valid since they came from the same `(jobId, query)`
+    // tuple against the same snapshot-stable boxel_index.
+    let currentJobMap = this.#byJob.get(args.jobId);
+    if (!currentJobMap) {
+      currentJobMap = new Map();
+      this.#byJob.set(args.jobId, currentJobMap);
+    }
+    let prior = currentJobMap.get(innerKey);
+    if (prior) {
+      clearTimeout(prior.timer);
+      this.#fifo.delete(prior.fifoSeq);
+    }
+    let fifoSeq = this.#nextFifoSeq++;
+    let timer = setTimeout(() => {
+      this.#evictByKey(args.jobId, innerKey, timer);
+    }, this.#ttlMs);
+    if (typeof (timer as { unref?: () => void }).unref === 'function') {
+      (timer as { unref: () => void }).unref();
+    }
+    currentJobMap.set(innerKey, { result, timer, fifoSeq });
+    this.#fifo.set(fifoSeq, { jobId: args.jobId, innerKey });
+
+    // Cap enforcement: evict FIFO-oldest until under the limit. Map
+    // preserves insertion order, so the first key is the oldest. We
+    // skip-over any keys whose entries are already gone (TTL fired)
+    // without rewriting the ring.
+    while (this.#fifo.size > this.#maxEntries) {
+      let oldestSeq = this.#fifo.keys().next().value;
+      if (oldestSeq === undefined) break;
+      let slot = this.#fifo.get(oldestSeq)!;
+      this.#fifo.delete(oldestSeq);
+      let jm = this.#byJob.get(slot.jobId);
+      let entry = jm?.get(slot.innerKey);
+      if (entry?.fifoSeq === oldestSeq) {
+        clearTimeout(entry.timer);
+        jm!.delete(slot.innerKey);
+        if (jm!.size === 0) {
+          this.#byJob.delete(slot.jobId);
+        }
+      }
+    }
+
+    return result;
+  }
+
+  #evictByKey(
+    jobId: string,
+    innerKey: string,
+    expectedTimer: ReturnType<typeof setTimeout>,
+  ): void {
+    let jm = this.#byJob.get(jobId);
+    if (!jm) return;
+    let entry = jm.get(innerKey);
+    if (entry?.timer === expectedTimer) {
+      this.#fifo.delete(entry.fifoSeq);
+      jm.delete(innerKey);
+      if (jm.size === 0) {
+        this.#byJob.delete(jobId);
+      }
+    }
+  }
+
+  // Drop every entry for a given job. Wired in by the NOTIFY-driven
+  // eviction path so the cache releases memory as soon as the worker
+  // signals job completion, rather than waiting on TTL.
+  clearJob(jobId: string): void {
+    let jobMap = this.#byJob.get(jobId);
+    if (!jobMap) return;
+    for (let entry of jobMap.values()) {
+      clearTimeout(entry.timer);
+      this.#fifo.delete(entry.fifoSeq);
+    }
+    this.#byJob.delete(jobId);
+  }
+
+  // Total entry count across all jobs. Useful for tests + observability.
+  size(): number {
+    let total = 0;
+    for (let jm of this.#byJob.values()) {
+      total += jm.size;
+    }
+    return total;
+  }
+
+  jobIds(): string[] {
+    return [...this.#byJob.keys()];
+  }
+}
+
+// Compose the per-job inner key. Excludes jobId since the outer Map is
+// already partitioned by jobId — this keeps inner-key length bounded
+// regardless of how the call site formats the jobId. Excludes the
+// realms array (the cache gate already enforces same-realm-only), so
+// two requests with `realms: [R]` produce the same inner key
+// regardless of array identity.
+function buildInnerKey(query: Query, opts: unknown | undefined): string {
+  return JSON.stringify([
+    normalizeQueryForSignature(query),
+    opts ? sortKeysDeep(opts) : null,
+  ]);
+}

packages/realm-server/prerender/prerender-app.ts

@@ -741,6 +741,14 @@ export function buildPrerenderApp(options: {
           ? rawBatchId
           : undefined;
 
+      // Indexer job correlation id. Already carried on the inbound
+      // `x-boxel-job-id` header for log-tagging; forward it to the
+      // prerenderer so it can be exposed to the host SPA via a global
+      // (`__boxelJobId`) — the host's `_federated-search` fetch
+      // wrapper reads it and re-stamps the header on outbound calls
+      // so `handle-search` can gate the job-scoped search cache.
+      let jobId = sanitizePrerenderJobId(ctxt.get(PRERENDER_JOB_ID_HEADER));
+
       let start = Date.now();
       let execPromise = prerenderer
         .prerenderVisit({
@@ -754,6 +762,7 @@ export function buildPrerenderApp(options: {
           ...(Array.isArray(types) ? { types } : {}),
           ...(batchId ? { batchId } : {}),
           ...(priority !== undefined ? { priority } : {}),
+          ...(jobId ? { jobId } : {}),
           signal: ac.signal,
         })
         .then((result) => ({ result }));