Break prerender deadlock during indexing-time federated search

packages/host/app/services/store.ts

@@ -23,6 +23,7 @@ import {
   baseFileRef,
   CardError,
   cardIdToURL,
+  DURING_PRERENDER_HEADER,
   isRegisteredPrefix,
   hasExecutableExtension,
   isCardError,
@@ -117,6 +118,18 @@ let waiter = buildWaiter('store-service');
 
 const realmEventsLogger = logger('realm:events');
 const storeLogger = logger('store');
+
+// Set by the prerender server's `evaluateOnNewDocument` before the
+// SPA boots — `__boxelDuringPrerender = true`. Read here so the
+// federated-search fetch wrapper can attach the marker header on
+// realm-server-bound calls only, narrowly scoping the signal to the
+// endpoint that needs it. See realm.ts:DURING_PRERENDER_HEADER for
+// the full chain.
+function duringPrerenderHeaders(): Record<string, string> {
+  let flag = (globalThis as unknown as { __boxelDuringPrerender?: boolean })
+    .__boxelDuringPrerender;
+  return flag ? { [DURING_PRERENDER_HEADER]: '1' } : {};
+}
 const queryFieldSeedFromSearchSymbol = Symbol.for(
   'cardstack-query-field-seed-from-search',
 );
@@ -826,6 +839,7 @@ export default class StoreService extends Service implements StoreInterface {
         headers: {
           Accept: SupportedMimeType.CardJson,
           'Content-Type': 'application/json',
+          ...duringPrerenderHeaders(),
         },
         body: JSON.stringify({ ...query, realms }),
       },
@@ -892,6 +906,7 @@ export default class StoreService extends Service implements StoreInterface {
         headers: {
           Accept: SupportedMimeType.CardJson,
           'Content-Type': 'application/json',
+          ...duringPrerenderHeaders(),
         },
         body: JSON.stringify({ ...query, realms }),
       },

packages/postgres/pg-adapter.ts

@@ -31,14 +31,33 @@ function config() {
 
 type Config = ReturnType<typeof config>;
 
-function configuredPoolMax(): number | undefined {
+// node-postgres' default pool max is 10. That's too small for the
+// realm-server under parallel indexing — a single file render can
+// fire several federated-search calls, each running primaryQuery +
+// loadLinks-layer queries that each hold a connection for the
+// duration of the SQL round-trip. With INDEX_RUNNER_MAX_CONCURRENCY=4
+// renders in flight at peak, observed pg connection demand reaches
+// 20+ — and any waiter past max sits in node-postgres' internal
+// acquire queue, which is indistinguishable from "the SQL is slow"
+// in diagnostic logs (we saw primaryQuery=73s for queries returning
+// 3 rows during the ambitious-piranha benchmark). 40 gives a margin
+// over that peak for non-search realm-server work (advisory locks,
+// indexer writes, NOTIFY dispatch) so a search burst doesn't crowd
+// out the indexer's own commits. Hosted RDS sizing (staging
+// db.r7g.large ≈ 1700, prod db.r7g.xlarge ≈ 3500 default
+// max_connections) leaves plenty of headroom even with 4-6 client
+// processes each opening their own pool. Operators can raise it
+// further via the env var for fleets with bigger pg instances; lower
+// it to throttle a noisy realm.
+const DEFAULT_POOL_MAX = 40;
+function configuredPoolMax(): number {
   let rawValue = process.env.PG_POOL_MAX;
   if (!rawValue) {
-    return undefined;
+    return DEFAULT_POOL_MAX;
   }
 
   let value = Number(rawValue);
-  return Number.isInteger(value) && value > 0 ? value : undefined;
+  return Number.isInteger(value) && value > 0 ? value : DEFAULT_POOL_MAX;
 }
 
 export type NotificationHandler = (notification: Notification) => void;
@@ -90,7 +109,7 @@ export class PgAdapter implements DBAdapter {
       database,
       password,
       port,
-      ...(max ? { max } : {}),
+      max,
     });
   }
 

packages/realm-server/handlers/handle-search.ts

@@ -1,6 +1,7 @@
 import type Koa from 'koa';
 import {
   buildSearchErrorResponse,
+  DURING_PRERENDER_HEADER,
   SupportedMimeType,
   parseSearchQueryFromPayload,
   parseSearchQueryFromRequest,
@@ -41,9 +42,11 @@ export default function handleSearch(): (ctxt: Koa.Context) => Promise<void> {
       throw e;
     }
 
+    let cacheOnlyDefinitions = ctxt.get(DURING_PRERENDER_HEADER).length > 0;
     let combined = await searchRealms(
       realmList.map((realmURL) => realmByURL.get(realmURL)),
       cardsQuery,
+      cacheOnlyDefinitions ? { cacheOnlyDefinitions: true } : undefined,
     );
 
     await setContextResponse(

packages/realm-server/prerender/page-pool.ts

@@ -1256,6 +1256,7 @@ export class PagePool {
           'Expected each prerender page to use its own browser context for localStorage isolation',
         );
       }
+      await this.#markPageAsInPrerender(page);
       let pageId = uuidv4();
       await this.#attachPageObservability(page, 'standby', pageId);
       await this.#loadStandbyPage(page, pageId);
@@ -1845,6 +1846,7 @@ export class PagePool {
     let page: Page | undefined;
     try {
       page = await shared.context.newPage();
+      await this.#markPageAsInPrerender(page);
       let pageId = uuidv4();
       await this.#attachPageObservability(page, affinityKey, pageId);
       await this.#loadStandbyPage(page, pageId);
@@ -2191,6 +2193,31 @@ export class PagePool {
     }
   }
 
+  // Inject a window global into every page before any document
+  // loads, so the host SPA can synchronously detect at boot that it's
+  // running inside a prerender tab. Used by the host's
+  // realm-server fetch wrapper to attach `x-boxel-during-prerender`
+  // on _federated-search / _search calls only — narrowly scoped so
+  // unrelated services (icons, vite, etc.) don't see the header on
+  // their CORS preflights. The inbound signal the realm server reads
+  // tells it to pass cacheOnlyDefinitions:true to searchCards,
+  // short-circuiting the recursive lookupDefinition fan-out in
+  // populateQueryFields that causes self-referential prerender
+  // deadlocks under parallel indexing.
+  async #markPageAsInPrerender(page: Page): Promise<void> {
+    // Test stubs of Page in prerender-deadlock-test.ts don't expose
+    // every puppeteer method — guard the call so this stays optional
+    // observability rather than a load-bearing side effect.
+    if (typeof page.evaluateOnNewDocument !== 'function') {
+      return;
+    }
+    await page.evaluateOnNewDocument(() => {
+      (
+        globalThis as unknown as { __boxelDuringPrerender?: boolean }
+      ).__boxelDuringPrerender = true;
+    });
+  }
+
   // Attach all per-page error/exception observability surfaces. The
   // console-message listener catches things that surfaced via the JS
   // event layer (page logs, Chrome's late "Uncaught (in promise)..."

packages/realm-server/prerender/prerender-constants.ts

@@ -36,6 +36,22 @@ export function sanitizePrerenderRequestId(
 // in worker logs.
 export const PRERENDER_JOB_ID_HEADER = 'x-boxel-job-id';
 
+// Stamped on the host's outbound _federated-search / _search calls
+// when the host SPA detects it's running inside a prerender tab. The
+// prerender server signals "you are in a prerender" by injecting
+// `globalThis.__boxelDuringPrerender = true` via evaluateOnNewDocument
+// before the host SPA boots. The host's realm-server fetch wrapper
+// reads that flag and attaches this header to the request; the
+// search handlers read it inbound and pass `cacheOnlyDefinitions:true`
+// to searchCards, short-circuiting the recursive lookupDefinition
+// fan-out in populateQueryFields that causes self-referential
+// prerender deadlocks under parallel indexing.
+//
+// Defined in runtime-common's realm.ts as the single source of truth
+// so the Realm class can read it without depending on realm-server.
+// Re-exported here for the host fetch wrapper's import-locality.
+export { DURING_PRERENDER_HEADER } from '@cardstack/runtime-common';
+
 // Sanitize the inbound job-id header. Format is `<digits>.<digits>`
 // (job.id + reservation.id, both bigint-shaped); accept up to 32
 // digits per side (so up to 65 chars total including the separator)

packages/realm-server/server.ts

@@ -194,7 +194,7 @@ export class RealmServer {
         cors({
           origin: '*',
           allowHeaders:
-            'Authorization, Content-Type, If-Match, If-None-Match, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename',
+            'Authorization, Content-Type, If-Match, If-None-Match, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename, X-Boxel-During-Prerender',
           allowMethods: 'GET,HEAD,PUT,POST,DELETE,PATCH,OPTIONS,QUERY',
         }),
       )

packages/realm-test-harness/src/shared.ts

@@ -188,10 +188,16 @@ export const DEFAULT_ICONS_PROBE_URL = new URL(
 export const DEFAULT_PG_PORT = process.env.TEST_HARNESS_PGPORT ?? '55436';
 export const DEFAULT_PG_HOST = process.env.TEST_HARNESS_PGHOST ?? '127.0.0.1';
 export const DEFAULT_PG_USER = process.env.TEST_HARNESS_PGUSER ?? 'postgres';
-// The seeded test Postgres used by the harness runs with max_connections=50, so
-// isolated workers need a smaller per-process pool cap to keep workers=3 stable.
+// The seeded test Postgres runs with max_connections=50 (see
+// realm-server/tests/scripts/boot_preseeded.sh). A single stack runs two
+// pg-pool clients (realm-server + worker), so the ceiling for one stack is
+// pool_max × 2 processes ≈ peak connections. With pool_max=20 that's 40
+// connections at peak, leaving ~10 headroom for the harness's own pg client
+// and the migration template DB — comfortably under 50 even when every pool
+// slot is saturated. Production uses pg-adapter's default of 40 because the
+// hosted RDS has 1700+ max_connections and never sees the test pg's cap.
 export const DEFAULT_PG_POOL_MAX = Number(
-  process.env.TEST_HARNESS_PG_POOL_MAX ?? 2,
+  process.env.TEST_HARNESS_PG_POOL_MAX ?? 20,
 );
 export const DEFAULT_MIGRATED_TEMPLATE_DB =
   process.env.TEST_HARNESS_MIGRATED_TEMPLATE_DB ?? 'boxel_migrated_template';

packages/runtime-common/realm.ts

@@ -201,6 +201,26 @@ export type RealmInfo = {
 
 const PROTECTED_REALM_CONFIG_PROPERTIES = ['showAsCatalog'];
 
+// Marker header the host SPA attaches to outbound _federated-search /
+// _search calls when it's running inside a prerender tab. The prerender
+// server uses puppeteer's `evaluateOnNewDocument` to inject a window
+// global (`__boxelDuringPrerender = true`) into every Chrome tab before
+// the host loads; the host's realm-server fetch wrapper then reads that
+// flag and adds this header on its own outbound search requests only —
+// narrowly scoped so non-realm-server origins (icons, vite, etc.) don't
+// see it on a CORS preflight. When the realm sees this on an inbound
+// _search request it knows the caller is the host SPA mid-render and
+// switches the search to cacheOnlyDefinitions:true, which short-circuits
+// the recursive lookupDefinition → prerenderModule path in
+// populateQueryFields that causes self-referential prerender deadlocks
+// under parallel indexing. Kept as a bare string here so runtime-common
+// stays independent of realm-server. The realm-server prerender side
+// re-exports the same value from prerender-constants.ts.
+export const DURING_PRERENDER_HEADER = 'x-boxel-during-prerender';
+function isDuringPrerenderRequest(request: Request): boolean {
+  return (request.headers.get(DURING_PRERENDER_HEADER) ?? '').length > 0;
+}
+
 // Fields owned by the RealmConfig card instance at /realm.json. Anything not
 // in this set is still written to the legacy .realm.json sidecar until
 // CS-10055 moves hostHome / interactHome off-file.
@@ -4224,10 +4244,14 @@ export class Realm {
     return this.#realmIndexUpdater.isIgnored(url);
   }
 
-  public async search(query: Query): Promise<LinkableCollectionDocument> {
+  public async search(
+    query: Query,
+    opts?: { cacheOnlyDefinitions?: boolean },
+  ): Promise<LinkableCollectionDocument> {
     assertQuery(query);
     return await this.#realmIndexQueryEngine.searchCards(query, {
       loadLinks: true,
+      ...(opts?.cacheOnlyDefinitions ? { cacheOnlyDefinitions: true } : {}),
     });
   }
 
@@ -4278,7 +4302,9 @@ export class Realm {
 
     try {
       assertQuery(cardsQuery);
-      let doc = await this.search(cardsQuery);
+      let doc = await this.search(cardsQuery, {
+        cacheOnlyDefinitions: isDuringPrerenderRequest(request),
+      });
       return createResponse({
         body: JSON.stringify(doc, null, 2),
         init: {

packages/runtime-common/search-utils.ts

@@ -317,13 +317,17 @@ export function combinePrerenderedSearchResults(
 }
 
 type SearchableRealm = {
-  search: (query: Query) => Promise<LinkableCollectionDocument>;
+  search: (
+    query: Query,
+    opts?: { cacheOnlyDefinitions?: boolean },
+  ) => Promise<LinkableCollectionDocument>;
   url?: string;
 };
 
 export async function searchRealms(
   realms: Array<SearchableRealm | null | undefined>,
   query: Query,
+  opts?: { cacheOnlyDefinitions?: boolean },
 ): Promise<LinkableCollectionDocument> {
   let realmEntries = realms
     .filter((realm): realm is SearchableRealm => Boolean(realm))
@@ -332,7 +336,7 @@ export async function searchRealms(
       label: realm.url ? String(realm.url) : undefined,
     }));
   let searchPromises = realmEntries.map(({ realm }) =>
-    Promise.resolve().then(() => realm.search(query)),
+    Promise.resolve().then(() => realm.search(query, opts)),
   );
   let results = await Promise.allSettled(searchPromises);
   let queryLabel = '[unserializable query]';