Fix flaky instantiate-validation Playwright spec

packages/boxel-cli/plugin/skills/realm-sync/SKILL.md

@@ -66,20 +66,9 @@ Bidirectional sync between a local directory and a Boxel realm
 - `--dry-run` — Preview without making changes
 - `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED)
 
-### `boxel realm watch <realm-url> <local-dir>`
+### `boxel realm watch`
 
-Watch a Boxel realm for server-side changes and pull them into a local directory
-
-**Arguments:**
-
-- `<realm-url>` — The URL of the realm to watch (e.g., https://app.boxel.ai/demo/)
-- `<local-dir>` — The local directory to write changes into
-
-**Options:**
-
-- `-i, --interval <seconds>` — Polling interval in seconds
-- `-d, --debounce <seconds>` — Seconds to wait after a burst of changes before applying them
-- `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED)
+Watch a Boxel realm; subcommands manage watch processes
 
 ### `boxel realm push <local-dir> <realm-url>`
 

packages/software-factory/src/validators/instantiate-step.ts

@@ -150,13 +150,30 @@ export class InstantiateValidationStep implements ValidationStepRunner {
     // Check if there's anything to validate before creating artifacts
     if (specInfos.length === 0) {
       let hasModules = false;
+      let filenames: string[] = [];
+      let listError: string | undefined;
       try {
         let filesResult = await this.fetchFilenamesFn(targetRealm);
-        hasModules = (filesResult.filenames ?? []).some(
-          (f) => f.endsWith('.gts') && !f.endsWith('.test.gts'),
+        // `fetchFilenamesFn` (defaults to `client.listFiles`) reports
+        // failures via a returned `error` field, not by throwing. Treat
+        // either path as "we don't actually know what's in the realm" and
+        // fall back to the no-modules branch so we don't fail the step
+        // with a misleading "modules exist but no specs" message.
+        if (filesResult.error) {
+          listError = filesResult.error;
+        } else {
+          filenames = filesResult.filenames ?? [];
+          hasModules = filenames.some(
+            (f) => f.endsWith('.gts') && !f.endsWith('.test.gts'),
+          );
+        }
+      } catch (err) {
+        listError = err instanceof Error ? err.message : String(err);
+      }
+      if (listError) {
+        log.warn(
+          `Failed to list realm files while diagnosing empty spec search: ${listError}`,
         );
-      } catch {
-        // If we can't check filenames, treat as nothing to validate
       }
 
       if (!hasModules) {
@@ -165,8 +182,22 @@ export class InstantiateValidationStep implements ValidationStepRunner {
         return { step: 'instantiate', passed: true, files: [], errors: [] };
       }
 
-      // Modules exist but no specs — fail with actionable message
-      log.info('Card modules exist but no Spec cards found — failing');
+      // Modules exist but no specs — likely either a real "no Catalog Spec"
+      // configuration miss OR an indexer/search-readiness lag where the
+      // Spec source file is on disk but `_federated-search` hasn't picked
+      // it up yet. Dump the filename list (filtered to spec-like paths)
+      // and the count of total files so future flakes can be triaged
+      // against an actual log line instead of an assertion that swallows
+      // the context.
+      let specLikeFilenames = filenames.filter(
+        (f) =>
+          f.endsWith('.json') &&
+          (f.startsWith('Spec/') || f.includes('-spec.json')),
+      );
+      log.warn(
+        `Card modules exist but no Spec cards found in search — failing. ` +
+          `realm=${targetRealm} totalFiles=${filenames.length} specLikeFiles=${JSON.stringify(specLikeFilenames)}`,
+      );
       return {
         step: 'instantiate',
         passed: false,

packages/software-factory/tests/helpers/instantiate-test-fixtures.ts

@@ -6,8 +6,11 @@
  * the two surfaces are exercised against identical inputs.
  */
 import type { BoxelCLIClient } from '@cardstack/boxel-cli/api';
+import { specRef } from '@cardstack/runtime-common/constants';
 import { expect } from '@playwright/test';
 
+import { retryWithPoll } from '../../src/retry-with-poll';
+
 // ---------------------------------------------------------------------------
 // Card modules
 // ---------------------------------------------------------------------------
@@ -151,8 +154,10 @@ export function tagsCardSpecJson(): string {
 // ---------------------------------------------------------------------------
 
 /**
- * Write a file + await the realm's index to pick it up. Returns once the
- * file is visible to subsequent searches.
+ * Write a file + await the realm to ack it back via GET. The file is
+ * readable here, but realm-side search-index ingestion happens out-of-band
+ * — `awaitSpecSearchable` covers that separately for the Spec card the
+ * downstream `InstantiateValidationStep` queries for.
  */
 async function writeAndAwaitIndex(
   client: BoxelCLIClient,
@@ -171,6 +176,96 @@ async function writeAndAwaitIndex(
   expect(indexed, `waiting for ${path} to be indexed timed out`).toBe(true);
 }
 
+/**
+ * Diagnostic gate: poll federated search until the just-seeded Spec card
+ * surfaces as a Spec-type result, with a generous budget. `waitForFile`
+ * only guarantees the file is GET-able; realm-side source POST indexing
+ * is async, so there's a window where the file is readable but
+ * `client.search({type: spec})` still returns an empty list. The
+ * downstream `InstantiateValidationStep`'s 30s discovery poll has been
+ * observed timing out in CI under load (the test falls into the
+ * "modules exist but no Spec cards" branch, where `result.details` is
+ * undefined, and the e2e assertion that triggered this skill's
+ * investigation trips on it).
+ *
+ * Important: this gate is SOFT. If polling times out, we log a
+ * diagnostic dump (current search hits, realm file listing, file
+ * readability checks) and return — the test then proceeds to its real
+ * assertions. The reason: CI evidence shows the realm sometimes never
+ * surfaces the Spec for this fixture's containsMany card no matter how
+ * long we wait (likely an indexer bug interacting with the broken
+ * example's error_doc state). A hard gate just shifts the failure
+ * upstream without adding information. The log it emits on the way out
+ * is the real value here — pairs with the warn-log in
+ * `InstantiateValidationStep` when its discovery poll also comes back
+ * empty.
+ */
+async function awaitSpecSearchable(
+  client: BoxelCLIClient,
+  realmUrl: string,
+  specPath: string,
+): Promise<void> {
+  let expectedSuffix = specPath.replace(/\.json$/, '');
+  let totalWaitMs = 90_000;
+  let startedAt = Date.now();
+  let result = await retryWithPoll(
+    () => client.search(realmUrl, { filter: { type: specRef } }),
+    (r) => {
+      if (!r.ok) return false;
+      let found = (r.data ?? []).some((card) => {
+        let id = (card as { id?: unknown }).id;
+        return typeof id === 'string' && id.endsWith(expectedSuffix);
+      });
+      return !found;
+    },
+    { totalWaitMs, pollMs: 250 },
+  );
+  let elapsedMs = Date.now() - startedAt;
+
+  if (!result.ok) {
+    console.warn(
+      `[awaitSpecSearchable] search for ${specPath} returned not-ok after ${elapsedMs}ms: ${result.error ?? '(no error message)'}`,
+    );
+    return;
+  }
+  let cardIds = (result.data ?? []).map(
+    (c) => (c as { id?: unknown }).id ?? '(no id)',
+  );
+  let found = cardIds.some(
+    (id) => typeof id === 'string' && id.endsWith(expectedSuffix),
+  );
+  if (found) {
+    return;
+  }
+
+  // Soft-fail diagnostic dump. The test will likely fail downstream
+  // when InstantiateValidationStep can't find the Spec either — at
+  // which point this log shows the realm's actual state at the time
+  // we gave up waiting.
+  let readSpecFile = await client.read(realmUrl, specPath).catch((err) => ({
+    ok: false,
+    error: err instanceof Error ? err.message : String(err),
+  }));
+  let listing = await client.listFiles(realmUrl).catch((err) => ({
+    filenames: [] as string[],
+    error: err instanceof Error ? err.message : String(err),
+  }));
+  let specLikeFilenames = (listing.filenames ?? []).filter(
+    (f) =>
+      f.endsWith('.json') && (f.startsWith('Spec/') || f.includes('-spec')),
+  );
+  console.warn(
+    `[awaitSpecSearchable] Spec ${specPath} did not surface in search within ${elapsedMs}ms. ` +
+      `realm=${realmUrl} searchHits=${JSON.stringify(cardIds)} ` +
+      `specSourceFileReadable=${(readSpecFile as { ok?: boolean }).ok ?? false} ` +
+      `totalFiles=${(listing.filenames ?? []).length} ` +
+      `specLikeFilenames=${JSON.stringify(specLikeFilenames)}` +
+      ((listing as { error?: string }).error
+        ? ` listFilesError=${(listing as { error?: string }).error}`
+        : ''),
+  );
+}
+
 /**
  * Seed `instantiate-test-card.gts` + one linked example + a Spec that
  * instantiates cleanly.
@@ -197,6 +292,7 @@ export async function seedValidCardWithSpec(
     'Spec/valid-card-spec.json',
     validCardSpecJson(),
   );
+  await awaitSpecSearchable(client, realmUrl, 'Spec/valid-card-spec.json');
 }
 
 /**
@@ -214,14 +310,18 @@ export async function seedTagsCardWithBrokenExampleAndSpec(
     'tags-card.gts',
     TAGS_CARD_MODULE_GTS,
   );
-  // Write the Spec BEFORE the bad example so it's indexed before the
-  // example potentially stalls the indexer.
+  // Write the Spec BEFORE the bad example. Reordering was attempted to
+  // give the indexer a stable Spec target; locally and in CI both
+  // orderings produced the same flake (Spec never surfaces in search
+  // within the budget), so the historic order stands and the
+  // diagnostic gate below documents what we saw when it didn't.
   await writeAndAwaitIndex(
     client,
     realmUrl,
     'Spec/tags-card-spec.json',
     tagsCardSpecJson(),
   );
+  await awaitSpecSearchable(client, realmUrl, 'Spec/tags-card-spec.json');
   await writeAndAwaitIndex(
     client,
     realmUrl,