diff --git a/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md b/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md index fc28c2eef7..9b75bef04a 100644 --- a/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md +++ b/packages/boxel-cli/plugin/skills/realm-sync/SKILL.md @@ -66,20 +66,9 @@ Bidirectional sync between a local directory and a Boxel realm - `--dry-run` — Preview without making changes - `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED) -### `boxel realm watch ` +### `boxel realm watch` -Watch a Boxel realm for server-side changes and pull them into a local directory - -**Arguments:** - -- `` — The URL of the realm to watch (e.g., https://app.boxel.ai/demo/) -- `` — The local directory to write changes into - -**Options:** - -- `-i, --interval ` — Polling interval in seconds -- `-d, --debounce ` — Seconds to wait after a burst of changes before applying them -- `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED) +Watch a Boxel realm; subcommands manage watch processes ### `boxel realm push ` diff --git a/packages/software-factory/src/validators/instantiate-step.ts b/packages/software-factory/src/validators/instantiate-step.ts index 3c9819ae2b..919a341b74 100644 --- a/packages/software-factory/src/validators/instantiate-step.ts +++ b/packages/software-factory/src/validators/instantiate-step.ts @@ -150,13 +150,30 @@ export class InstantiateValidationStep implements ValidationStepRunner { // Check if there's anything to validate before creating artifacts if (specInfos.length === 0) { let hasModules = false; + let filenames: string[] = []; + let listError: string | undefined; try { let filesResult = await this.fetchFilenamesFn(targetRealm); - hasModules = (filesResult.filenames ?? []).some( - (f) => f.endsWith('.gts') && !f.endsWith('.test.gts'), + // `fetchFilenamesFn` (defaults to `client.listFiles`) reports + // failures via a returned `error` field, not by throwing. Treat + // either path as "we don't actually know what's in the realm" and + // fall back to the no-modules branch so we don't fail the step + // with a misleading "modules exist but no specs" message. + if (filesResult.error) { + listError = filesResult.error; + } else { + filenames = filesResult.filenames ?? []; + hasModules = filenames.some( + (f) => f.endsWith('.gts') && !f.endsWith('.test.gts'), + ); + } + } catch (err) { + listError = err instanceof Error ? err.message : String(err); + } + if (listError) { + log.warn( + `Failed to list realm files while diagnosing empty spec search: ${listError}`, ); - } catch { - // If we can't check filenames, treat as nothing to validate } if (!hasModules) { @@ -165,8 +182,22 @@ export class InstantiateValidationStep implements ValidationStepRunner { return { step: 'instantiate', passed: true, files: [], errors: [] }; } - // Modules exist but no specs — fail with actionable message - log.info('Card modules exist but no Spec cards found — failing'); + // Modules exist but no specs — likely either a real "no Catalog Spec" + // configuration miss OR an indexer/search-readiness lag where the + // Spec source file is on disk but `_federated-search` hasn't picked + // it up yet. Dump the filename list (filtered to spec-like paths) + // and the count of total files so future flakes can be triaged + // against an actual log line instead of an assertion that swallows + // the context. + let specLikeFilenames = filenames.filter( + (f) => + f.endsWith('.json') && + (f.startsWith('Spec/') || f.includes('-spec.json')), + ); + log.warn( + `Card modules exist but no Spec cards found in search — failing. ` + + `realm=${targetRealm} totalFiles=${filenames.length} specLikeFiles=${JSON.stringify(specLikeFilenames)}`, + ); return { step: 'instantiate', passed: false, diff --git a/packages/software-factory/tests/helpers/instantiate-test-fixtures.ts b/packages/software-factory/tests/helpers/instantiate-test-fixtures.ts index 78229bcb67..caeba38bd4 100644 --- a/packages/software-factory/tests/helpers/instantiate-test-fixtures.ts +++ b/packages/software-factory/tests/helpers/instantiate-test-fixtures.ts @@ -6,8 +6,11 @@ * the two surfaces are exercised against identical inputs. */ import type { BoxelCLIClient } from '@cardstack/boxel-cli/api'; +import { specRef } from '@cardstack/runtime-common/constants'; import { expect } from '@playwright/test'; +import { retryWithPoll } from '../../src/retry-with-poll'; + // --------------------------------------------------------------------------- // Card modules // --------------------------------------------------------------------------- @@ -151,8 +154,10 @@ export function tagsCardSpecJson(): string { // --------------------------------------------------------------------------- /** - * Write a file + await the realm's index to pick it up. Returns once the - * file is visible to subsequent searches. + * Write a file + await the realm to ack it back via GET. The file is + * readable here, but realm-side search-index ingestion happens out-of-band + * — `awaitSpecSearchable` covers that separately for the Spec card the + * downstream `InstantiateValidationStep` queries for. */ async function writeAndAwaitIndex( client: BoxelCLIClient, @@ -171,6 +176,96 @@ async function writeAndAwaitIndex( expect(indexed, `waiting for ${path} to be indexed timed out`).toBe(true); } +/** + * Diagnostic gate: poll federated search until the just-seeded Spec card + * surfaces as a Spec-type result, with a generous budget. `waitForFile` + * only guarantees the file is GET-able; realm-side source POST indexing + * is async, so there's a window where the file is readable but + * `client.search({type: spec})` still returns an empty list. The + * downstream `InstantiateValidationStep`'s 30s discovery poll has been + * observed timing out in CI under load (the test falls into the + * "modules exist but no Spec cards" branch, where `result.details` is + * undefined, and the e2e assertion that triggered this skill's + * investigation trips on it). + * + * Important: this gate is SOFT. If polling times out, we log a + * diagnostic dump (current search hits, realm file listing, file + * readability checks) and return — the test then proceeds to its real + * assertions. The reason: CI evidence shows the realm sometimes never + * surfaces the Spec for this fixture's containsMany card no matter how + * long we wait (likely an indexer bug interacting with the broken + * example's error_doc state). A hard gate just shifts the failure + * upstream without adding information. The log it emits on the way out + * is the real value here — pairs with the warn-log in + * `InstantiateValidationStep` when its discovery poll also comes back + * empty. + */ +async function awaitSpecSearchable( + client: BoxelCLIClient, + realmUrl: string, + specPath: string, +): Promise { + let expectedSuffix = specPath.replace(/\.json$/, ''); + let totalWaitMs = 90_000; + let startedAt = Date.now(); + let result = await retryWithPoll( + () => client.search(realmUrl, { filter: { type: specRef } }), + (r) => { + if (!r.ok) return false; + let found = (r.data ?? []).some((card) => { + let id = (card as { id?: unknown }).id; + return typeof id === 'string' && id.endsWith(expectedSuffix); + }); + return !found; + }, + { totalWaitMs, pollMs: 250 }, + ); + let elapsedMs = Date.now() - startedAt; + + if (!result.ok) { + console.warn( + `[awaitSpecSearchable] search for ${specPath} returned not-ok after ${elapsedMs}ms: ${result.error ?? '(no error message)'}`, + ); + return; + } + let cardIds = (result.data ?? []).map( + (c) => (c as { id?: unknown }).id ?? '(no id)', + ); + let found = cardIds.some( + (id) => typeof id === 'string' && id.endsWith(expectedSuffix), + ); + if (found) { + return; + } + + // Soft-fail diagnostic dump. The test will likely fail downstream + // when InstantiateValidationStep can't find the Spec either — at + // which point this log shows the realm's actual state at the time + // we gave up waiting. + let readSpecFile = await client.read(realmUrl, specPath).catch((err) => ({ + ok: false, + error: err instanceof Error ? err.message : String(err), + })); + let listing = await client.listFiles(realmUrl).catch((err) => ({ + filenames: [] as string[], + error: err instanceof Error ? err.message : String(err), + })); + let specLikeFilenames = (listing.filenames ?? []).filter( + (f) => + f.endsWith('.json') && (f.startsWith('Spec/') || f.includes('-spec')), + ); + console.warn( + `[awaitSpecSearchable] Spec ${specPath} did not surface in search within ${elapsedMs}ms. ` + + `realm=${realmUrl} searchHits=${JSON.stringify(cardIds)} ` + + `specSourceFileReadable=${(readSpecFile as { ok?: boolean }).ok ?? false} ` + + `totalFiles=${(listing.filenames ?? []).length} ` + + `specLikeFilenames=${JSON.stringify(specLikeFilenames)}` + + ((listing as { error?: string }).error + ? ` listFilesError=${(listing as { error?: string }).error}` + : ''), + ); +} + /** * Seed `instantiate-test-card.gts` + one linked example + a Spec that * instantiates cleanly. @@ -197,6 +292,7 @@ export async function seedValidCardWithSpec( 'Spec/valid-card-spec.json', validCardSpecJson(), ); + await awaitSpecSearchable(client, realmUrl, 'Spec/valid-card-spec.json'); } /** @@ -214,14 +310,18 @@ export async function seedTagsCardWithBrokenExampleAndSpec( 'tags-card.gts', TAGS_CARD_MODULE_GTS, ); - // Write the Spec BEFORE the bad example so it's indexed before the - // example potentially stalls the indexer. + // Write the Spec BEFORE the bad example. Reordering was attempted to + // give the indexer a stable Spec target; locally and in CI both + // orderings produced the same flake (Spec never surfaces in search + // within the budget), so the historic order stands and the + // diagnostic gate below documents what we saw when it didn't. await writeAndAwaitIndex( client, realmUrl, 'Spec/tags-card-spec.json', tagsCardSpecJson(), ); + await awaitSpecSearchable(client, realmUrl, 'Spec/tags-card-spec.json'); await writeAndAwaitIndex( client, realmUrl,