From 0e6d527cc88f83effd97c30234bf5cbac7c1fc79 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 18 May 2026 15:46:02 -0400 Subject: [PATCH 01/12] CS-11170: Include FileDefs in CardsGrid Surfaces FileDef instances alongside CardDef instances in the CardsGrid sidebar. Implements the six-step plan from the Linear project description. - realm_meta.value partitions into { instances, files } via a per-type aggregation in index-writer; normalizeRealmMetaValue tolerates the legacy array shape during rollout. - File rows now carry display_names from a FileDef class-chain walk in FileDefAttributesExtractor, parallel to the card-side meta route. - _types endpoint emits a flat list with a kind: 'instance' | 'file' discriminator; CardsGrid partitions by kind to render an All Files group beside All Cards (hidden when the realm has no files). - detectStackItemTypeForTarget consults knownFileMetaUrls so files clicked from a freshly-loaded CardsGrid resolve to file stack items before the file-meta resource lands in the store; an Open in Code Mode menu entry is added to the file kebab in interact context. - Coverage: kind discriminator asserted on the existing types endpoint test, new index-writer case for file-row partitioning, and a normalize-realm-meta-value shared test for backward compat. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/base/cards-grid.gts | 62 ++++++++- packages/base/file-menu-items.ts | 15 +++ .../components/prerendered-card-search.gts | 16 +-- packages/host/app/lib/known-file-meta-urls.ts | 17 +++ packages/host/app/lib/stack-item.ts | 15 ++- packages/host/app/services/realm-server.ts | 1 + .../utils/file-def-attributes-extractor.ts | 29 ++++ .../tests/helpers/realm-server-mock/routes.ts | 7 +- packages/host/tests/unit/index-writer-test.ts | 127 +++++++++++++++++- packages/realm-server/tests/index.ts | 1 + .../tests/normalize-realm-meta-value-test.ts | 32 +++++ .../realm-server/tests/types-endpoint-test.ts | 31 ++++- packages/runtime-common/document-types.ts | 48 ++++++- packages/runtime-common/index-query-engine.ts | 10 +- .../index-runner/file-indexer.ts | 2 +- packages/runtime-common/index-structure.ts | 34 ++++- packages/runtime-common/index-writer.ts | 55 +++++--- packages/runtime-common/index.ts | 4 + .../tests/normalize-realm-meta-value-test.ts | 73 ++++++++++ 19 files changed, 532 insertions(+), 47 deletions(-) create mode 100644 packages/host/app/lib/known-file-meta-urls.ts create mode 100644 packages/realm-server/tests/normalize-realm-meta-value-test.ts create mode 100644 packages/runtime-common/tests/normalize-realm-meta-value-test.ts diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index c96caf781b0..1255cd8cb87 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -12,12 +12,15 @@ import { HighlightIcon } from '@cardstack/boxel-ui/icons'; import LayoutGridPlusIcon from '@cardstack/boxel-icons/layout-grid-plus'; import Captions from '@cardstack/boxel-icons/captions'; import AllCardsIcon from '@cardstack/boxel-icons/square-stack'; +import AllFilesIcon from '@cardstack/boxel-icons/files'; +import FileIcon from '@cardstack/boxel-icons/file'; import { cardIdToURL, chooseCard, specRef, baseRealm, + baseFileRef, isCardInstance, SupportedMimeType, subscribeToRealm, @@ -101,8 +104,9 @@ class Isolated extends Component { private cardTypeFilters: FilterOption[] = new TrackedArray(); + private fileTypeFilters: FilterOption[] = new TrackedArray(); private highlightsCards: BoxComponent[] = new TrackedArray(); - private filterOptions: FilterOption[] = []; + private filterOptions: FilterOption[] = new TrackedArray(); private viewOptions: ViewOption[] = new TrackedArray([StripView, GridView]); private sortOptions: SortOption[] = new TrackedArray(SORT_OPTIONS); @@ -174,12 +178,34 @@ class Isolated extends Component { }; } + // The All Files group is only added to the sidebar after `loadFilterList` + // detects file-kind summaries in the realm — see how `setupFilterOptions` is + // re-called from `loadFilterList`. This satisfies the "hide empty groups" + // decision in the Linear plan: realms without any FileDef instances don't + // get a stub File branch. + private get allFilesFilter(): FilterOption { + return { + displayName: 'All Files', + icon: AllFilesIcon, + query: { + filter: { + type: baseFileRef, + }, + }, + filters: this.fileTypeFilters, + isExpanded: false, + }; + } + private setupFilterOptions() { this.filterOptions.splice(0, this.filterOptions.length); if (this.isPersonalRealm) { this.filterOptions.push(this.highlightFilter); } this.filterOptions.push(this.allCardsFilter); + if (this.fileTypeFilters.length > 0) { + this.filterOptions.push(this.allFilesFilter); + } } private teardownRealmSubscription() { @@ -293,23 +319,50 @@ class Isolated extends Component { displayName: string; total: number; iconHTML: string | null; + // Older realm-server builds may not stamp `kind` yet — treat missing + // as 'instance' so this client stays compatible during a rolling + // deploy. New servers always set the discriminator. + kind?: 'instance' | 'file'; }; }[]; let excludedCardTypeIds = [ `${baseRealm.url}card-api/CardDef`, `${baseRealm.url}cards-grid/CardsGrid`, ]; + // The "All Files" group already represents the bare FileDef root — listing + // it again as a leaf would just be a duplicate row. + let excludedFileTypeIds = [`${baseRealm.url}card-api/FileDef`]; this.cardTypeFilters.splice(0, this.cardTypeFilters.length); + this.fileTypeFilters.splice(0, this.fileTypeFilters.length); cardTypeSummaries.forEach((summary) => { - if (!summary.id || excludedCardTypeIds.includes(summary.id)) { + if (!summary.id) { return; } let codeRef = codeRefFromInternalKey(summary.id); if (!codeRef) { return; } + let kind = summary.attributes.kind ?? 'instance'; + if (kind === 'file') { + if (excludedFileTypeIds.includes(summary.id)) { + return; + } + this.fileTypeFilters.push({ + displayName: summary.attributes.displayName ?? codeRef.name, + icon: summary.attributes.iconHTML ?? FileIcon, + query: { + filter: { + type: codeRef, + }, + }, + }); + return; + } + if (excludedCardTypeIds.includes(summary.id)) { + return; + } this.cardTypeFilters.push({ displayName: summary.attributes.displayName ?? codeRef.name, icon: summary.attributes.iconHTML ?? Captions, @@ -321,6 +374,11 @@ class Isolated extends Component { }); }); + // Re-run setup so the All Files group appears/disappears as the file leaf + // list grows or shrinks. setupFilterOptions inspects `fileTypeFilters` to + // decide whether to include the group at all. + this.setupFilterOptions(); + let flattenedFilters: FilterOption[] = []; this.filterOptions.map((f) => f.filters?.length diff --git a/packages/base/file-menu-items.ts b/packages/base/file-menu-items.ts index 88f99802915..cfec9c375da 100644 --- a/packages/base/file-menu-items.ts +++ b/packages/base/file-menu-items.ts @@ -35,6 +35,21 @@ export function getDefaultFileMenuItems( }); } if (params.menuContext === 'interact') { + if (fileDefInstanceId) { + // Files are read-only in interact-mode (no edit format). The "Open in + // Code Mode" entry is the canonical way for a user who opened a file + // via CardsGrid's All Files group to jump to the editing surface. + menuItems.push({ + label: 'Open in Code Mode', + action: async () => { + await new SwitchSubmodeCommand(params.commandContext).execute({ + submode: 'code', + codePath: fileDefInstanceId, + }); + }, + icon: CodeIcon, + }); + } if (fileDefInstanceId && params.canEdit) { // TODO: add menu item to delete the file } diff --git a/packages/host/app/components/prerendered-card-search.gts b/packages/host/app/components/prerendered-card-search.gts index 0888b7cdb52..39a406a8fa7 100644 --- a/packages/host/app/components/prerendered-card-search.gts +++ b/packages/host/app/components/prerendered-card-search.gts @@ -32,14 +32,14 @@ import { getSearch } from '../resources/search'; const OWNER_DESTROYED_ERROR = "Cannot call `.lookup('renderer:-dom')` after the owner has been destroyed"; -// Internal registry of URLs known to be file-meta from prerendered search. -// Used by the overlay system to correctly identify FileDef cards when they -// haven't been loaded into the store yet (prerendered results are HTML-only). -export const knownFileMetaUrls = new Set(); - -export function clearKnownFileMetaUrls() { - knownFileMetaUrls.clear(); -} +// The Set itself lives in `lib/known-file-meta-urls` so non-component code +// (e.g. `lib/stack-item.ts`) can read it without importing a component file. +// Re-exported here for back-compat with existing call sites. +import { + knownFileMetaUrls, + clearKnownFileMetaUrls, +} from '../lib/known-file-meta-urls'; +export { knownFileMetaUrls, clearKnownFileMetaUrls }; export class PrerenderedCard implements PrerenderedCardLike { component: HTMLComponent; diff --git a/packages/host/app/lib/known-file-meta-urls.ts b/packages/host/app/lib/known-file-meta-urls.ts new file mode 100644 index 00000000000..602ec29c661 --- /dev/null +++ b/packages/host/app/lib/known-file-meta-urls.ts @@ -0,0 +1,17 @@ +// In-memory registry of URLs that prerendered search has identified as +// file-meta (FileDef) rather than card instances. Populated as +// PrerenderedCard wrappers are constructed in `prerendered-card-search.gts` +// and consulted by `detectStackItemTypeForTarget` (stack-item.ts) and the +// operator-mode-overlays so that clicking a file row in CardsGrid / +// embedded-card overlays routes to a file stack item even when the +// file-meta resource hasn't been loaded into the store yet (prerendered +// results carry HTML only). +// +// Lives in `lib/` rather than under a component so it can be imported from +// both component-side code (which adds entries) and lib-side type detection +// (which only reads entries) without creating a lib → components cycle. +export const knownFileMetaUrls = new Set(); + +export function clearKnownFileMetaUrls() { + knownFileMetaUrls.clear(); +} diff --git a/packages/host/app/lib/stack-item.ts b/packages/host/app/lib/stack-item.ts index 88efabd936e..fb4642916cf 100644 --- a/packages/host/app/lib/stack-item.ts +++ b/packages/host/app/lib/stack-item.ts @@ -6,6 +6,8 @@ import type { Store, StoreReadType } from '@cardstack/runtime-common'; import type { Format } from 'https://cardstack.com/base/card-api'; +import { knownFileMetaUrls } from './known-file-meta-urls'; + interface Args { format: Format; request?: Deferred; @@ -50,7 +52,18 @@ export function detectStackItemTypeForTarget( let fileMetaInstanceOrError = store.peek(cardId, { type: 'file-meta' }) ?? store.peekError(cardId, { type: 'file-meta' }); - return fileMetaInstanceOrError ? 'file' : 'card'; + if (fileMetaInstanceOrError) { + return 'file'; + } + // CardsGrid and other prerendered-search consumers click URLs whose + // file-meta resources may not yet be in the store (prerendered results are + // HTML-only). `knownFileMetaUrls` is populated as PrerenderedCard wrappers + // are built, so consulting it covers the common "user clicks a file row in + // a freshly opened CardsGrid" path. + if (knownFileMetaUrls.has(cardId)) { + return 'file'; + } + return 'card'; } let nextInteractionSequence = 0; diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 6ba512fee04..52a0c159bd6 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -554,6 +554,7 @@ export default class RealmServerService extends Service { displayName: string; total: number; iconHTML: string; + kind?: 'instance' | 'file'; }; meta?: { realmURL: string }; }[]; diff --git a/packages/host/app/utils/file-def-attributes-extractor.ts b/packages/host/app/utils/file-def-attributes-extractor.ts index 1a4495cb99a..12f49f13804 100644 --- a/packages/host/app/utils/file-def-attributes-extractor.ts +++ b/packages/host/app/utils/file-def-attributes-extractor.ts @@ -40,6 +40,7 @@ export type FileDefExtractResult = { searchDoc: Record | null; resource?: FileMetaResource; types?: string[]; + displayNames?: string[]; deps: string[]; error?: RenderError; mismatch?: true; @@ -200,6 +201,7 @@ export class FileDefAttributesExtractor { if (searchDoc) { let typeCodeRefs = getTypes(klass); let types = typeCodeRefs.map((type) => internalKeyFor(type, undefined)); + let displayNames = getDisplayNames(klass); let adoptsFrom = typeCodeRefs[0] ?? this.#fileDefCodeRef; let queryFieldDefs = await this.extractQueryFieldDefs(klass); return { @@ -212,6 +214,7 @@ export class FileDefAttributesExtractor { queryFieldDefs, ), types, + displayNames, deps, ...(error ? { error } : {}), ...(mismatch ? { mismatch: true } : {}), @@ -377,6 +380,32 @@ export function getTypes(klass: FileDefConstructor): CodeRef[] { return types; } +// Walks the FileDef subclass prototype chain and collects each level's +// `static displayName` (e.g. `MarkdownDef.displayName === 'Markdown'`). +// Mirrors the card-side getDisplayNames in routes/render/meta.ts so the same +// `boxel_index.display_names` semantics apply to file rows. +export function getDisplayNames(klass: FileDefConstructor): string[] { + let displayNames: string[] = []; + let current: FileDefConstructor | undefined = klass; + while (current) { + let ref = identifyCard(current as unknown as typeof BaseDef); + if (!ref || isEqual(ref, baseRef)) { + break; + } + let kAny = current as { + displayName?: string; + name?: string; + }; + if (typeof kAny.displayName === 'string' && kAny.displayName) { + displayNames.push(kAny.displayName); + } else if (typeof kAny.name === 'string' && kAny.name) { + displayNames.push(kAny.name); + } + current = Reflect.getPrototypeOf(current) as FileDefConstructor | undefined; + } + return displayNames; +} + export function buildFileResource( fileURL: string, attributes: Record, diff --git a/packages/host/tests/helpers/realm-server-mock/routes.ts b/packages/host/tests/helpers/realm-server-mock/routes.ts index b74ef5efdb3..467d524bf3c 100644 --- a/packages/host/tests/helpers/realm-server-mock/routes.ts +++ b/packages/host/tests/helpers/realm-server-mock/routes.ts @@ -263,7 +263,12 @@ function registerTypesRoutes() { let allEntries: { id: string; type: 'card-type-summary'; - attributes: { displayName: string; total: number; iconHTML: string }; + attributes: { + displayName: string; + total: number; + iconHTML: string; + kind: 'instance' | 'file'; + }; meta: { realmURL: string }; }[] = []; diff --git a/packages/host/tests/unit/index-writer-test.ts b/packages/host/tests/unit/index-writer-test.ts index 01415b626fb..a639811add3 100644 --- a/packages/host/tests/unit/index-writer-test.ts +++ b/packages/host/tests/unit/index-writer-test.ts @@ -85,9 +85,13 @@ const fetchRealmMetaRows = async (adapter: SQLiteAdapter) => const fetchRealmMeta = async (adapter: SQLiteAdapter) => { let rows = await fetchRealmMetaRows(adapter); + let raw = rows[0]?.value as + | { instances?: RealmMetaValue[]; files?: RealmMetaValue[] } + | undefined; return { rows, - value: (rows[0]?.value ?? []) as RealmMetaValue[], + value: (raw?.instances ?? []) as RealmMetaValue[], + files: (raw?.files ?? []) as RealmMetaValue[], }; }; @@ -1668,6 +1672,127 @@ module('Unit | index-writer', function (hooks) { ); }); + test('update realm meta partitions file rows into the files array', async function (assert) { + // CS-prep for "Include FileDefs in CardsGrid": file rows in boxel_index + // (type='file') should be aggregated into `realm_meta.value.files`, + // independently of the cards group. This is what powers CardsGrid's + // "All Files" sidebar leaves. + let iconHTML = 'file icon'; + let baseFileTypes = internalKeysFor({ + module: rri('./card-api'), + name: 'FileDef', + }); + let markdownTypes = internalKeysFor( + { + module: rri('./markdown-file-def'), + name: 'MarkdownDef', + }, + { module: rri('./card-api'), name: 'FileDef' }, + ); + + await setupIndex( + adapter, + [{ realm_url: testRealmURL, current_version: 1 }], + [ + // Plain instance row — should land in `instances`. + { + url: `${testRealmURL}1.json`, + realm_version: 1, + realm_url: testRealmURL, + type: 'instance', + pristine_doc: makeCardResource('1', 'Mango', { + module: rri('./person'), + name: 'Person', + }) as LooseCardResource, + search_doc: { name: 'Mango' }, + display_names: ['Person'], + deps: [`${testRealmURL}person`], + types: internalKeysFor( + { module: rri('./person'), name: 'Person' }, + baseCardRef, + ), + icon_html: iconHTML, + }, + // Two markdown files — same code_ref so they collapse into one + // summary row with total: 2. + { + url: `${testRealmURL}notes/a.md`, + realm_version: 1, + realm_url: testRealmURL, + type: 'file', + search_doc: { name: 'a.md', url: `${testRealmURL}notes/a.md` }, + display_names: ['Markdown', 'File'], + types: markdownTypes, + icon_html: iconHTML, + }, + { + url: `${testRealmURL}notes/b.md`, + realm_version: 1, + realm_url: testRealmURL, + type: 'file', + search_doc: { name: 'b.md', url: `${testRealmURL}notes/b.md` }, + display_names: ['Markdown', 'File'], + types: markdownTypes, + icon_html: iconHTML, + }, + // A bare-FileDef file — base type used when the extension isn't mapped. + { + url: `${testRealmURL}misc/raw.bin`, + realm_version: 1, + realm_url: testRealmURL, + type: 'file', + search_doc: { name: 'raw.bin', url: `${testRealmURL}misc/raw.bin` }, + display_names: ['File'], + types: baseFileTypes, + icon_html: iconHTML, + }, + ], + ); + + let batch = await indexWriter.createBatch(new URL(testRealmURL)); + // No new writes — just finalize so updateRealmMeta runs against the + // working table that setupIndex seeded. + await batch.done(); + + let realmMeta = await fetchRealmMeta(adapter); + assert.strictEqual( + realmMeta.rows.length, + 1, + 'one realm_meta row was written', + ); + assert.deepEqual( + realmMeta.value, + [ + makeCardTypeSummary( + `${testRealmURL}person/Person`, + 'Person', + iconHTML, + 1, + ), + ], + 'instance summaries only reflect CardDef rows', + ); + // Files are ordered by display_name ASC; "File" sorts before "Markdown". + assert.deepEqual( + realmMeta.files, + [ + makeCardTypeSummary( + `${testRealmURL}card-api/FileDef`, + 'File', + iconHTML, + 1, + ), + makeCardTypeSummary( + `${testRealmURL}markdown-file-def/MarkdownDef`, + 'Markdown', + iconHTML, + 2, + ), + ], + 'file summaries collapse duplicates and live in their own arm', + ); + }); + test('update realm meta includes error entries with last known good state', async function (assert) { let iconHTML = 'test icon'; let personTypes = internalKeysFor( diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index 200bdebf7db..ba4dba6aae1 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -269,6 +269,7 @@ const ALL_TEST_FILES: string[] = [ './query-matches-filter-test', './matches-filter-integration-test', './search-in-flight-key-test', + './normalize-realm-meta-value-test', './job-scoped-search-cache-test', './consuming-realm-header-test', './delete-boxel-claimed-domain-test', diff --git a/packages/realm-server/tests/normalize-realm-meta-value-test.ts b/packages/realm-server/tests/normalize-realm-meta-value-test.ts new file mode 100644 index 00000000000..d68baa8d7e4 --- /dev/null +++ b/packages/realm-server/tests/normalize-realm-meta-value-test.ts @@ -0,0 +1,32 @@ +import { module, test } from 'qunit'; +import { basename } from 'path'; +import { runSharedTest } from '@cardstack/runtime-common/helpers'; +import normalizeRealmMetaValueTests from '@cardstack/runtime-common/tests/normalize-realm-meta-value-test'; + +module(basename(__filename), function () { + module('normalizeRealmMetaValue', function () { + test('undefined value normalizes to empty groups', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + + test('null value normalizes to empty groups', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + + test('legacy array shape maps to instances, files defaults to empty', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + + test('partitioned shape passes through', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + + test('missing arms default to empty arrays', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + + test('unrecognized object shape normalizes to empty groups', async function (assert) { + await runSharedTest(normalizeRealmMetaValueTests, assert, {}); + }); + }); +}); diff --git a/packages/realm-server/tests/types-endpoint-test.ts b/packages/realm-server/tests/types-endpoint-test.ts index 42acc8e5c3a..f7a8cd4fbcc 100644 --- a/packages/realm-server/tests/types-endpoint-test.ts +++ b/packages/realm-server/tests/types-endpoint-test.ts @@ -136,7 +136,7 @@ module(basename(__filename), function () { return aName.localeCompare(bName); }); assert.strictEqual(response.status, 200, 'HTTP 200 status'); - let expectedData = [ + let instanceEntries = [ { type: 'card-type-summary', id: `${testRealm.url}chess-gallery/ChessGallery`, @@ -144,6 +144,7 @@ module(basename(__filename), function () { displayName: 'Chess Gallery', total: 3, iconHTML: chessIconHTML, + kind: 'instance' as const, }, }, { @@ -153,6 +154,7 @@ module(basename(__filename), function () { displayName: 'Family Photo Card', total: 2, iconHTML, + kind: 'instance' as const, }, }, { @@ -162,6 +164,7 @@ module(basename(__filename), function () { displayName: 'Friend', total: 2, iconHTML, + kind: 'instance' as const, }, }, { @@ -171,6 +174,7 @@ module(basename(__filename), function () { displayName: 'FriendWithUsedLink', total: 2, iconHTML, + kind: 'instance' as const, }, }, { @@ -180,6 +184,7 @@ module(basename(__filename), function () { displayName: 'Home', total: 1, iconHTML, + kind: 'instance' as const, }, }, { @@ -189,6 +194,7 @@ module(basename(__filename), function () { displayName: 'Person', total: 3, iconHTML, + kind: 'instance' as const, }, }, { @@ -198,6 +204,7 @@ module(basename(__filename), function () { displayName: 'Person', total: 4, iconHTML, + kind: 'instance' as const, }, }, { @@ -207,6 +214,7 @@ module(basename(__filename), function () { displayName: 'Realm Config', total: 1, iconHTML: fileSettingsIconHTML, + kind: 'instance' as const, }, }, { @@ -216,12 +224,29 @@ module(basename(__filename), function () { displayName: 'TimersCard', total: 1, iconHTML, + kind: 'instance' as const, }, }, ]; + let actualInstances = response.body.data.filter( + (entry: any) => entry.attributes.kind === 'instance', + ); assert.deepEqual( - sortCardTypeSummaries(response.body.data), - sortCardTypeSummaries(expectedData), + sortCardTypeSummaries(actualInstances), + sortCardTypeSummaries(instanceEntries), + 'instance summaries match expected fixture', + ); + // Files may or may not be present in this realistic fixture; the more + // specific file-presence assertions live in the dedicated FileDefs test + // below. Here we only insist that every entry on the response carries a + // `kind` discriminator. + assert.ok( + response.body.data.every( + (entry: any) => + entry.attributes.kind === 'instance' || + entry.attributes.kind === 'file', + ), + 'every summary entry carries an instance/file kind', ); }); }); diff --git a/packages/runtime-common/document-types.ts b/packages/runtime-common/document-types.ts index 604622f9b10..2f76da256f2 100644 --- a/packages/runtime-common/document-types.ts +++ b/packages/runtime-common/document-types.ts @@ -1,6 +1,6 @@ import type { RealmInfo } from './realm'; import type { QueryResultsMeta, PrerenderedCard } from './index-query-engine'; -import type { CardTypeSummary } from './index-structure'; +import type { CardTypeSummary, RealmMetaValue } from './index-structure'; import { type CardResource, type FileMetaResource, @@ -162,17 +162,56 @@ export function transformResultsToPrerenderedCardsDoc(results: { }; } -export function makeCardTypeSummaryDoc(summaries: CardTypeSummary[]) { - let data = summaries.map((summary) => ({ +export type CardTypeSummaryKind = 'instance' | 'file'; + +// JSON:API representation of one entry from `realm_meta.value`. Clients +// partition the flat array by `kind` on read — see CardsGrid's +// `loadFilterList`. Keeping a single resource shape with a discriminator +// (rather than two parallel `data` arrays) preserves the existing contract for +// callers that only care about one kind. +export interface CardTypeSummaryEntry { + type: 'card-type-summary'; + id: string; + attributes: { + displayName: string; + total: number; + iconHTML: string; + kind: CardTypeSummaryKind; + }; +} + +function summaryToEntry( + summary: CardTypeSummary, + kind: CardTypeSummaryKind, +): CardTypeSummaryEntry { + return { type: 'card-type-summary', id: summary.code_ref, attributes: { displayName: summary.display_name, total: summary.total, iconHTML: summary.icon_html, + kind, }, - })); + }; +} +// Accepts either the partitioned RealmMetaValue (current shape) or a bare +// CardTypeSummary[] (legacy callers that pre-filtered to instances). In both +// cases the output is a flat list of entries discriminated by `kind`. +export function makeCardTypeSummaryDoc( + summaries: RealmMetaValue | CardTypeSummary[], +) { + let value: RealmMetaValue; + if (Array.isArray(summaries)) { + value = { instances: summaries, files: [] }; + } else { + value = summaries; + } + let data: CardTypeSummaryEntry[] = [ + ...value.instances.map((s) => summaryToEntry(s, 'instance')), + ...value.files.map((s) => summaryToEntry(s, 'file')), + ]; return { data }; } @@ -183,6 +222,7 @@ export interface FederatedCardTypeSummaryEntry { displayName: string; total: number; iconHTML: string; + kind: CardTypeSummaryKind; }; meta: { realmURL: string; diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index a9b2c886070..252ca066203 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -55,11 +55,11 @@ import { } from './query'; import type { SerializedError } from './error'; import type { DBAdapter } from './db'; -import type { RealmMetaTable } from './index-structure'; import { coerceTypes, + normalizeRealmMetaValue, type BoxelIndexTable, - type CardTypeSummary, + type RealmMetaValue, } from './index-structure'; import { getFieldDef, @@ -923,15 +923,15 @@ export class IndexQueryEngine { return usedRenderTypeColumnExpression; } - async fetchCardTypeSummary(realmURL: URL): Promise { + async fetchCardTypeSummary(realmURL: URL): Promise { let results = (await this.#query([ `SELECT value FROM realm_meta rm WHERE`, ...every([['rm.realm_url =', param(realmURL.href)]]), - ] as Expression)) as Pick[]; + ] as Expression)) as { value: unknown }[]; - return (results[0]?.value ?? []) as unknown as CardTypeSummary[]; + return normalizeRealmMetaValue(results[0]?.value); } private filterCondition(filter: Filter, onRef: CodeRef): CardExpression { diff --git a/packages/runtime-common/index-runner/file-indexer.ts b/packages/runtime-common/index-runner/file-indexer.ts index 3845988932d..0130199572d 100644 --- a/packages/runtime-common/index-runner/file-indexer.ts +++ b/packages/runtime-common/index-runner/file-indexer.ts @@ -194,7 +194,7 @@ export async function performFileIndexing({ ...(extractResult.searchDoc ?? {}), }, types: fileTypes, - displayNames: [], + displayNames: extractResult.displayNames ?? [], isolatedHtml: renderResult?.isolatedHTML ?? undefined, headHtml: renderResult?.headHTML ?? undefined, atomHtml: renderResult?.atomHTML ?? undefined, diff --git a/packages/runtime-common/index-structure.ts b/packages/runtime-common/index-structure.ts index 3e863646658..649920927e8 100644 --- a/packages/runtime-common/index-structure.ts +++ b/packages/runtime-common/index-structure.ts @@ -60,13 +60,45 @@ export interface CardTypeSummary { icon_html: string; } +// Top-level shape of `realm_meta.value`. `instances` summarizes CardDef rows +// (boxel_index.type='instance') and `files` summarizes FileDef rows +// (boxel_index.type='file'). Both arrays use the same per-type-summary shape. +// CardsGrid's sidebar partitions these into the "All Cards" and "All Files" +// top-level groups. +// +// Legacy realms written before this column was partitioned stored `value` as a +// bare `CardTypeSummary[]` (instances only). Readers should call +// `normalizeRealmMetaValue` (see index-query-engine.ts) to tolerate that shape +// during the transition until every realm has been reindexed. +export interface RealmMetaValue { + instances: CardTypeSummary[]; + files: CardTypeSummary[]; +} + export interface RealmMetaTable { realm_version: number; realm_url: string; - value: Record[]; + value: RealmMetaValue; indexed_at: string | null; } +// Tolerate the legacy `value` shape (a bare CardTypeSummary[]) stored before +// realm_meta was partitioned into instances/files. Once every realm has been +// reindexed, the array branch becomes dead code and can be removed. +export function normalizeRealmMetaValue(raw: unknown): RealmMetaValue { + if (!raw) { + return { instances: [], files: [] }; + } + if (Array.isArray(raw)) { + return { instances: raw as CardTypeSummary[], files: [] }; + } + let value = raw as Partial; + return { + instances: value.instances ?? [], + files: value.files ?? [], + }; +} + export const coerceTypes = Object.freeze({ deps: 'JSON', last_known_good_deps: 'JSON', diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index ed0d7baac08..39478c9a196 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -40,6 +40,7 @@ import type { TimingDiagnostics } from './index'; import { coerceTypes, type BoxelIndexTable, + type CardTypeSummary, type RealmVersionsTable, } from './index-structure'; import { v4 as uuidv4 } from '@lukeed/uuid'; @@ -640,31 +641,16 @@ export class Batch { } private async updateRealmMeta() { - let results = await this.#query([ - `SELECT CAST(count(DISTINCT i.url) AS INTEGER) as total, i.display_names->>0 as display_name, i.types->>0 as code_ref, MAX(i.icon_html) as icon_html - FROM boxel_index_working as i - WHERE`, - ...every([ - ['i.realm_url =', param(this.realmURL.href)], - ['i.type = ', param('instance')], - ['i.types IS NOT NULL'], - [ - dbExpression({ - pg: `(i.types->>0) IS NOT NULL`, - sqlite: `json_extract(i.types, '$[0]') IS NOT NULL`, - }), - ], - any([['i.is_deleted = false'], ['i.is_deleted IS NULL']]), - ]), - `GROUP BY i.display_names->>0, i.types->>0`, - `ORDER BY i.display_names->>0 ASC`, - ] as Expression); + let instances = await this.#fetchTypeSummary('instance'); + let files = await this.#fetchTypeSummary('file'); + + let value = { instances, files }; let { nameExpressions, valueExpressions } = asExpressions( { realm_url: this.realmURL.href, realm_version: this.realmVersion, - value: results, + value, indexed_at: unixTime(new Date().getTime()), } as Omit & { indexed_at: number; @@ -684,6 +670,35 @@ export class Batch { ]); } + // Aggregates per-type summaries (count, display name, code-ref key, icon) + // for one kind of row in boxel_index_working — either CardDef instances or + // FileDef files. The shape of the result rows matches `CardTypeSummary` + // exactly, so callers can drop them straight into `realm_meta.value`. + async #fetchTypeSummary( + indexType: BoxelIndexTable['type'], + ): Promise { + let results = await this.#query([ + `SELECT CAST(count(DISTINCT i.url) AS INTEGER) as total, i.display_names->>0 as display_name, i.types->>0 as code_ref, MAX(i.icon_html) as icon_html + FROM boxel_index_working as i + WHERE`, + ...every([ + ['i.realm_url =', param(this.realmURL.href)], + ['i.type = ', param(indexType)], + ['i.types IS NOT NULL'], + [ + dbExpression({ + pg: `(i.types->>0) IS NOT NULL`, + sqlite: `json_extract(i.types, '$[0]') IS NOT NULL`, + }), + ], + any([['i.is_deleted = false'], ['i.is_deleted IS NULL']]), + ]), + `GROUP BY i.display_names->>0, i.types->>0`, + `ORDER BY i.display_names->>0 ASC`, + ] as Expression); + return results as unknown as CardTypeSummary[]; + } + private async applyBatchUpdates() { let { nameExpressions, valueExpressions } = asExpressions({ realm_url: this.realmURL.href, diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 2a0e21c7a6a..61cb6401740 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -225,6 +225,10 @@ export interface FileExtractResponse { searchDoc: Record | null; resource?: FileMetaResource | null; types?: string[] | null; + // Display names walked from the resolved FileDef subclass up its prototype + // chain (e.g. `['Markdown', 'File']`). Persisted as `boxel_index.display_names` + // so CardsGrid's "All Files" sidebar can label each subtype. + displayNames?: string[] | null; deps: string[]; error?: RenderError; mismatch?: true; diff --git a/packages/runtime-common/tests/normalize-realm-meta-value-test.ts b/packages/runtime-common/tests/normalize-realm-meta-value-test.ts new file mode 100644 index 00000000000..816d637a598 --- /dev/null +++ b/packages/runtime-common/tests/normalize-realm-meta-value-test.ts @@ -0,0 +1,73 @@ +import type { SharedTests } from '../helpers'; +import { + normalizeRealmMetaValue, + type CardTypeSummary, +} from '../index-structure'; + +const personSummary: CardTypeSummary = { + code_ref: 'http://example.test/realm/person/Person', + display_name: 'Person', + total: 3, + icon_html: '', +}; + +const markdownSummary: CardTypeSummary = { + code_ref: 'https://cardstack.com/base/markdown-file-def/MarkdownDef', + display_name: 'Markdown', + total: 5, + icon_html: '', +}; + +const tests = Object.freeze({ + 'undefined value normalizes to empty groups': async (assert) => { + let normalized = normalizeRealmMetaValue(undefined); + assert.deepEqual(normalized, { instances: [], files: [] }); + }, + + 'null value normalizes to empty groups': async (assert) => { + let normalized = normalizeRealmMetaValue(null); + assert.deepEqual(normalized, { instances: [], files: [] }); + }, + + 'legacy array shape maps to instances, files defaults to empty': async ( + assert, + ) => { + // Realms indexed before realm_meta.value was partitioned stored a bare + // CardTypeSummary[] (instances only). Readers must accept that shape until + // every realm has been reindexed. + let normalized = normalizeRealmMetaValue([personSummary]); + assert.deepEqual(normalized, { + instances: [personSummary], + files: [], + }); + }, + + 'partitioned shape passes through': async (assert) => { + let value = { instances: [personSummary], files: [markdownSummary] }; + let normalized = normalizeRealmMetaValue(value); + assert.deepEqual(normalized, value); + }, + + 'missing arms default to empty arrays': async (assert) => { + // A partial object (e.g. data inserted by a test that only wrote one arm) + // shouldn't propagate undefined into consumers — both arms always exist on + // the normalized result. + let normalized = normalizeRealmMetaValue({ instances: [personSummary] }); + assert.deepEqual(normalized, { + instances: [personSummary], + files: [], + }); + }, + + 'unrecognized object shape normalizes to empty groups': async (assert) => { + // Defensive: if some other writer parked an unrelated JSONB blob in + // realm_meta.value (older delete-realm test fixture used this), neither + // arm contains those rows but readers don't crash. + let normalized = normalizeRealmMetaValue({ + somethingElse: { foo: 'bar' }, + }); + assert.deepEqual(normalized, { instances: [], files: [] }); + }, +} as SharedTests<{}>); + +export default tests; From 7d1e29c6ff6053de50c31e4d62786b4109a6a1c7 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 18 May 2026 16:20:35 -0400 Subject: [PATCH 02/12] AGENTS.md: capture host test output to a file A host test run produces hundreds of KB of output that's lost if you only pipe through tail/grep. Re-running to recover detail is slow and the original failure may not reproduce. Capture once with tee, then grep the file as needed. Co-Authored-By: Claude Opus 4.7 (1M context) --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index e9240f13cab..6a8fefb4efd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,12 @@ - To run a subset of the tests: `ember test --path dist --filter "some text that appears in module name or test name"` Note that the filter is matched against the module name and test name, not the file name! Try to avoid using pipe characters in the filter, since they can confuse auto-approval tool use filters set up by the user. +- **Always capture test output to a file.** A host test run can produce hundreds of KB of output (browser logs, indexer warnings, per-test diagnostics). If you only pipe through `tail`/`grep`, you lose everything else and have to re-run — which is slow and the bug may not reproduce. Redirect the full run to a file, then grep that file for failures, browser logs around a specific test, etc. + ``` + pnpm exec ember test --path dist --filter "Foo" 2>&1 | tee /tmp/host-test-foo.log + grep -E "^(not )?ok |^# " /tmp/host-test-foo.log # summary + per-test status + grep -B2 -A40 "not ok 27" /tmp/host-test-foo.log # detail for a specific failure + ``` - run `pnpm lint` in this directory to lint changes made to this package - run `pnpm lint:fix` directly in this directory to apply fixes for lint failures made to this package that can be automatically fixed. - the host tests report this error: From ba2b022a1f9ec8bb1734ae93f04c4ba12f17b1cd Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 18 May 2026 16:42:15 -0400 Subject: [PATCH 03/12] CardsGrid: derive filterOptions from tracked state via @cached getter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation made `filterOptions` a TrackedArray and called setupFilterOptions() — which does splice+push — from the component constructor. The constructor runs during render, so mutating tracked state there fires Glimmer's "you attempted to update X but it had already been used in this computation" assertion the moment a parent re-renders the CardsGrid card. The crash propagates up through OperatorModeStackItem and tears down the whole interact-mode view. Switch to a @cached getter that derives the group list from `isPersonalRealm` and `fileTypeFilters.length`. The per-group wrappers (highlightFilter, allCardsFilter, allFilesFilter) are also @cached so their object identity stays stable across re-computations — FilterList's isSelected is an identity comparison and would otherwise lose the active highlight after the first render. No imperative array mutation remains, so the assertion can't fire. Removed setupFilterOptions and its call site in loadFilterList; the getter re-runs naturally when fileTypeFilters.length transitions from 0 to >0 as `_types` results stream in. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/base/cards-grid.gts | 48 +++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index 1255cd8cb87..87185c14292 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -1,7 +1,7 @@ import { registerDestructor } from '@ember/destroyable'; import { on } from '@ember/modifier'; import { action } from '@ember/object'; -import { tracked } from '@glimmer/tracking'; +import { cached, tracked } from '@glimmer/tracking'; import { restartableTask } from 'ember-concurrency'; import { modifier } from 'ember-modifier'; import { TrackedArray } from 'tracked-built-ins'; @@ -106,7 +106,6 @@ class Isolated extends Component { private cardTypeFilters: FilterOption[] = new TrackedArray(); private fileTypeFilters: FilterOption[] = new TrackedArray(); private highlightsCards: BoxComponent[] = new TrackedArray(); - private filterOptions: FilterOption[] = new TrackedArray(); private viewOptions: ViewOption[] = new TrackedArray([StripView, GridView]); private sortOptions: SortOption[] = new TrackedArray(SORT_OPTIONS); @@ -119,7 +118,6 @@ class Isolated extends Component { constructor(owner: any, args: any) { super(owner, args); - this.setupFilterOptions(); this.activeFilter = this.filterOptions[0]; this.loadHighlightsCards.perform(); registerDestructor(this, () => this.teardownRealmSubscription()); @@ -152,6 +150,12 @@ class Isolated extends Component { return realmHref?.includes('/personal/') ?? false; } + // The per-group filter wrappers are `@cached` so their object identity + // stays stable across re-computations of `filterOptions`. `FilterList`'s + // `isSelected` is an identity comparison (`@filter === @activeFilter`), so + // returning a fresh object on every getter access would break the active + // highlight after the first render. + @cached private get highlightFilter(): FilterOption { return { displayName: 'Highlights', @@ -160,6 +164,7 @@ class Isolated extends Component { }; } + @cached private get allCardsFilter(): FilterOption { return { displayName: 'All Cards', @@ -178,11 +183,7 @@ class Isolated extends Component { }; } - // The All Files group is only added to the sidebar after `loadFilterList` - // detects file-kind summaries in the realm — see how `setupFilterOptions` is - // re-called from `loadFilterList`. This satisfies the "hide empty groups" - // decision in the Linear plan: realms without any FileDef instances don't - // get a stub File branch. + @cached private get allFilesFilter(): FilterOption { return { displayName: 'All Files', @@ -197,15 +198,28 @@ class Isolated extends Component { }; } - private setupFilterOptions() { - this.filterOptions.splice(0, this.filterOptions.length); + // Derived from tracked state — `isPersonalRealm` and `fileTypeFilters.length` + // are the only deps that change the visible group set. We avoid the + // previous `splice + push` approach because mutating a `TrackedArray` + // during the component constructor (which runs mid-render) fires Glimmer's + // "you attempted to update X but it was already used in this computation" + // assertion. A `@cached` getter only re-runs when its tracked deps change + // and stays stable otherwise, so identity-based comparisons keep working. + @cached + private get filterOptions(): FilterOption[] { + let options: FilterOption[] = []; if (this.isPersonalRealm) { - this.filterOptions.push(this.highlightFilter); + options.push(this.highlightFilter); } - this.filterOptions.push(this.allCardsFilter); + options.push(this.allCardsFilter); + // Hide the All Files group when the realm has no file rows — matches the + // "empty groups: hide" decision in the Linear plan. Reading `.length` on + // a TrackedArray sets up a dep so this getter re-runs once + // `loadFilterList` finishes its first push into `fileTypeFilters`. if (this.fileTypeFilters.length > 0) { - this.filterOptions.push(this.allFilesFilter); + options.push(this.allFilesFilter); } + return options; } private teardownRealmSubscription() { @@ -374,10 +388,10 @@ class Isolated extends Component { }); }); - // Re-run setup so the All Files group appears/disappears as the file leaf - // list grows or shrinks. setupFilterOptions inspects `fileTypeFilters` to - // decide whether to include the group at all. - this.setupFilterOptions(); + // `filterOptions` is a @cached getter that derives the group list from + // tracked state — `fileTypeFilters.length` changing here causes it to + // re-compute on the next read, which adds/removes the All Files group + // without any imperative array mutation. let flattenedFilters: FilterOption[] = []; this.filterOptions.map((f) => From 92356c2dae64835d8fae49346eb78d69786dfd0a Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 18 May 2026 17:05:06 -0400 Subject: [PATCH 04/12] CardList: fallback render when prerendered HTML is missing (CS-11171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CardsGrid's All Files group lists .gts/.ts file rows, but the fused visit currently skips the FileRender pass for executable modules (visit-file.ts:144), so their fitted_html / embedded_html / etc. are NULL. Without a fallback, renders nothing and the content area looks empty even though the search returned 137 rows. Expose `hasHtml` on PrerenderedCard (and PrerenderedCardLike) and render a minimal `` placeholder in CardList when it's false. The placeholder lives inside the existing clickable
  • , so clicking it still routes through viewCard into interact-mode (where the FileDef stack item can take over and the kebab offers "Open in Code Mode"). This is a workaround for CS-11171. The real fix is to drop the `!isModule` gate so executable modules get a FileRender pass — see that issue for the plan. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/base/components/card-list.gts | 77 ++++++++++++++++++- .../components/prerendered-card-search.gts | 8 ++ .../runtime-common/prerendered-card-search.ts | 6 ++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/packages/base/components/card-list.gts b/packages/base/components/card-list.gts index 99dbcc0fa9b..6f4b70bdf53 100644 --- a/packages/base/components/card-list.gts +++ b/packages/base/components/card-list.gts @@ -7,6 +7,8 @@ import { consume } from 'ember-provide-consume-context'; import { LoadingIndicator } from '@cardstack/boxel-ui/components'; +import FileIcon from '@cardstack/boxel-icons/file'; + import { cn, eq } from '@cardstack/boxel-ui/helpers'; import { @@ -48,6 +50,21 @@ export default class CardList extends Component { } } + // Last URL segment, used as a visible label when the prerender pipeline + // didn't produce HTML for a file row (CS-11171 — `.gts`/`.ts` FileDef rows + // currently skip the FileRender pass, so their `fitted_html` is null and + // `` renders nothing). + fileNameFromUrl(url: string): string { + try { + let pathname = new URL(url).pathname; + let segment = pathname.split('/').filter(Boolean).pop(); + return segment ?? url; + } catch { + let segments = url.split('/').filter(Boolean); + return segments[segments.length - 1] ?? url; + } + } +