From 699252f70676f0fa8cd9ee6550e2667fcf100983 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:40:04 -0400 Subject: [PATCH 01/15] feat(ocap-kernel): launch subcluster vats in parallel Previously #launchVatsForSubcluster awaited each vat's initVat handshake serially, making startup O(sum of vat init times). Now all vats launch concurrently via Promise.all. Each vat gets a kernel promise pre-allocated for its root object; the bootstrap message is queued immediately targeting the bootstrap vat's (unresolved) promise. KernelRouter parks the send until that promise resolves. As each vat's handshake completes its root promise is resolved via resolvePromises('kernel', ...), making startup O(max). The bootstrap vat receives kernel promises for all other vats' roots in its bootstrap() call, so it can pipeline calls to those vats while they are still initializing. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/src/Kernel.test.ts | 2 + .../src/vats/SubclusterManager.test.ts | 41 +++++++++- .../ocap-kernel/src/vats/SubclusterManager.ts | 81 +++++++++++++++---- 3 files changed, 104 insertions(+), 20 deletions(-) diff --git a/packages/ocap-kernel/src/Kernel.test.ts b/packages/ocap-kernel/src/Kernel.test.ts index ca1dde78b8..09e7d2d3a5 100644 --- a/packages/ocap-kernel/src/Kernel.test.ts +++ b/packages/ocap-kernel/src/Kernel.test.ts @@ -34,6 +34,8 @@ const mocks = vi.hoisted(() => { } waitForCrank = vi.fn().mockResolvedValue(undefined); + + resolvePromises = vi.fn(); } class RemoteManager { diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index 43c15657cf..79adafc5eb 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -4,7 +4,7 @@ import type { Mocked } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { KernelQueue } from '../KernelQueue.ts'; -import { kser } from '../liveslots/kernel-marshal.ts'; +import { kser, kslot } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { VatId, @@ -65,10 +65,15 @@ describe('SubclusterManager', () => { getRootObject: vi.fn(), deleteVatConfig: vi.fn(), markVatAsTerminated: vi.fn(), + initKernelPromise: vi + .fn() + .mockReturnValue(['kp1', { state: 'unresolved', subscribers: [] }]), + setPromiseDecider: vi.fn(), } as unknown as Mocked; mockKernelQueue = { waitForCrank: vi.fn().mockResolvedValue(undefined), + resolvePromises: vi.fn(), } as unknown as Mocked; mockVatManager = { @@ -116,10 +121,15 @@ describe('SubclusterManager', () => { 'testVat', 's1', ); - expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + // queueMessage targets the kernel promise for the bootstrap vat's root + expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ { testVat: expect.anything() }, {}, ]); + // kernel promise resolved to the actual root KRef once the vat launched + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp1', false, kser(kslot('ko1'))], + ]); expect(result).toStrictEqual({ subclusterId: 's1', rootKref: 'ko1', @@ -135,6 +145,10 @@ describe('SubclusterManager', () => { bob: { sourceSpec: 'bob.js' }, }, }; + // Distinct kernel promise KRefs for alice and bob + (mockKernelStore.initKernelPromise as ReturnType) + .mockReturnValueOnce(['kp1', { state: 'unresolved', subscribers: [] }]) + .mockReturnValueOnce(['kp2', { state: 'unresolved', subscribers: [] }]); mockVatManager.launchVat .mockResolvedValueOnce('ko1' as KRef) .mockResolvedValueOnce('ko2' as KRef); @@ -152,6 +166,13 @@ describe('SubclusterManager', () => { 'bob', 's1', ); + // Both root kernel promises resolved to their respective rootKRefs + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp1', false, kser(kslot('ko1'))], + ]); + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp2', false, kser(kslot('ko2'))], + ]); }); it('includes unrestricted kernel services when specified', async () => { @@ -170,7 +191,7 @@ describe('SubclusterManager', () => { await subclusterManager.launchSubcluster(config); expect(mockGetKernelService).toHaveBeenCalledWith('testService'); - expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ expect.anything(), { testService: expect.anything() }, ]); @@ -210,7 +231,7 @@ describe('SubclusterManager', () => { await subclusterManager.launchSubcluster(config, { isSystem: true }); expect(mockGetKernelService).toHaveBeenCalledWith('kernelFacet'); - expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ expect.anything(), { kernelFacet: expect.anything() }, ]); @@ -303,6 +324,12 @@ describe('SubclusterManager', () => { }); mockVatManager.launchVat.mockRejectedValue(new Error('vat boom')); + // Service lookup now happens before vat launch, so the IO channel + // service must be registered for the test to reach the vat launch step. + (mockGetKernelService as ReturnType).mockReturnValue({ + kref: 'ko99', + systemOnly: false, + }); const config: ClusterConfig = { bootstrap: 'testVat', @@ -365,6 +392,12 @@ describe('SubclusterManager', () => { }); mockVatManager.launchVat.mockRejectedValue(new Error('launch boom')); + // Service lookup now happens before vat launch, so the IO channel + // service must be registered for the test to reach the vat launch step. + (mockGetKernelService as ReturnType).mockReturnValue({ + kref: 'ko99', + systemOnly: false, + }); const config: ClusterConfig = { bootstrap: 'testVat', diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index a32ac7a5d4..53416960d6 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -5,7 +5,7 @@ import { Logger } from '@metamask/logger'; import type { IOManager } from '../io/IOManager.ts'; import type { KernelQueue } from '../KernelQueue.ts'; import type { VatManager } from './VatManager.ts'; -import { kslot, kunser } from '../liveslots/kernel-marshal.ts'; +import { kser, kslot, kunser } from '../liveslots/kernel-marshal.ts'; import type { SlotValue } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { @@ -302,17 +302,27 @@ export class SubclusterManager { rootKref: KRef; bootstrapResult: CapData | undefined; }> { - const rootIds: Record = {}; + const vatEntries = Object.entries(config.vats); + + // Pre-allocate a kernel promise for each vat's root object. All sync + // work runs here, before any launch's first await, so vatId allocation + // and store writes are ordered deterministically. + const rootPromiseKrefs: Record = {}; + for (const [vatName] of vatEntries) { + const [kpid] = this.#kernelStore.initKernelPromise(); + this.#kernelStore.setPromiseDecider(kpid, 'kernel'); + rootPromiseKrefs[vatName] = kpid; + } + + // Build the roots map from kernel promise KRefs. Bootstrap receives these + // as promises that resolve as each vat comes online. const roots: Record = {}; - for (const [vatName, vatConfig] of Object.entries(config.vats)) { - const rootRef = await this.#vatManager.launchVat( - vatConfig, - vatName, - subclusterId, - ); - rootIds[vatName] = rootRef; - roots[vatName] = kslot(rootRef, 'vatRoot'); + for (const [vatName] of vatEntries) { + // vatName was added in the loop above, so the entry is always present + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + roots[vatName] = kslot(rootPromiseKrefs[vatName]!, 'vatRoot'); } + const services: Record = {}; const ioNames = config.io ? new Set(Object.keys(config.io)) @@ -334,16 +344,55 @@ export class SubclusterManager { throw Error(`no registered kernel service '${lookupName}'`); } } - const rootKref = rootIds[config.bootstrap]; + + // Queue the bootstrap message immediately, targeting the bootstrap vat's + // (still-unresolved) root kernel promise. KernelRouter.#routeMessage sees + // an unresolved kp and parks the send on the promise via + // enqueuePromiseMessage; it is forwarded once that promise resolves. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const bootstrapRootPromiseKref = rootPromiseKrefs[config.bootstrap]!; + const bootstrapResultPromise = this.#queueMessage( + bootstrapRootPromiseKref, + 'bootstrap', + [roots, services], + ); + // Attach a no-op rejection handler so that if Promise.all below rejects + // first (a vat fails to launch) we don't get an unhandled-rejection + // warning while the bootstrap promise waits for kernel cleanup to settle. + bootstrapResultPromise.catch(() => undefined); + + // Launch all vats concurrently. As each vat's initVat handshake + // completes, resolve its root kernel promise so queued messages are + // forwarded by the run loop. + let rootKref: KRef | undefined; + await Promise.all( + vatEntries.map(async ([vatName, vatConfig]) => + this.#vatManager + .launchVat(vatConfig, vatName, subclusterId) + .then((resolvedRootRef) => { + if (vatName === config.bootstrap) { + rootKref = resolvedRootRef; + } + return this.#kernelQueue.resolvePromises('kernel', [ + [ + // vatName was added in the pre-allocation loop above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + rootPromiseKrefs[vatName]!, + false, + kser(kslot(resolvedRootRef)), + ], + ]); + }), + ), + ); + if (!rootKref) { throw new Error( - `Bootstrap vat "${config.bootstrap}" not found in rootIds`, + `Bootstrap vat "${config.bootstrap}" did not yield a root KRef`, ); } - const bootstrapResult = await this.#queueMessage(rootKref, 'bootstrap', [ - roots, - services, - ]); + + const bootstrapResult = await bootstrapResultPromise; const unserialized = kunser(bootstrapResult); if (unserialized instanceof Error) { throw unserialized; From 39cb67c49c8edb196a310db1adb9ad714ad02ccc Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:55:04 -0400 Subject: [PATCH 02/15] refactor(ocap-kernel): merge rootPromiseKrefs and roots loops in #launchVatsForSubcluster Both loops iterate vatEntries identically. Building roots[vatName] from the locally-scoped kpid in the same pass eliminates the second loop, the ! non-null assertion, and the eslint-disable comment that accompanied it. Co-Authored-By: Claude Sonnet 4.6 --- .../ocap-kernel/src/vats/SubclusterManager.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 53416960d6..244801807d 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -304,23 +304,17 @@ export class SubclusterManager { }> { const vatEntries = Object.entries(config.vats); - // Pre-allocate a kernel promise for each vat's root object. All sync - // work runs here, before any launch's first await, so vatId allocation - // and store writes are ordered deterministically. + // Pre-allocate a kernel promise for each vat's root object and build the + // roots map in one pass. All sync work runs here, before any launch's + // first await, so vatId allocation and store writes are ordered + // deterministically. const rootPromiseKrefs: Record = {}; + const roots: Record = {}; for (const [vatName] of vatEntries) { const [kpid] = this.#kernelStore.initKernelPromise(); this.#kernelStore.setPromiseDecider(kpid, 'kernel'); rootPromiseKrefs[vatName] = kpid; - } - - // Build the roots map from kernel promise KRefs. Bootstrap receives these - // as promises that resolve as each vat comes online. - const roots: Record = {}; - for (const [vatName] of vatEntries) { - // vatName was added in the loop above, so the entry is always present - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - roots[vatName] = kslot(rootPromiseKrefs[vatName]!, 'vatRoot'); + roots[vatName] = kslot(kpid, 'vatRoot'); } const services: Record = {}; From c4d3087f438add0add7421740b5d2ba8b60e4821 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:56:53 -0400 Subject: [PATCH 03/15] refactor(ocap-kernel): replace mutable let rootKref with indexed Promise.all result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of mutating an outer variable via a .then() side-effect, collect the resolved root refs as the return value of Promise.all and extract the bootstrap vat's ref by index. Promise.all preserves insertion order, so the index is stable. config.bootstrap is guaranteed present (validated before this method is called), so the as-KRef cast is correct. Also removes the dead-code guard that followed — if Promise.all resolves, every vat (including bootstrap) launched successfully, so rootRefs[idx] is always defined. Co-Authored-By: Claude Sonnet 4.6 --- .../ocap-kernel/src/vats/SubclusterManager.ts | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 244801807d..91f27d6d05 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -343,10 +343,9 @@ export class SubclusterManager { // (still-unresolved) root kernel promise. KernelRouter.#routeMessage sees // an unresolved kp and parks the send on the promise via // enqueuePromiseMessage; it is forwarded once that promise resolves. - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const bootstrapRootPromiseKref = rootPromiseKrefs[config.bootstrap]!; + // config.bootstrap is guaranteed present — validated in launchSubcluster. const bootstrapResultPromise = this.#queueMessage( - bootstrapRootPromiseKref, + rootPromiseKrefs[config.bootstrap] as KRef, 'bootstrap', [roots, services], ); @@ -358,33 +357,30 @@ export class SubclusterManager { // Launch all vats concurrently. As each vat's initVat handshake // completes, resolve its root kernel promise so queued messages are // forwarded by the run loop. - let rootKref: KRef | undefined; - await Promise.all( + const rootRefs = await Promise.all( vatEntries.map(async ([vatName, vatConfig]) => this.#vatManager .launchVat(vatConfig, vatName, subclusterId) .then((resolvedRootRef) => { - if (vatName === config.bootstrap) { - rootKref = resolvedRootRef; - } - return this.#kernelQueue.resolvePromises('kernel', [ + this.#kernelQueue.resolvePromises('kernel', [ + // vatName is always present — added in the pre-allocation loop [ - // vatName was added in the pre-allocation loop above - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - rootPromiseKrefs[vatName]!, + rootPromiseKrefs[vatName] as KRef, false, kser(kslot(resolvedRootRef)), ], ]); + return resolvedRootRef; }), ), ); - if (!rootKref) { - throw new Error( - `Bootstrap vat "${config.bootstrap}" did not yield a root KRef`, - ); - } + // Promise.all preserves insertion order; config.bootstrap is guaranteed + // present — validated in launchSubcluster before this method is called. + const bootstrapIdx = vatEntries.findIndex( + ([name]) => name === config.bootstrap, + ); + const rootKref = rootRefs[bootstrapIdx] as KRef; const bootstrapResult = await bootstrapResultPromise; const unserialized = kunser(bootstrapResult); From e8ec4500fdfa739fa13c6f7c0053ef8fb156825a Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:00:16 -0400 Subject: [PATCH 04/15] fix(ocap-kernel): reject peer kernel promises when a vat fails to launch When a vat in a multi-vat subcluster fails to launch, its pre-allocated kernel promise was left unresolved. Any bootstrap message pipelined through that promise (E(roots.failingPeer).method()) would park forever. Add a .catch() to the launchVat chain that calls resolvePromises with rejected=true and a VAT_TERMINATED kernel error, then re-throws so the outer Promise.all still rejects and launchSubcluster surfaces the error. Co-Authored-By: Claude Sonnet 4.6 --- .../src/vats/SubclusterManager.test.ts | 44 ++++++++++++++++++- .../ocap-kernel/src/vats/SubclusterManager.ts | 19 +++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index 79adafc5eb..de9129a514 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -4,7 +4,7 @@ import type { Mocked } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { KernelQueue } from '../KernelQueue.ts'; -import { kser, kslot } from '../liveslots/kernel-marshal.ts'; +import { kser, kslot, makeKernelError } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { VatId, @@ -175,6 +175,48 @@ describe('SubclusterManager', () => { ]); }); + it('rejects peer kernel promise when a non-bootstrap vat fails to launch', async () => { + const config: ClusterConfig = { + bootstrap: 'alice', + vats: { + alice: { sourceSpec: 'alice.js' }, + bob: { sourceSpec: 'bob.js' }, + }, + }; + (mockKernelStore.initKernelPromise as ReturnType) + .mockReturnValueOnce(['kp1', { state: 'unresolved', subscribers: [] }]) + .mockReturnValueOnce(['kp2', { state: 'unresolved', subscribers: [] }]); + const bobError = new Error('bob exploded'); + mockVatManager.launchVat + .mockResolvedValueOnce('ko1' as KRef) + .mockRejectedValueOnce(bobError); + + await expect(subclusterManager.launchSubcluster(config)).rejects.toThrow( + 'bob exploded', + ); + + // alice's promise resolved successfully + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp1', false, kser(kslot('ko1'))], + ]); + // bob's promise rejected with a VAT_TERMINATED kernel error + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp2', true, makeKernelError('VAT_TERMINATED', 'bob exploded')], + ]); + }); + + it('queues bootstrap message before launching vats', async () => { + const config = createMockClusterConfig(); + + await subclusterManager.launchSubcluster(config); + + const queueOrder = (mockQueueMessage as ReturnType).mock + .invocationCallOrder[0]; + const launchOrder = (mockVatManager.launchVat as ReturnType) + .mock.invocationCallOrder[0]; + expect(queueOrder).toBeLessThan(launchOrder as number); + }); + it('includes unrestricted kernel services when specified', async () => { const config: ClusterConfig = { bootstrap: 'testVat', diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 91f27d6d05..a25dc5d76f 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -5,7 +5,12 @@ import { Logger } from '@metamask/logger'; import type { IOManager } from '../io/IOManager.ts'; import type { KernelQueue } from '../KernelQueue.ts'; import type { VatManager } from './VatManager.ts'; -import { kser, kslot, kunser } from '../liveslots/kernel-marshal.ts'; +import { + kser, + kslot, + kunser, + makeKernelError, +} from '../liveslots/kernel-marshal.ts'; import type { SlotValue } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { @@ -371,6 +376,18 @@ export class SubclusterManager { ], ]); return resolvedRootRef; + }) + .catch((error: unknown) => { + const detail = + error instanceof Error ? error.message : String(error); + this.#kernelQueue.resolvePromises('kernel', [ + [ + rootPromiseKrefs[vatName] as KRef, + true, + makeKernelError('VAT_TERMINATED', detail), + ], + ]); + throw error; }), ), ); From 1d0d59e8442565fd5b8d48a701e951af50fc89c0 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:07:59 -0400 Subject: [PATCH 05/15] test(kernel-test): exercise unhappy path where bootstrap observes peer rejection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a peer-rejection-bootstrap vat that calls E(peer).ping() and logs whether the call resolves or rejects, making peer-vat-launch failures observable from the bootstrap vat. Add an integration test that launches a two-vat cluster where the peer vat (error-build-throw) always fails to build its root object. The test asserts that launchSubcluster rejects and that, after the kernel run loop drains, the bootstrap vat has logged a rejection message containing 'VAT_TERMINATED' — confirming that the rejected kernel promise propagates to the bootstrap vat via E() pipelining. Co-Authored-By: Claude Sonnet 4.6 --- .../kernel-test/src/cluster-launch.test.ts | 48 +++++++++++++++++++ .../src/vats/peer-rejection-bootstrap.ts | 28 +++++++++++ 2 files changed, 76 insertions(+) create mode 100644 packages/kernel-test/src/vats/peer-rejection-bootstrap.ts diff --git a/packages/kernel-test/src/cluster-launch.test.ts b/packages/kernel-test/src/cluster-launch.test.ts index bc28f72249..4b7e884a77 100644 --- a/packages/kernel-test/src/cluster-launch.test.ts +++ b/packages/kernel-test/src/cluster-launch.test.ts @@ -1,4 +1,5 @@ import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; +import { waitUntilQuiescent } from '@metamask/kernel-utils'; import { Logger } from '@metamask/logger'; import type { LogEntry } from '@metamask/logger'; import type { Kernel } from '@metamask/ocap-kernel'; @@ -123,3 +124,50 @@ describe('cluster initialization', { timeout: 4_000 }, () => { ]); }); }); + +describe('peer rejection propagation', { timeout: 10_000 }, () => { + let logger: Logger; + let entries: LogEntry[]; + let kernel: Kernel; + + beforeEach(async () => { + const testLogger = makeTestLogger(); + logger = testLogger.logger; + entries = testLogger.entries; + const database = await makeSQLKernelDatabase({}); + kernel = await makeKernel( + database, + true, + logger.subLogger({ tags: ['test'] }), + ); + }); + + it('bootstrap observes peer rejection when a peer vat fails to launch', async () => { + await expect( + kernel.launchSubcluster({ + bootstrap: 'main', + vats: { + main: { + bundleSpec: getBundleSpec('peer-rejection-bootstrap'), + parameters: {}, + }, + peer: { + bundleSpec: getBundleSpec('error-build-throw'), + parameters: {}, + }, + }, + }), + ).rejects.toMatchObject({ + message: expect.stringMatching(/^Failed to launch vat \S+ \(peer\)$/u), + }); + + // Let the kernel run loop deliver the parked bootstrap message to the + // main vat, which will observe the peer's rejected root promise. + await waitUntilQuiescent(200); + + const vatLogs = extractTestLogs(entries, 'console'); + expect(vatLogs).toContainEqual( + expect.stringMatching(/^peer rejected:.*VAT_TERMINATED/u), + ); + }); +}); diff --git a/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts b/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts new file mode 100644 index 0000000000..890102ef9f --- /dev/null +++ b/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts @@ -0,0 +1,28 @@ +import { E } from '@endo/eventual-send'; +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Bootstrap vat for testing peer-rejection propagation. + * Receives a `peer` root reference that may be a rejected kernel promise + * (e.g. if the peer vat failed to launch), and logs whether calls resolve + * or reject so integration tests can inspect the outcome. + * + * @returns The root object for this vat. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function buildRootObject() { + return makeDefaultExo('root', { + async bootstrap({ peer }: { peer: unknown }) { + await E(peer as object) + .ping() + // eslint-disable-next-line no-console + .then(() => console.log('peer resolved')) + .catch((error: unknown) => { + const message = + error instanceof Error ? error.message : String(error); + // eslint-disable-next-line no-console + console.log(`peer rejected: ${message}`); + }); + }, + }); +} From a0e79808d781e4edb7ca40f85309216b6029f05a Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:55:39 -0400 Subject: [PATCH 06/15] fix(ocap-kernel): send bootstrap after allSettled to avoid kunser [Promise] failures Switch #launchVatsForSubcluster from Promise.all with pre-allocated kp refs to Promise.allSettled. Wait for all vats to settle, then send bootstrap with real ko refs for succeeded vats and immediately-rejected kp for failed peers. The previous approach sent bootstrap before any vat launched, so bootstrap received pending kp refs. If bootstrap returned a value pipelined through those refs, the result CapData contained kp slots, and kunser() would fail with "value is not durable: '[Promise]'" when trying to deserialize them. The new approach preserves concurrent vat launch (allSettled, not allSettled- then-serial) while ensuring bootstrap always gets concrete refs. Failed peer vats still get an immediately-rejected kernel promise so bootstrap can observe the failure via E(roots.peer).method() pipelining. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/CHANGELOG.md | 3 + .../src/vats/SubclusterManager.test.ts | 57 +++------ .../ocap-kernel/src/vats/SubclusterManager.ts | 118 ++++++++---------- 3 files changed, 73 insertions(+), 105 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index de6c99c0e5..d643b10b2b 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Launch all vats in a subcluster concurrently during `launchSubcluster`, reducing startup latency from serial to parallel ([#983](https://github.com/MetaMask/ocap-kernel/pull/983)) + - Bootstrap receives a rejected kernel promise for any peer vat that fails to launch, observable via `E(roots.peer).method()` pipelining + - Add `fetch`, `Request`, `Headers`, and `Response` to available vat endowments ([#942](https://github.com/MetaMask/ocap-kernel/pull/942)) - Add `VatConfig.network: { allowedHosts: string[] }`; requesting `'fetch'` without it rejects `initVat` - Integrate Snaps attenuated endowment factories into vat globals ([#937](https://github.com/MetaMask/ocap-kernel/pull/937)) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index de9129a514..5e2050926c 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -4,7 +4,7 @@ import type { Mocked } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { KernelQueue } from '../KernelQueue.ts'; -import { kser, kslot, makeKernelError } from '../liveslots/kernel-marshal.ts'; +import { kser, makeKernelError } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { VatId, @@ -121,15 +121,12 @@ describe('SubclusterManager', () => { 'testVat', 's1', ); - // queueMessage targets the kernel promise for the bootstrap vat's root - expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ + // queueMessage targets the real root KRef once all vats have launched + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ { testVat: expect.anything() }, {}, ]); - // kernel promise resolved to the actual root KRef once the vat launched - expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ - ['kp1', false, kser(kslot('ko1'))], - ]); + expect(mockKernelQueue.resolvePromises).not.toHaveBeenCalled(); expect(result).toStrictEqual({ subclusterId: 's1', rootKref: 'ko1', @@ -145,10 +142,6 @@ describe('SubclusterManager', () => { bob: { sourceSpec: 'bob.js' }, }, }; - // Distinct kernel promise KRefs for alice and bob - (mockKernelStore.initKernelPromise as ReturnType) - .mockReturnValueOnce(['kp1', { state: 'unresolved', subscribers: [] }]) - .mockReturnValueOnce(['kp2', { state: 'unresolved', subscribers: [] }]); mockVatManager.launchVat .mockResolvedValueOnce('ko1' as KRef) .mockResolvedValueOnce('ko2' as KRef); @@ -166,13 +159,12 @@ describe('SubclusterManager', () => { 'bob', 's1', ); - // Both root kernel promises resolved to their respective rootKRefs - expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ - ['kp1', false, kser(kslot('ko1'))], - ]); - expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ - ['kp2', false, kser(kslot('ko2'))], + // bootstrap receives real ko refs for both vats; no kernel promises allocated + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + { alice: expect.anything(), bob: expect.anything() }, + {}, ]); + expect(mockKernelQueue.resolvePromises).not.toHaveBeenCalled(); }); it('rejects peer kernel promise when a non-bootstrap vat fails to launch', async () => { @@ -183,9 +175,6 @@ describe('SubclusterManager', () => { bob: { sourceSpec: 'bob.js' }, }, }; - (mockKernelStore.initKernelPromise as ReturnType) - .mockReturnValueOnce(['kp1', { state: 'unresolved', subscribers: [] }]) - .mockReturnValueOnce(['kp2', { state: 'unresolved', subscribers: [] }]); const bobError = new Error('bob exploded'); mockVatManager.launchVat .mockResolvedValueOnce('ko1' as KRef) @@ -195,28 +184,20 @@ describe('SubclusterManager', () => { 'bob exploded', ); - // alice's promise resolved successfully - expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ - ['kp1', false, kser(kslot('ko1'))], + // bootstrap receives alice's real ko1 root ref as queueMessage target + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + { alice: expect.anything(), bob: expect.anything() }, + {}, ]); + // initKernelPromise called once — only for bob's rejected promise + expect(mockKernelStore.initKernelPromise).toHaveBeenCalledTimes(1); // bob's promise rejected with a VAT_TERMINATED kernel error + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledTimes(1); expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ - ['kp2', true, makeKernelError('VAT_TERMINATED', 'bob exploded')], + ['kp1', true, makeKernelError('VAT_TERMINATED', 'bob exploded')], ]); }); - it('queues bootstrap message before launching vats', async () => { - const config = createMockClusterConfig(); - - await subclusterManager.launchSubcluster(config); - - const queueOrder = (mockQueueMessage as ReturnType).mock - .invocationCallOrder[0]; - const launchOrder = (mockVatManager.launchVat as ReturnType) - .mock.invocationCallOrder[0]; - expect(queueOrder).toBeLessThan(launchOrder as number); - }); - it('includes unrestricted kernel services when specified', async () => { const config: ClusterConfig = { bootstrap: 'testVat', @@ -233,7 +214,7 @@ describe('SubclusterManager', () => { await subclusterManager.launchSubcluster(config); expect(mockGetKernelService).toHaveBeenCalledWith('testService'); - expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ expect.anything(), { testService: expect.anything() }, ]); @@ -273,7 +254,7 @@ describe('SubclusterManager', () => { await subclusterManager.launchSubcluster(config, { isSystem: true }); expect(mockGetKernelService).toHaveBeenCalledWith('kernelFacet'); - expect(mockQueueMessage).toHaveBeenCalledWith('kp1', 'bootstrap', [ + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ expect.anything(), { kernelFacet: expect.anything() }, ]); diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index a25dc5d76f..dba059f870 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -5,12 +5,7 @@ import { Logger } from '@metamask/logger'; import type { IOManager } from '../io/IOManager.ts'; import type { KernelQueue } from '../KernelQueue.ts'; import type { VatManager } from './VatManager.ts'; -import { - kser, - kslot, - kunser, - makeKernelError, -} from '../liveslots/kernel-marshal.ts'; +import { kslot, kunser, makeKernelError } from '../liveslots/kernel-marshal.ts'; import type { SlotValue } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { @@ -309,19 +304,6 @@ export class SubclusterManager { }> { const vatEntries = Object.entries(config.vats); - // Pre-allocate a kernel promise for each vat's root object and build the - // roots map in one pass. All sync work runs here, before any launch's - // first await, so vatId allocation and store writes are ordered - // deterministically. - const rootPromiseKrefs: Record = {}; - const roots: Record = {}; - for (const [vatName] of vatEntries) { - const [kpid] = this.#kernelStore.initKernelPromise(); - this.#kernelStore.setPromiseDecider(kpid, 'kernel'); - rootPromiseKrefs[vatName] = kpid; - roots[vatName] = kslot(kpid, 'vatRoot'); - } - const services: Record = {}; const ioNames = config.io ? new Set(Object.keys(config.io)) @@ -344,66 +326,68 @@ export class SubclusterManager { } } - // Queue the bootstrap message immediately, targeting the bootstrap vat's - // (still-unresolved) root kernel promise. KernelRouter.#routeMessage sees - // an unresolved kp and parks the send on the promise via - // enqueuePromiseMessage; it is forwarded once that promise resolves. - // config.bootstrap is guaranteed present — validated in launchSubcluster. - const bootstrapResultPromise = this.#queueMessage( - rootPromiseKrefs[config.bootstrap] as KRef, - 'bootstrap', - [roots, services], - ); - // Attach a no-op rejection handler so that if Promise.all below rejects - // first (a vat fails to launch) we don't get an unhandled-rejection - // warning while the bootstrap promise waits for kernel cleanup to settle. - bootstrapResultPromise.catch(() => undefined); - - // Launch all vats concurrently. As each vat's initVat handshake - // completes, resolve its root kernel promise so queued messages are - // forwarded by the run loop. - const rootRefs = await Promise.all( + // Launch all vats concurrently. getNextVatId() runs synchronously before + // each launchVat's first await, so vat-ID allocation order is deterministic. + const launchResults = await Promise.allSettled( vatEntries.map(async ([vatName, vatConfig]) => - this.#vatManager - .launchVat(vatConfig, vatName, subclusterId) - .then((resolvedRootRef) => { - this.#kernelQueue.resolvePromises('kernel', [ - // vatName is always present — added in the pre-allocation loop - [ - rootPromiseKrefs[vatName] as KRef, - false, - kser(kslot(resolvedRootRef)), - ], - ]); - return resolvedRootRef; - }) - .catch((error: unknown) => { - const detail = - error instanceof Error ? error.message : String(error); - this.#kernelQueue.resolvePromises('kernel', [ - [ - rootPromiseKrefs[vatName] as KRef, - true, - makeKernelError('VAT_TERMINATED', detail), - ], - ]); - throw error; - }), + this.#vatManager.launchVat(vatConfig, vatName, subclusterId), ), ); - // Promise.all preserves insertion order; config.bootstrap is guaranteed - // present — validated in launchSubcluster before this method is called. + // If the bootstrap vat itself failed to launch, throw immediately. const bootstrapIdx = vatEntries.findIndex( ([name]) => name === config.bootstrap, ); - const rootKref = rootRefs[bootstrapIdx] as KRef; + const bootstrapLaunchResult = + launchResults[bootstrapIdx] ?? + Fail`no launch result for bootstrap vat '${config.bootstrap}'`; + if (bootstrapLaunchResult.status === 'rejected') { + throw bootstrapLaunchResult.reason; + } + const rootKref = bootstrapLaunchResult.value; - const bootstrapResult = await bootstrapResultPromise; + // Build the roots map. Succeeded vats receive real ko refs; failed peer + // vats receive an immediately-rejected kernel promise so bootstrap can + // observe the failure via E(roots.peer).method() pipelining. + const roots: Record = {}; + let firstPeerFailure: Error | undefined; + for (let i = 0; i < vatEntries.length; i++) { + const [vatName] = vatEntries[i] ?? Fail`missing vat entry at index ${i}`; + const result = + launchResults[i] ?? Fail`missing launch result at index ${i}`; + if (result.status === 'fulfilled') { + roots[vatName] = kslot(result.value); + } else { + // launchVat always wraps failures in new Error(...), so reason is an Error + const peerError = + result.reason instanceof Error + ? result.reason + : new Error(String(result.reason)); + firstPeerFailure ??= peerError; + const [kpid] = this.#kernelStore.initKernelPromise(); + this.#kernelStore.setPromiseDecider(kpid, 'kernel'); + this.#kernelQueue.resolvePromises('kernel', [ + [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], + ]); + roots[vatName] = kslot(kpid, 'vatRoot'); + } + } + + const bootstrapResult = await this.#queueMessage(rootKref, 'bootstrap', [ + roots, + services, + ]); const unserialized = kunser(bootstrapResult); if (unserialized instanceof Error) { throw unserialized; } + + // If any peer vats failed to launch, propagate after bootstrap has had a + // chance to observe and handle the failures. + if (firstPeerFailure !== undefined) { + throw firstPeerFailure; + } + return { rootKref, bootstrapResult }; } From 00c9d43835415b8c46519e6ad8071f0fecaf3172 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:17:08 -0400 Subject: [PATCH 07/15] fix(kernel-test): remove hardcoded ko refs broken by concurrent vat launch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With concurrent subcluster startup (PR #983), ko refs are assigned in the order vat workers respond — not the order vats are declared. Tests that hardcoded ko4/ko5/ko6 for alice/bob/carol broke because those refs are now non-deterministic. - Add `vatRootKrefs: Record` to `SubclusterLaunchResult` so callers can look up the root KRef of any vat by name - persistence.test.ts: use `rootKref` from `launchSubcluster` instead of hardcoded ko4 for the multi-vat coordinator test - resume.test.ts: use `vatRootKrefs.alice/bob/carol` instead of ko4/ko5/ko6 - rejection.test.ts: sort before comparing `getVatIds()` output; Map insertion order is non-deterministic with concurrent launch Co-Authored-By: Claude Sonnet 4.6 --- packages/kernel-test/src/persistence.test.ts | 14 ++++++--- packages/kernel-test/src/rejection.test.ts | 2 +- packages/kernel-test/src/resume.test.ts | 29 ++++++++++++------- packages/ocap-kernel/src/Kernel.test.ts | 1 + packages/ocap-kernel/src/types.ts | 2 ++ .../src/vats/SubclusterManager.test.ts | 2 ++ .../ocap-kernel/src/vats/SubclusterManager.ts | 13 +++++---- 7 files changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/kernel-test/src/persistence.test.ts b/packages/kernel-test/src/persistence.test.ts index c4dd6ec814..e8b6e27815 100644 --- a/packages/kernel-test/src/persistence.test.ts +++ b/packages/kernel-test/src/persistence.test.ts @@ -96,10 +96,15 @@ describe('persistent storage', { timeout: 20_000 }, () => { false, logger.logger.subLogger({ tags: ['test'] }), ); - const result1 = await runTestVats(kernel1, multiVatCluster); - expect(result1).toBe('Coordinator initialized with 2 workers'); + // Capture rootKref directly: concurrent vat launch means the coordinator + // may not be assigned ko4, so we cannot use a hardcoded ref here. + const { bootstrapResult: launch1Result, rootKref: coordinatorRoot } = + await kernel1.launchSubcluster(multiVatCluster); await waitUntilQuiescent(); - const workResult1 = await runResume(kernel1, v1Root); + expect(kunser(launch1Result as CapData)).toBe( + 'Coordinator initialized with 2 workers', + ); + const workResult1 = await runResume(kernel1, coordinatorRoot); expect(workResult1).toBe('Work completed: Worker1(1), Worker2(1)'); await waitUntilQuiescent(); await kernel1.stop(); @@ -110,7 +115,8 @@ describe('persistent storage', { timeout: 20_000 }, () => { logger.logger.subLogger({ tags: ['test'] }), ); await new Promise((resolve) => setTimeout(resolve, 1000)); - const workResult2 = await runResume(kernel2, v1Root); + // coordinatorRoot (ko) is stable across kernel restarts. + const workResult2 = await runResume(kernel2, coordinatorRoot); expect(workResult2).toBe('Work completed: Worker1(2), Worker2(2)'); await kernel2.stop(); }); diff --git a/packages/kernel-test/src/rejection.test.ts b/packages/kernel-test/src/rejection.test.ts index 3e6c8266e4..ba268e4846 100644 --- a/packages/kernel-test/src/rejection.test.ts +++ b/packages/kernel-test/src/rejection.test.ts @@ -31,7 +31,7 @@ describe('rejection', () => { }); expect(vat).toBeDefined(); const vats = kernel.getVatIds(); - expect(vats).toStrictEqual(vatIds); + expect([...vats].sort()).toStrictEqual([...vatIds].sort()); await waitUntilQuiescent(); const vatLogs = vatIds.map((vatId) => extractTestLogs(entries, vatId)); diff --git a/packages/kernel-test/src/resume.test.ts b/packages/kernel-test/src/resume.test.ts index 0f74aff9fa..68808368f4 100644 --- a/packages/kernel-test/src/resume.test.ts +++ b/packages/kernel-test/src/resume.test.ts @@ -1,6 +1,7 @@ +import type { CapData } from '@endo/marshal'; import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { waitUntilQuiescent } from '@metamask/kernel-utils'; -import type { KRef } from '@metamask/ocap-kernel'; +import { kunser } from '@metamask/ocap-kernel'; import { describe, expect, it } from 'vitest'; import { @@ -8,7 +9,6 @@ import { makeKernel, makeTestLogger, runResume, - runTestVats, sortLogs, extractTestLogs, } from './utils.ts'; @@ -100,11 +100,6 @@ const reference = sortLogs([ ...carolResumeReference, ]); -// Vat root objects start with ko4 due to the kernel facet and other kernel service objects being created before any vats. -const v1Root: KRef = 'ko4'; -const v2Root: KRef = 'ko5'; -const v3Root: KRef = 'ko6'; - describe('restarting vats', async () => { it('exercise restart vats individually', async () => { const kernelDatabase = await makeSQLKernelDatabase({ @@ -112,9 +107,15 @@ describe('restarting vats', async () => { }); const { logger, entries } = makeTestLogger(); const kernel = await makeKernel(kernelDatabase, true, logger); - const bootstrapResult = await runTestVats(kernel, testSubcluster); - expect(bootstrapResult).toBe('bootstrap Alice'); + // Use launchSubcluster directly to get vatRootKrefs: concurrent vat launch + // means ko assignment order depends on worker startup speed. + const { bootstrapResult, vatRootKrefs } = + await kernel.launchSubcluster(testSubcluster); await waitUntilQuiescent(); + expect(kunser(bootstrapResult as CapData)).toBe('bootstrap Alice'); + const v1Root = vatRootKrefs.alice; + const v2Root = vatRootKrefs.bob; + const v3Root = vatRootKrefs.carol; await kernel.restartVat('v1'); await kernel.restartVat('v2'); await kernel.restartVat('v3'); @@ -136,9 +137,15 @@ describe('restarting vats', async () => { }); const { logger: logger1, entries: entries1 } = makeTestLogger(); const kernel1 = await makeKernel(kernelDatabase, true, logger1); - const bootstrapResult = await runTestVats(kernel1, testSubcluster); - expect(bootstrapResult).toBe('bootstrap Alice'); + // Capture vatRootKrefs from first kernel: ko refs are stable across + // kernel restarts because they are persisted in the kernel store. + const { bootstrapResult, vatRootKrefs } = + await kernel1.launchSubcluster(testSubcluster); await waitUntilQuiescent(); + expect(kunser(bootstrapResult as CapData)).toBe('bootstrap Alice'); + const v1Root = vatRootKrefs.alice; + const v2Root = vatRootKrefs.bob; + const v3Root = vatRootKrefs.carol; const { logger: logger2, entries: entries2 } = makeTestLogger(); const kernel2 = await makeKernel(kernelDatabase, false, logger2); await new Promise((resolve) => setTimeout(resolve, 1000)); diff --git a/packages/ocap-kernel/src/Kernel.test.ts b/packages/ocap-kernel/src/Kernel.test.ts index 09e7d2d3a5..625b5378d1 100644 --- a/packages/ocap-kernel/src/Kernel.test.ts +++ b/packages/ocap-kernel/src/Kernel.test.ts @@ -301,6 +301,7 @@ describe('Kernel', () => { subclusterId: 's1', bootstrapResult: { body: '{"result":"ok"}', slots: [] }, rootKref: expect.stringMatching(/^ko\d+$/u), + vatRootKrefs: { alice: expect.stringMatching(/^ko\d+$/u) }, }); }); }); diff --git a/packages/ocap-kernel/src/types.ts b/packages/ocap-kernel/src/types.ts index e5ea314b01..b265e4775d 100644 --- a/packages/ocap-kernel/src/types.ts +++ b/packages/ocap-kernel/src/types.ts @@ -746,6 +746,8 @@ export type SubclusterLaunchResult = { rootKref: KRef; /** The CapData result of calling bootstrap() on the root object, if any. */ bootstrapResult: CapData | undefined; + /** Map from vat name to root kref for all successfully launched vats. */ + vatRootKrefs: Record; }; const RemoteCommsDisconnectedStruct = object({ diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index 5e2050926c..5f44e7d0fa 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -131,6 +131,7 @@ describe('SubclusterManager', () => { subclusterId: 's1', rootKref: 'ko1', bootstrapResult: { body: '{"result":"ok"}', slots: [] }, + vatRootKrefs: { testVat: 'ko1' }, }); }); @@ -486,6 +487,7 @@ describe('SubclusterManager', () => { subclusterId: 's1', rootKref: 'ko1', bootstrapResult, + vatRootKrefs: { testVat: 'ko1' }, }); }); diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index dba059f870..2054471b93 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -135,11 +135,9 @@ export class SubclusterManager { } this.#validateServices(config, isSystem); - const { rootKref, bootstrapResult } = await this.#launchVatsForSubcluster( - subclusterId, - config, - ); - return { subclusterId, rootKref, bootstrapResult }; + const { rootKref, bootstrapResult, vatRootKrefs } = + await this.#launchVatsForSubcluster(subclusterId, config); + return { subclusterId, rootKref, bootstrapResult, vatRootKrefs }; } catch (error) { // Roll back IO channels and persisted subcluster on failure. // Cleanup is best-effort — errors must not mask the original failure. @@ -301,6 +299,7 @@ export class SubclusterManager { ): Promise<{ rootKref: KRef; bootstrapResult: CapData | undefined; + vatRootKrefs: Record; }> { const vatEntries = Object.entries(config.vats); @@ -350,6 +349,7 @@ export class SubclusterManager { // vats receive an immediately-rejected kernel promise so bootstrap can // observe the failure via E(roots.peer).method() pipelining. const roots: Record = {}; + const vatRootKrefs: Record = {}; let firstPeerFailure: Error | undefined; for (let i = 0; i < vatEntries.length; i++) { const [vatName] = vatEntries[i] ?? Fail`missing vat entry at index ${i}`; @@ -357,6 +357,7 @@ export class SubclusterManager { launchResults[i] ?? Fail`missing launch result at index ${i}`; if (result.status === 'fulfilled') { roots[vatName] = kslot(result.value); + vatRootKrefs[vatName] = result.value; } else { // launchVat always wraps failures in new Error(...), so reason is an Error const peerError = @@ -388,7 +389,7 @@ export class SubclusterManager { throw firstPeerFailure; } - return { rootKref, bootstrapResult }; + return { rootKref, bootstrapResult, vatRootKrefs }; } /** From 36d0a06d33e350307368cd6c9bf820700b8742cf Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:01:07 -0400 Subject: [PATCH 08/15] fix(ocap-kernel): replace Fail?? pattern with explicit undefined checks The local Fail tag is declared as returning Error (not never) to work around a TypeScript control-flow analysis bug. Using `x ?? Fail` therefore widens the type to `T | Error`, breaking tsc on CI even though ts-jest's transpile-only mode accepts it. Replace each occurrence with an explicit undefined guard and throw. Co-Authored-By: Claude Sonnet 4.6 --- .../ocap-kernel/src/vats/SubclusterManager.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 2054471b93..8cdbb6612a 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -337,9 +337,12 @@ export class SubclusterManager { const bootstrapIdx = vatEntries.findIndex( ([name]) => name === config.bootstrap, ); - const bootstrapLaunchResult = - launchResults[bootstrapIdx] ?? - Fail`no launch result for bootstrap vat '${config.bootstrap}'`; + // bootstrapIdx is guaranteed ≥ 0 because launchSubcluster validates that + // config.vats[config.bootstrap] exists. The undefined guard satisfies tsc. + const bootstrapLaunchResult = launchResults[bootstrapIdx]; + if (bootstrapLaunchResult === undefined) { + throw Fail`no launch result for bootstrap vat '${config.bootstrap}'`; + } if (bootstrapLaunchResult.status === 'rejected') { throw bootstrapLaunchResult.reason; } @@ -352,9 +355,12 @@ export class SubclusterManager { const vatRootKrefs: Record = {}; let firstPeerFailure: Error | undefined; for (let i = 0; i < vatEntries.length; i++) { - const [vatName] = vatEntries[i] ?? Fail`missing vat entry at index ${i}`; - const result = - launchResults[i] ?? Fail`missing launch result at index ${i}`; + const vatEntry = vatEntries[i]; + const result = launchResults[i]; + if (vatEntry === undefined || result === undefined) { + throw Fail`missing entry at index ${i}`; + } + const [vatName] = vatEntry; if (result.status === 'fulfilled') { roots[vatName] = kslot(result.value); vatRootKrefs[vatName] = result.value; From cff762ab452796588961896191d0e9999d8d6a6d Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:15:54 -0400 Subject: [PATCH 09/15] fix(ocap-kernel): flatten nested changelog bullet rejected by validator --- packages/ocap-kernel/CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index d643b10b2b..a452df3640 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -9,8 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Launch all vats in a subcluster concurrently during `launchSubcluster`, reducing startup latency from serial to parallel ([#983](https://github.com/MetaMask/ocap-kernel/pull/983)) - - Bootstrap receives a rejected kernel promise for any peer vat that fails to launch, observable via `E(roots.peer).method()` pipelining +- Launch all vats in a subcluster concurrently during `launchSubcluster`, reducing startup latency from serial to parallel; failed peer vats receive a rejected kernel promise observable via `E(roots.peer).method()` pipelining ([#983](https://github.com/MetaMask/ocap-kernel/pull/983)) - Add `fetch`, `Request`, `Headers`, and `Response` to available vat endowments ([#942](https://github.com/MetaMask/ocap-kernel/pull/942)) - Add `VatConfig.network: { allowedHosts: string[] }`; requesting `'fetch'` without it rejects `initVat` From c68db87e150c534f3050376fdfa227d5ab4f65b0 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:17:06 -0400 Subject: [PATCH 10/15] fix(ocap-kernel): guard against refcount underflow in cleanupTerminatedVat When BOYD (bringOutYourDead) runs for an importing vat before terminateAllVats, dropImports/retireImports decrements the exported object's refcount to (0,0). cleanupTerminatedVat then tries to decrement the baseline (1,1) that was set at object creation, causing an underflow. Guard: check the current reachable count before the baseline decrement and skip it if reachable is already zero. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/src/store/methods/vat.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/ocap-kernel/src/store/methods/vat.ts b/packages/ocap-kernel/src/store/methods/vat.ts index 84e065ee55..48d431dc10 100644 --- a/packages/ocap-kernel/src/store/methods/vat.ts +++ b/packages/ocap-kernel/src/store/methods/vat.ts @@ -44,7 +44,7 @@ export function getVatMethods(ctx: StoreContext) { getKernelPromise, addPromiseSubscriber, } = getPromiseMethods(ctx); - const { initKernelObject } = getObjectMethods(ctx); + const { initKernelObject, getObjectRefCount } = getObjectMethods(ctx); const { addCListEntry } = getCListMethods(ctx); const { incrementRefCount, decrementRefCount } = getRefCountMethods(ctx); @@ -261,8 +261,11 @@ export function getVatMethods(ctx: StoreContext) { const { vatSlot } = getReachableAndVatSlot(vatID, kref); ctx.kv.delete(getSlotKey(vatID, kref)); ctx.kv.delete(getSlotKey(vatID, vatSlot)); - // Decrease refcounts that belonged to the terminating vat - decrementRefCount(kref, 'cleanup|export|baseline'); + // Skip baseline decrement if GC already zeroed reachable via dropImports. + const { reachable } = getObjectRefCount(kref); + if (reachable > 0) { + decrementRefCount(kref, 'cleanup|export|baseline'); + } ctx.maybeFreeKrefs.add(kref); work.exports += 1; } From e9323e22a2e11e9481dd226880256dd0d39ae665 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:50:37 -0400 Subject: [PATCH 11/15] fix(ocap-kernel): use null-prototype objects to prevent property injection vatName and service names come from ClusterConfig (user-provided). Using a plain {} as the accumulator allowed prototype pollution if a name like '__proto__' or 'constructor' was supplied. Object.create(null) removes the prototype chain, eliminating the risk. vatRootKrefs is spread into a plain {} at the return site so callers that use toStrictEqual continue to work. Co-Authored-By: Claude Sonnet 4.6 --- .../ocap-kernel/src/vats/SubclusterManager.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 8cdbb6612a..892167431f 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -303,7 +303,10 @@ export class SubclusterManager { }> { const vatEntries = Object.entries(config.vats); - const services: Record = {}; + const services: Record = Object.create(null) as Record< + string, + SlotValue + >; const ioNames = config.io ? new Set(Object.keys(config.io)) : new Set(); @@ -351,8 +354,14 @@ export class SubclusterManager { // Build the roots map. Succeeded vats receive real ko refs; failed peer // vats receive an immediately-rejected kernel promise so bootstrap can // observe the failure via E(roots.peer).method() pipelining. - const roots: Record = {}; - const vatRootKrefs: Record = {}; + const roots: Record = Object.create(null) as Record< + string, + SlotValue + >; + const vatRootKrefs: Record = Object.create(null) as Record< + string, + KRef + >; let firstPeerFailure: Error | undefined; for (let i = 0; i < vatEntries.length; i++) { const vatEntry = vatEntries[i]; @@ -395,7 +404,7 @@ export class SubclusterManager { throw firstPeerFailure; } - return { rootKref, bootstrapResult, vatRootKrefs }; + return { rootKref, bootstrapResult, vatRootKrefs: { ...vatRootKrefs } }; } /** From 7e9324dcecea5f804f2a438af8733cebaea072f4 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:03:39 -0400 Subject: [PATCH 12/15] fix(ocap-kernel): guard against prototype injection via vat names Validate that no vat name shadows Object.prototype built-ins (e.g. __proto__, constructor) before using vatName as a property key on the roots/vatRootKrefs maps. Adds // lgtm[js/remote-property-injection] suppressions on the three write sites as a belt-and-suspenders measure for CodeQL, which does not track the null-prototype path through the TypeScript cast. Object.create(null) was considered but rejected: @endo/marshal requires Object.prototype-chained objects for CopyRecord serialization. Co-Authored-By: Claude Sonnet 4.6 --- .../ocap-kernel/src/vats/SubclusterManager.ts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 892167431f..fbc1429cbd 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -303,10 +303,7 @@ export class SubclusterManager { }> { const vatEntries = Object.entries(config.vats); - const services: Record = Object.create(null) as Record< - string, - SlotValue - >; + const services: Record = {}; const ioNames = config.io ? new Set(Object.keys(config.io)) : new Set(); @@ -354,14 +351,13 @@ export class SubclusterManager { // Build the roots map. Succeeded vats receive real ko refs; failed peer // vats receive an immediately-rejected kernel promise so bootstrap can // observe the failure via E(roots.peer).method() pipelining. - const roots: Record = Object.create(null) as Record< - string, - SlotValue - >; - const vatRootKrefs: Record = Object.create(null) as Record< - string, - KRef - >; + const roots: Record = {}; + const vatRootKrefs: Record = {}; + // Reject vat names that shadow Object.prototype built-ins (__proto__, + // constructor, etc.) before using them as property keys. + for (const [name] of vatEntries) { + !(name in Object.prototype) || Fail`invalid vat name '${name}'`; + } let firstPeerFailure: Error | undefined; for (let i = 0; i < vatEntries.length; i++) { const vatEntry = vatEntries[i]; @@ -371,8 +367,8 @@ export class SubclusterManager { } const [vatName] = vatEntry; if (result.status === 'fulfilled') { - roots[vatName] = kslot(result.value); - vatRootKrefs[vatName] = result.value; + roots[vatName] = kslot(result.value); // lgtm[js/remote-property-injection] + vatRootKrefs[vatName] = result.value; // lgtm[js/remote-property-injection] } else { // launchVat always wraps failures in new Error(...), so reason is an Error const peerError = @@ -385,7 +381,7 @@ export class SubclusterManager { this.#kernelQueue.resolvePromises('kernel', [ [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], ]); - roots[vatName] = kslot(kpid, 'vatRoot'); + roots[vatName] = kslot(kpid, 'vatRoot'); // lgtm[js/remote-property-injection] } } @@ -404,7 +400,7 @@ export class SubclusterManager { throw firstPeerFailure; } - return { rootKref, bootstrapResult, vatRootKrefs: { ...vatRootKrefs } }; + return { rootKref, bootstrapResult, vatRootKrefs }; } /** From 79efe8db6124398ffe893642bfcf588aff5c310d Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:10:23 -0400 Subject: [PATCH 13/15] fix: correct lgtm suppression comment format (space before bracket) GitHub Code Scanning requires '// lgtm [rule]' with a space, not '// lgtm[rule]' without one. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/src/vats/SubclusterManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index fbc1429cbd..f9559a710a 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -367,8 +367,8 @@ export class SubclusterManager { } const [vatName] = vatEntry; if (result.status === 'fulfilled') { - roots[vatName] = kslot(result.value); // lgtm[js/remote-property-injection] - vatRootKrefs[vatName] = result.value; // lgtm[js/remote-property-injection] + roots[vatName] = kslot(result.value); // lgtm [js/remote-property-injection] + vatRootKrefs[vatName] = result.value; // lgtm [js/remote-property-injection] } else { // launchVat always wraps failures in new Error(...), so reason is an Error const peerError = @@ -381,7 +381,7 @@ export class SubclusterManager { this.#kernelQueue.resolvePromises('kernel', [ [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], ]); - roots[vatName] = kslot(kpid, 'vatRoot'); // lgtm[js/remote-property-injection] + roots[vatName] = kslot(kpid, 'vatRoot'); // lgtm [js/remote-property-injection] } } From 26f1e79604c0133d7dd139d5e006f27e73cd83b3 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:41:51 -0400 Subject: [PATCH 14/15] fix(ocap-kernel): eliminate property injection via Object.fromEntries Accumulate roots and vatRootKrefs into entry arrays and call Object.fromEntries at the end, eliminating the obj[taintedKey] = value write pattern that CodeQL's js/remote-property-injection rule flags. @endo/marshal requires Object.prototype-chained CopyRecords, so Object.create(null) was not viable. The // lgtm suppression syntax is also not active on this repo. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/src/vats/SubclusterManager.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index f9559a710a..ff73a46683 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -351,13 +351,13 @@ export class SubclusterManager { // Build the roots map. Succeeded vats receive real ko refs; failed peer // vats receive an immediately-rejected kernel promise so bootstrap can // observe the failure via E(roots.peer).method() pipelining. - const roots: Record = {}; - const vatRootKrefs: Record = {}; // Reject vat names that shadow Object.prototype built-ins (__proto__, // constructor, etc.) before using them as property keys. for (const [name] of vatEntries) { !(name in Object.prototype) || Fail`invalid vat name '${name}'`; } + const rootsEntries: [string, SlotValue][] = []; + const vatRootKrefsEntries: [string, KRef][] = []; let firstPeerFailure: Error | undefined; for (let i = 0; i < vatEntries.length; i++) { const vatEntry = vatEntries[i]; @@ -367,8 +367,8 @@ export class SubclusterManager { } const [vatName] = vatEntry; if (result.status === 'fulfilled') { - roots[vatName] = kslot(result.value); // lgtm [js/remote-property-injection] - vatRootKrefs[vatName] = result.value; // lgtm [js/remote-property-injection] + rootsEntries.push([vatName, kslot(result.value)]); + vatRootKrefsEntries.push([vatName, result.value]); } else { // launchVat always wraps failures in new Error(...), so reason is an Error const peerError = @@ -381,9 +381,11 @@ export class SubclusterManager { this.#kernelQueue.resolvePromises('kernel', [ [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], ]); - roots[vatName] = kslot(kpid, 'vatRoot'); // lgtm [js/remote-property-injection] + rootsEntries.push([vatName, kslot(kpid, 'vatRoot')]); } } + const roots = Object.fromEntries(rootsEntries); + const vatRootKrefs = Object.fromEntries(vatRootKrefsEntries); const bootstrapResult = await this.#queueMessage(rootKref, 'bootstrap', [ roots, From 6647a0c5284902d9f16cfed6a2ffe50a75584f5a Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:37:03 -0400 Subject: [PATCH 15/15] refactor(ocap-kernel): inline vat-name guard and drop dead kslot iface arg - Move the Object.prototype name check into the main loop body, eliminating a separate pre-pass over vatEntries. - Drop the 'vatRoot' iface arg from kslot(kpid): kslot ignores iface for kp-prefixed refs (returns makeStandinPromise early), so the arg was dead code that implied it had an effect. Co-Authored-By: Claude Sonnet 4.6 --- packages/ocap-kernel/src/vats/SubclusterManager.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index ff73a46683..517d635b3c 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -351,11 +351,6 @@ export class SubclusterManager { // Build the roots map. Succeeded vats receive real ko refs; failed peer // vats receive an immediately-rejected kernel promise so bootstrap can // observe the failure via E(roots.peer).method() pipelining. - // Reject vat names that shadow Object.prototype built-ins (__proto__, - // constructor, etc.) before using them as property keys. - for (const [name] of vatEntries) { - !(name in Object.prototype) || Fail`invalid vat name '${name}'`; - } const rootsEntries: [string, SlotValue][] = []; const vatRootKrefsEntries: [string, KRef][] = []; let firstPeerFailure: Error | undefined; @@ -366,6 +361,9 @@ export class SubclusterManager { throw Fail`missing entry at index ${i}`; } const [vatName] = vatEntry; + // Reject vat names that shadow Object.prototype built-ins (__proto__, + // constructor, etc.) before using them as property keys. + !(vatName in Object.prototype) || Fail`invalid vat name '${vatName}'`; if (result.status === 'fulfilled') { rootsEntries.push([vatName, kslot(result.value)]); vatRootKrefsEntries.push([vatName, result.value]); @@ -381,7 +379,7 @@ export class SubclusterManager { this.#kernelQueue.resolvePromises('kernel', [ [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], ]); - rootsEntries.push([vatName, kslot(kpid, 'vatRoot')]); + rootsEntries.push([vatName, kslot(kpid)]); } } const roots = Object.fromEntries(rootsEntries);