From a310fe47cc8bdb681426864208321831df2d1c34 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Aug 2026 15:29:18 +0200 Subject: [PATCH 01/34] fix(ocap-kernel): make c-list import accounting symmetric (#1006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating an import c-list entry changed no refcount while tearing one down decremented both, and `initKernelObject` compensated by minting every object at (1, 1). That constant is correct for exactly one importer, which is why nothing caught it: with two importers a live capability gets dropped and retired out from under a holder, and the same unit is claimed by both an importer's drop and the owner's termination, so cleanup underflows and leaves a vat half-cleaned. Restore the increment and rebase the baseline to (0, 0), matching SwingSet, so `collectGarbage` — already a faithful port — receives the inputs it was written for. Build the invariant checker first, since every existing compensation becomes a double-count the moment the increment lands. It recomputes each kref's counts from ground truth (c-list entries and their reachable flags, run-queue and promise-queue messages, promise resolution values, pins) and reports drift in both directions: too low collects a live capability, too high leaks it. Enabled via `Kernel.make`'s `auditRefCounts` and run after every crank; on in kernel-test. The audit found four more unbalanced paths that the phantom baseline had been absorbing, each fixed here: a delivered message charged its target against the routed kref rather than the run-queue item's own, so a message routed through a resolved promise decremented an object nobody charged and leaked the promise; a notification leaked its reference on both early-return paths and decremented promises retired alongside it that nobody had taken; a message queued on an unresolved promise duplicated every reference it carried on re-enqueue; and `resolve|kpid` incremented with no matching release. Two things the baseline was silently standing in for, now explicit: vat roots are pinned for the lifetime of their vat (a root is addressable whether or not anyone imports it), and GC action delivery moves the kernel's own c-list so a dropped export's flag clears and retired entries don't outlive their objects. Also fixes the stale `cle.`/`clk.` key prefixes in `getPromisesByDecider` and `deleteEndpoint`, which stopped matching the `${endpointId}.c.` layout. `getPromisesByDecider` matched nothing, so promises a terminating vat was deciding were never rejected — load bearing here, because releasing a promise's unsettled reference is what makes the cleanup path's accounting add up. Refcounts are persisted, so counts written under the old scheme are recomputed from ground truth on first open, keyed off a new `refCountScheme` entry. Co-Authored-By: Claude Opus 5 (1M context) --- .../extension/test/e2e/control-panel.test.ts | 1 - .../src/garbage-collection.test.ts | 142 ++++++- packages/kernel-test/src/persistence.test.ts | 3 +- packages/kernel-test/src/utils.ts | 3 + packages/ocap-kernel/CHANGELOG.md | 21 ++ packages/ocap-kernel/src/Kernel.ts | 11 + packages/ocap-kernel/src/KernelQueue.test.ts | 9 +- packages/ocap-kernel/src/KernelQueue.ts | 2 +- packages/ocap-kernel/src/KernelRouter.test.ts | 72 +++- packages/ocap-kernel/src/KernelRouter.ts | 52 ++- .../src/remotes/kernel/RemoteHandle.test.ts | 6 +- .../src/remotes/kernel/RemoteManager.test.ts | 9 +- packages/ocap-kernel/src/store/index.test.ts | 40 +- packages/ocap-kernel/src/store/index.ts | 12 +- .../ocap-kernel/src/store/methods/base.ts | 14 +- .../store/methods/clist-accounting.test.ts | 278 ++++++++++++++ .../src/store/methods/clist.test.ts | 88 +++-- .../ocap-kernel/src/store/methods/clist.ts | 36 +- .../ocap-kernel/src/store/methods/gc.test.ts | 15 - packages/ocap-kernel/src/store/methods/gc.ts | 6 +- .../src/store/methods/object.test.ts | 24 +- .../ocap-kernel/src/store/methods/object.ts | 11 +- .../src/store/methods/promise.test.ts | 145 +++++--- .../ocap-kernel/src/store/methods/promise.ts | 69 +++- .../src/store/methods/reachable.test.ts | 56 ++- .../src/store/methods/reachable.ts | 27 ++ .../src/store/methods/refcount-audit.test.ts | 269 +++++++++++++ .../src/store/methods/refcount-audit.ts | 352 ++++++++++++++++++ .../src/store/methods/translators.test.ts | 6 + .../src/store/methods/translators.ts | 8 + .../ocap-kernel/src/store/methods/vat.test.ts | 90 ++--- packages/ocap-kernel/src/store/methods/vat.ts | 81 ++-- packages/ocap-kernel/src/store/types.ts | 1 + packages/ocap-kernel/src/vats/VatManager.ts | 13 + 34 files changed, 1637 insertions(+), 335 deletions(-) create mode 100644 packages/ocap-kernel/src/store/methods/clist-accounting.test.ts create mode 100644 packages/ocap-kernel/src/store/methods/refcount-audit.test.ts create mode 100644 packages/ocap-kernel/src/store/methods/refcount-audit.ts diff --git a/packages/extension/test/e2e/control-panel.test.ts b/packages/extension/test/e2e/control-panel.test.ts index 167429e76e..754a4017b5 100644 --- a/packages/extension/test/e2e/control-panel.test.ts +++ b/packages/extension/test/e2e/control-panel.test.ts @@ -252,7 +252,6 @@ test.describe('Control Panel', () => { `{"key":"v3.c.o+0","value":"${v3Root}"}`, `{"key":"v3.c.${v3Promise}","value":"R p-1"}`, `{"key":"v3.c.p-1","value":"${v3Promise}"}`, - `{"key":"${v3Root}.refCount","value":"1,1"}`, `{"key":"${v3Promise}.refCount","value":"2"}`, ]; // Derived too: v1 imports the two roots as the bootstrap's calls are diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 268ed7920e..67055414f1 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -21,9 +21,11 @@ import { /** * Make a test subcluster with vats for GC testing * + * @param extraImporters - Names of additional importer vats to include, for + * topologies where more than one vat shares the same exported object. * @returns The test subcluster */ -function makeTestSubcluster(): ClusterConfig { +function makeTestSubcluster(extraImporters: string[] = []): ClusterConfig { return { bootstrap: 'exporter', forceReset: true, @@ -40,6 +42,15 @@ function makeTestSubcluster(): ClusterConfig { name: 'Importer', }, }, + ...Object.fromEntries( + extraImporters.map((name) => [ + name, + { + bundleSpec: getBundleSpec('importer-vat'), + parameters: { name }, + }, + ]), + ), }, }; } @@ -81,10 +92,11 @@ describe('Garbage Collection', () => { [objectId], ); const createObjectRef = createObjectData.slots[0] as KRef; - // Verify initial reference counts from database - const initialRefCounts = kernelStore.getObjectRefCount(createObjectRef); - expect(initialRefCounts.reachable).toBe(2); - expect(initialRefCounts.recognizable).toBe(2); + // Held only by the resolved promise's value, which still carries the slot + expect(kernelStore.getObjectRefCount(createObjectRef)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); // Send the object to the importer vat const objectRef = kunser(createObjectData); await kernel.queueMessage(importerKRef, 'storeImport', [objectRef]); @@ -116,10 +128,10 @@ describe('Garbage Collection', () => { await waitUntilQuiescent(); const createObjectRef = createObjectData.slots[0] as KRef; - // Store initial reference count information - const initialRefCounts = kernelStore.getObjectRefCount(createObjectRef); - expect(initialRefCounts.reachable).toBe(2); - expect(initialRefCounts.recognizable).toBe(2); + expect(kernelStore.getObjectRefCount(createObjectRef)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); // Store the reference in the importer vat const objectRef = kunser(createObjectData); @@ -201,4 +213,116 @@ describe('Garbage Collection', () => { ); expect(parseReplyBody(exporterFinalCheck.body)).toBe(false); }, 40000); + + describe('an object shared by two importers', () => { + let secondImporterKRef: KRef; + let secondImporterVatId: VatId; + + beforeEach(async () => { + kernelDatabase = await makeSQLKernelDatabase({ dbFilename: ':memory:' }); + kernelStore = makeKernelStore(kernelDatabase); + kernel = await makeKernel(kernelDatabase, true, makeMockLogger()); + await runTestVats(kernel, makeTestSubcluster(['Importer2'])); + + const vats = kernel.getVats(); + const idOf = (name: string): VatId => + vats.find((row) => row.config.parameters?.name === name)?.id as VatId; + exporterVatId = idOf('Exporter'); + importerVatId = idOf('Importer'); + secondImporterVatId = idOf('Importer2'); + exporterKRef = kernelStore.getRootObject(exporterVatId) as KRef; + importerKRef = kernelStore.getRootObject(importerVatId) as KRef; + secondImporterKRef = kernelStore.getRootObject( + secondImporterVatId, + ) as KRef; + }); + + /** + * Give an importer a chance to notice a dropped object and tell the kernel. + * + * @param vatId - The vat to reap. + * @param rootKRef - That vat's root, to poke with cranks afterwards. + */ + async function reapAndSettle(vatId: VatId, rootKRef: KRef): Promise { + kernel.reapVats((id) => id === vatId); + for (let i = 0; i < 3; i++) { + await kernel.queueMessage(rootKRef, 'noop', []); + await waitUntilQuiescent(500); + } + } + + it('survives until both importers let go', async () => { + const objectId = 'shared-object'; + const createObjectData = await kernel.queueMessage( + exporterKRef, + 'createObject', + [objectId], + ); + const sharedKRef = createObjectData.slots[0] as KRef; + const objectRef = kunser(createObjectData); + + for (const importer of [importerKRef, secondImporterKRef]) { + await kernel.queueMessage(importer, 'storeImport', [ + objectRef, + objectId, + ]); + } + await waitUntilQuiescent(); + + expect(kernelStore.getImporters(sharedKRef)).toStrictEqual( + [importerVatId, secondImporterVatId].sort(), + ); + // Two importers, plus the resolved createObject promise whose value + // still carries the slot + expect(kernelStore.getObjectRefCount(sharedKRef)).toStrictEqual({ + reachable: 3, + recognizable: 3, + }); + + await kernel.queueMessage(importerKRef, 'makeWeak', [objectId]); + await kernel.queueMessage(importerKRef, 'forgetImport', []); + await waitUntilQuiescent(); + await reapAndSettle(importerVatId, importerKRef); + + // The exporter must not have been told to drop it: the second importer + // legitimately still holds it + expect(kernelStore.getReachableFlag(exporterVatId, sharedKRef)).toBe( + true, + ); + expect(kernelStore.getImporters(sharedKRef)).toStrictEqual([ + secondImporterVatId, + ]); + expect( + parseReplyBody( + ( + await kernel.queueMessage(exporterKRef, 'isObjectPresent', [ + objectId, + ]) + ).body, + ), + ).toBe(true); + + expect( + parseReplyBody( + ( + await kernel.queueMessage(secondImporterKRef, 'useImport', [ + objectId, + ]) + ).body, + ), + ).toBe(objectId); + + await kernel.queueMessage(secondImporterKRef, 'makeWeak', [objectId]); + await kernel.queueMessage(secondImporterKRef, 'forgetImport', []); + await waitUntilQuiescent(); + await reapAndSettle(secondImporterVatId, secondImporterKRef); + + expect(kernelStore.getImporters(sharedKRef)).toStrictEqual([]); + // Only the createObject result's stored value still names it + expect(kernelStore.getObjectRefCount(sharedKRef)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }, 60000); + }); }); diff --git a/packages/kernel-test/src/persistence.test.ts b/packages/kernel-test/src/persistence.test.ts index e8b6e27815..b1c91f3f45 100644 --- a/packages/kernel-test/src/persistence.test.ts +++ b/packages/kernel-test/src/persistence.test.ts @@ -176,7 +176,8 @@ describe('persistent storage', { timeout: 20_000 }, () => { // Enqueue a send message into the database kv1.set('queue.run.head', '4'); kv1.set('nextPromiseId', '4'); - kv1.set(`${v1Root}.refCount`, '3,3'); + // The root's pin, plus the send being injected below. + kv1.set(`${v1Root}.refCount`, '2,2'); kv1.set('queue.kp3.head', '1'); kv1.set('queue.kp3.tail', '1'); kv1.set('kp3.state', 'unresolved'); diff --git a/packages/kernel-test/src/utils.ts b/packages/kernel-test/src/utils.ts index c255347b89..7d867f2c28 100644 --- a/packages/kernel-test/src/utils.ts +++ b/packages/kernel-test/src/utils.ts @@ -93,6 +93,9 @@ export async function makeKernel( resetStorage, logger, keySeed, + // Refcount drift is invisible to ordinary assertions until something gets + // collected out from under a live holder, so check it every crank. + auditRefCounts: true, }); return kernel; } diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 8af7998ec7..0b9328172a 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -32,10 +32,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Log a warning when a vat requests an unknown global - Export `OcapURLIssuerService` and `OcapURLRedemptionService` types so vats can type the corresponding kernel-service endowments ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) - Reference-marker sigil (`@@NAME`) at the `queueMessage` RPC boundary lets JSON-RPC callers name a live kernel object as a call argument ([#984](https://github.com/MetaMask/ocap-kernel/pull/984)) + - Anywhere in the args tree, a string of the form `@@NAME` (NAME is one or more alphanumeric characters, currently a well-formed kref) is expanded to a `kslot` standin so `kser` encodes it as a real CapData slot in the dispatched message - Purely an RPC-boundary concern: internal callers of `Kernel.queueMessage` are unaffected - Caveat: a legitimate string argument that begins with `@@` followed by alphanumerics will be misinterpreted as a marker; wrap such literals inside an object +- Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high with no holder (a leak) + - Exports the `RefCountViolation` type +- Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + ### Changed - **BREAKING:** `Kernel.make`'s `ioChannelFactory` option is now `ioListenerFactory`, and the exported `IOChannelFactory` type is replaced by `IOListener` and `IOListenerFactory`. A cluster config's `io` entries now create listeners; vats call `accept()` to obtain a channel instead of reading and writing the endowment directly ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007)) @@ -65,6 +71,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Report the database error when an aborted crank cannot be rolled back, instead of a spurious "no such savepoint" ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - The abort path recorded the rollback only after it succeeded, so a throwing rollback had the run loop try again against the savepoint `rollbackCrank` had already discarded. The second attempt's "no such savepoint" then became the reported cause of death — and since only `error.message` crosses the wire, the real failure reached neither `getStatus` nor the daemon log - Reject the run loop's promise with the same `Error` its status reports, rather than re-throwing a non-`Error` for the embedder to normalize a second time ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) +- **BREAKING:** Make c-list reference accounting symmetric: creating an import c-list entry now takes a reference, as tearing one down has always released one ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + - `initKernelObject` now births objects at `(0, 0)` instead of `(1, 1)`. The old constant made the arithmetic come out right for exactly one importer, masking the missing increment; with two importers a live capability could be dropped and retired out from under a holder + - Removes the owner-side baseline decrements in `cleanupTerminatedVat` and `forgetEndpointImports`, which double-claimed the same unit an importer's drop also spent — the source of `"koNN" underflow -1,0` escaping mid-cleanup and leaving a vat half-cleaned + - `translateRefKtoE` now re-establishes reachability, so a vat handed an object it previously dropped is counted as holding it live again + - Renames `krefsToExistingErefs` to `krefsToErefs`, which now throws on an unmapped kref instead of silently dropping it +- Pin vat root objects for the lifetime of their vat, and release the pin on termination ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + - A root is addressable while its vat lives whether or not anyone imports it; the old `(1, 1)` birth baseline was standing in for this +- Garbage-collection action delivery now moves the kernel's own c-list: `dropExports` clears the owner's reachable flag and `retireExports`/`retireImports` tear the entry down ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + - Previously the owner's flag never cleared, so the same action could be re-derived, and retired entries outlived the objects they named +- Charge a delivered message's target reference against the run-queue item's own target rather than the routed target, so a message routed through a resolved promise no longer decrements an object nobody charged while leaking the promise ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Fix the stale `cle.`/`clk.` key prefixes in `getPromisesByDecider` and `deleteEndpoint`, which no longer matched the `${endpointId}.c.` c-list layout ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) + + - `getPromisesByDecider` matched nothing, so promises a terminating vat or restarting peer was deciding were never rejected - Deserialize CapData rejections in `Kernel.queueMessage` so vat errors surface as plain `Error` objects to all callers ([#928](https://github.com/MetaMask/ocap-kernel/pull/928)) - Detect peer restart across receiver state loss so the receiving kernel no longer silently drops a restarted peer's `seq=1` messages ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) - Persist the peer's last-observed incarnation and compare it on every successful handshake; on a detected restart, clear the peer's c-list contributions and reject the promises it was deciding before the new incarnation reuses any erefs diff --git a/packages/ocap-kernel/src/Kernel.ts b/packages/ocap-kernel/src/Kernel.ts index a312e60e65..94865cb049 100644 --- a/packages/ocap-kernel/src/Kernel.ts +++ b/packages/ocap-kernel/src/Kernel.ts @@ -109,6 +109,10 @@ export class Kernel { * @param options.ioListenerFactory - Optional factory for creating IO listeners. * @param options.allowedGlobalNames - Optional list of allowed global names for vat endowments. * @param options.onRunLoopFailure - Optional handler called if the run loop dies. + * @param options.auditRefCounts - If true, verify every kref's reference + * counts against the references the kernel actually holds at the end of each + * crank, and throw on any mismatch. Intended for tests and debugging; the + * audit walks the whole store. */ // eslint-disable-next-line no-restricted-syntax private constructor( @@ -122,6 +126,7 @@ export class Kernel { ioListenerFactory?: IOListenerFactory; allowedGlobalNames?: AllowedGlobalName[]; onRunLoopFailure?: OnRunLoopFailure; + auditRefCounts?: boolean; } = {}, ) { this.#platformServices = platformServices; @@ -129,6 +134,9 @@ export class Kernel { this.#onRunLoopFailure = options.onRunLoopFailure; this.#logger = options.logger ?? new Logger('ocap-kernel'); this.#kernelStore = makeKernelStore(kernelDatabase, this.#logger); + if (options.auditRefCounts) { + this.#kernelStore.setRefCountAuditing(true); + } if (!this.#kernelStore.isInitialized()) { this.#kernelStore.markInitialized(); } @@ -249,6 +257,8 @@ export class Kernel { * @param options.systemSubclusters - Optional array of system subcluster configurations. * @param options.allowedGlobalNames - Optional list of allowed global names for vat endowments. When set, only these names from the `VatSupervisor`'s configured endowments (see `createDefaultEndowments`) are available to vats. * @param options.onRunLoopFailure - Optional handler called if the run loop dies. The kernel must be restarted after that, so an embedder that outlives it (e.g. a daemon) should use this to terminate or restart. + * @param options.auditRefCounts - If true, verify reference counts against + * ground truth at the end of each crank and throw on any mismatch. * @returns A promise for the new kernel instance. */ static async make( @@ -263,6 +273,7 @@ export class Kernel { systemSubclusters?: SystemSubclusterConfig[]; allowedGlobalNames?: AllowedGlobalName[]; onRunLoopFailure?: OnRunLoopFailure; + auditRefCounts?: boolean; } = {}, ): Promise { const kernel = new Kernel(platformServices, kernelDatabase, options); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index d7beb0e2f1..1b3bd4a35a 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -44,6 +44,7 @@ describe('KernelQueue', () => { kernelStore = { nextTerminatedVatCleanup: vi.fn(), collectGarbage: vi.fn(), + assertRefCountsIfAuditing: vi.fn(), runQueueLength: vi.fn(), dequeueRun: vi.fn(), enqueueRun: vi.fn(), @@ -652,10 +653,6 @@ describe('KernelQueue', () => { reject: rejectHandler, }); kernelQueue.resolvePromises(endpointId, [resolution], false); - expect(kernelStore.incrementRefCount).toHaveBeenCalledWith( - kpid, - 'resolve|kpid', - ); expect(kernelStore.incrementRefCount).toHaveBeenCalledWith( 'ko1', 'resolve|slot', @@ -709,10 +706,6 @@ describe('KernelQueue', () => { const insistEndpointIdSpy = vi.spyOn(types, 'insistEndpointId'); kernelQueue.resolvePromises(undefined, [resolution], false); expect(insistEndpointIdSpy).not.toHaveBeenCalled(); - expect(kernelStore.incrementRefCount).toHaveBeenCalledWith( - kpid, - 'resolve|kpid', - ); expect(kernelStore.incrementRefCount).toHaveBeenCalledWith( 'ko1', 'resolve|slot', diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 3465e93cde..afda8139c7 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -344,6 +344,7 @@ export class KernelQueue { await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); + this.#kernelStore.assertRefCountsIfAuditing(); } /** @@ -504,7 +505,6 @@ export class KernelQueue { for (const resolution of resolutions) { const [kpid, rejected, data] = resolution; - this.#kernelStore.incrementRefCount(kpid, 'resolve|kpid'); for (const slot of data.slots || []) { this.#kernelStore.incrementRefCount(slot, 'resolve|slot'); } diff --git a/packages/ocap-kernel/src/KernelRouter.test.ts b/packages/ocap-kernel/src/KernelRouter.test.ts index ea833293a6..11aa8922c1 100644 --- a/packages/ocap-kernel/src/KernelRouter.test.ts +++ b/packages/ocap-kernel/src/KernelRouter.test.ts @@ -59,9 +59,12 @@ describe('KernelRouter', () => { krefToEref: vi.fn() as unknown as MockInstance, getKpidsToRetire: vi.fn().mockReturnValue([]), translateCapDataKtoE: vi.fn(), - krefsToExistingErefs: vi.fn((_endpointId: string, krefs: string[]) => + krefsToErefs: vi.fn((_endpointId: string, krefs: string[]) => krefs.map((kref: string) => `translated-${kref}`), ) as unknown as MockInstance, + clearReachableFlag: vi.fn(), + deleteCListEntry: vi.fn(), + forgetKref: vi.fn(), createCrankSavepoint: vi.fn(), } as unknown as KernelStore; @@ -283,8 +286,35 @@ describe('KernelRouter', () => { expect(endpointHandle.deliverMessage).not.toHaveBeenCalled(); expect(result).toBeUndefined(); - // Verify that no refcount decrementation happened since we're requeuing - expect(kernelStore.decrementRefCount).not.toHaveBeenCalled(); + expect(kernelStore.decrementRefCount).toHaveBeenCalledWith( + target, + 'requeue|target', + ); + }); + + it('hands over every reference a requeued message carries', async () => { + const target = 'kp123'; + ( + kernelStore.getKernelPromise as unknown as MockInstance + ).mockReturnValueOnce({ state: 'unresolved' }); + const message: KernelMessage = { + methargs: { body: 'method args', slots: ['ko1', 'ko2'] }, + result: 'kp9', + }; + await kernelRouter.deliver({ type: 'send', target, message }); + + expect(kernelStore.enqueuePromiseMessage).toHaveBeenCalledWith( + target, + message, + ); + expect( + (kernelStore.decrementRefCount as unknown as MockInstance).mock.calls, + ).toStrictEqual([ + [target, 'requeue|target'], + ['kp9', 'requeue|result'], + ['ko1', 'requeue|slot'], + ['ko2', 'requeue|slot'], + ]); }); it('splats message when promise resolves to a non-object', async () => { @@ -649,6 +679,42 @@ describe('KernelRouter', () => { expect(result).toStrictEqual(mockCrankResult); }, ); + + it('clears the reachable flag when delivering dropExports', async () => { + await kernelRouter.deliver({ + type: 'dropExports', + endpointId: 'v1', + krefs: ['ko1', 'ko2'], + }); + + expect( + (kernelStore.clearReachableFlag as unknown as MockInstance).mock + .calls, + ).toStrictEqual([ + ['v1', 'ko1'], + ['v1', 'ko2'], + ]); + expect(kernelStore.deleteCListEntry).not.toHaveBeenCalled(); + }); + + it.each(['retireExports', 'retireImports'] as const)( + 'tears down the c-list entry when delivering %s', + async (actionType) => { + await kernelRouter.deliver({ + type: actionType, + endpointId: 'v1', + krefs: ['ko1', 'ko2'], + }); + + expect( + (kernelStore.deleteCListEntry as unknown as MockInstance).mock + .calls, + ).toStrictEqual([ + ['v1', 'ko1', 'translated-ko1'], + ['v1', 'ko2', 'translated-ko2'], + ]); + }, + ); }); describe('bringOutYourDead', () => { diff --git a/packages/ocap-kernel/src/KernelRouter.ts b/packages/ocap-kernel/src/KernelRouter.ts index 5cfb8335d4..6bd080e7c3 100644 --- a/packages/ocap-kernel/src/KernelRouter.ts +++ b/packages/ocap-kernel/src/KernelRouter.ts @@ -11,6 +11,7 @@ import { isPromiseRef } from './store/utils/promise-ref.ts'; import type { EndpointId, EndpointHandle, + ERef, KRef, KernelMessage, RunQueueItem, @@ -255,7 +256,10 @@ export class KernelRouter { 'deliver|splat|result', ); } - this.#kernelStore.decrementRefCount(target, 'deliver|splat|target'); + this.#kernelStore.decrementRefCount( + item.target, + 'deliver|splat|target', + ); for (const slot of message.methargs.slots) { this.#kernelStore.decrementRefCount(slot, 'deliver|splat|slot'); } @@ -314,12 +318,24 @@ export class KernelRouter { } else { Fail`no owner for kernel object ${target}`; } - this.#kernelStore.decrementRefCount(target, 'deliver|send|target'); + // `item.target`, not the routed `target`: a message aimed at a promise + // is charged against the promise, and routing may have resolved it to a + // different object. + this.#kernelStore.decrementRefCount(item.target, 'deliver|send|target'); for (const slot of message.methargs.slots) { this.#kernelStore.decrementRefCount(slot, 'deliver|send|slot'); } } else { + // The references move from this run queue item to the promise's queue + // entry. New holder first, so nothing transiently looks unreferenced. this.#kernelStore.enqueuePromiseMessage(target, message); + this.#kernelStore.decrementRefCount(item.target, 'requeue|target'); + if (message.result) { + this.#kernelStore.decrementRefCount(message.result, 'requeue|result'); + } + for (const slot of message.methargs.slots) { + this.#kernelStore.decrementRefCount(slot, 'requeue|slot'); + } } return crankResult; @@ -362,6 +378,9 @@ export class KernelRouter { if (state === 'unresolved') { Fail`notification on unresolved promise ${kpid}`; } + // Release the queued notification's reference up front, so the paths that + // decide there is nothing to deliver don't leak it. + this.#kernelStore.decrementRefCount(kpid, 'deliver|notify'); if (!this.#kernelStore.krefToEref(endpointId, kpid)) { // no c-list entry, already done return { didDelivery: endpointId }; @@ -385,16 +404,13 @@ export class KernelRouter { tPromise.state === 'rejected', this.#kernelStore.translateCapDataKtoE(endpointId, tPromise.value), ]); - // decrement refcount for the promise being notified - if (toResolve !== kpid) { - this.#kernelStore.decrementRefCount(toResolve, 'deliver|notify|slot'); - } } + // TODO(#1006 follow-up): SwingSet also tears down the c-list entry for each + // promise in the batch here, since the endpoint can never refer to a + // settled promise by that eref again. Left alone for now because the + // debug UI discovers exported ocap URLs by scanning these entries. const endpoint = this.#getEndpoint(endpointId); - const crankResult = await endpoint.deliverNotify(resolutions); - // Decrement reference count for processed 'notify' item - this.#kernelStore.decrementRefCount(kpid, 'deliver|notify'); - return crankResult; + return await endpoint.deliverNotify(resolutions); } /** @@ -409,7 +425,21 @@ export class KernelRouter { `@@@@ deliver ${endpointId} ${type} ${JSON.stringify(krefs)}`, ); const endpoint = this.#getEndpoint(endpointId); - const erefs = this.#kernelStore.krefsToExistingErefs(endpointId, krefs); + const erefs = this.#kernelStore.krefsToErefs(endpointId, krefs); + // Telling an endpoint to let go is also the kernel letting go. Otherwise a + // dropped export stays flagged reachable, so the same action gets derived + // again, and retired entries outlive the objects they name. + krefs.forEach((kref, index) => { + if (type === 'dropExports') { + this.#kernelStore.clearReachableFlag(endpointId, kref); + } else { + this.#kernelStore.deleteCListEntry( + endpointId, + kref, + erefs[index] as ERef, + ); + } + }); const method = `deliver${(type[0] as string).toUpperCase()}${type.slice(1)}` as | 'deliverDropExports' diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index c14539bc6c..009834a65b 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -536,7 +536,8 @@ describe('RemoteHandle', () => { for (const kref of krefs) { const { isPromise } = parseRef(kref); if (isPromise) { - expect(mockKernelStore.getRefCount(kref)).toBe(1); + // 1 for the unsettled promise, 1 for the remote's c-list entry + expect(mockKernelStore.getRefCount(kref)).toBe(2); } else { expect(mockKernelStore.getObjectRefCount(kref)).toStrictEqual({ reachable: 1, @@ -557,7 +558,7 @@ describe('RemoteHandle', () => { for (const kref of krefs) { const { isPromise } = parseRef(kref); if (isPromise) { - expect(mockKernelStore.getRefCount(kref)).toBe(1); + expect(mockKernelStore.getRefCount(kref)).toBe(2); } else { expect(mockKernelStore.getObjectRefCount(kref)).toStrictEqual({ reachable: 0, @@ -623,7 +624,6 @@ describe('RemoteHandle', () => { // As if we're no longer using it (which, in fact, we weren't), which is a // prequisite for a valid 'retireImports' delivery - mockKernelStore.decrementRefCount(koref, 'test'); mockKernelStore.clearReachableFlag(remote.remoteId, koref); // Now have the "other end" retire the import (include seq for incoming message) diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index 34e9e91502..29ca0b2da7 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -23,16 +23,13 @@ describe('RemoteManager', () => { let remoteManager: RemoteManager; let mockPlatformServices: PlatformServices; let kernelStore: ReturnType; - let kernelKVStore: ReturnType['kernelKVStore']; let mockKernelQueue: KernelQueue; let logger: Logger; let mockRemoteComms: RemoteComms; let mockFactory: ReturnType; beforeEach(() => { - const kernelDatabase = makeMapKernelDatabase(); - kernelKVStore = kernelDatabase.kernelKVStore; - kernelStore = makeKernelStore(kernelDatabase); + kernelStore = makeKernelStore(makeMapKernelDatabase()); logger = new Logger('test'); mockFactory = createMockRemotesFactory({ @@ -776,7 +773,7 @@ describe('RemoteManager', () => { // Set up a promise where the remote is the decider const [kpid] = kernelStore.initKernelPromise(); kernelStore.setPromiseDecider(kpid, remoteId); - kernelKVStore.set(`cle.${remoteId}.p+1`, kpid); + kernelStore.addCListEntry(remoteId, kpid, 'rp+1'); const resolvePromisesSpy = vi.spyOn(mockKernelQueue, 'resolvePromises'); @@ -882,7 +879,7 @@ describe('RemoteManager', () => { const [kpid] = kernelStore.initKernelPromise(); kernelStore.setPromiseDecider(kpid, remoteId); - kernelKVStore.set(`cle.${remoteId}.p+1`, kpid); + kernelStore.addCListEntry(remoteId, kpid, 'rp+1'); kernelStore.setPeerIncarnation(peerId, 'incarnation-A'); const resolvePromisesSpy = vi.spyOn(mockKernelQueue, 'resolvePromises'); diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index 58fefc80c3..d4e68387cf 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -47,6 +47,8 @@ describe('kernel store', () => { 'addSubcluster', 'addSubclusterVat', 'allocateErefForKref', + 'assertRefCountsIfAuditing', + 'auditRefCounts', 'bufferCrankOutput', 'cleanupOrphanMessages', 'cleanupTerminatedVat', @@ -84,6 +86,7 @@ describe('kernel store', () => { 'forgetEref', 'forgetKref', 'forgetTerminatedVat', + 'formatRefCountViolations', 'getAllRemoteRecords', 'getAllSystemSubclusterMappings', 'getAllVatRecords', @@ -140,7 +143,7 @@ describe('kernel store', () => { 'isVatTerminated', 'kernelRefExists', 'krefToEref', - 'krefsToExistingErefs', + 'krefsToErefs', 'makeVatStore', 'markInitialized', 'markVatAsTerminated', @@ -148,6 +151,7 @@ describe('kernel store', () => { 'nextTerminatedVatCleanup', 'pinObject', 'provideIncarnationId', + 'recomputeRefCounts', 'recordLastActiveTime', 'releaseAllSavepoints', 'releaseSavepoint', @@ -167,6 +171,8 @@ describe('kernel store', () => { 'setPeerIncarnation', 'setPendingMessage', 'setPromiseDecider', + 'setReachableFlag', + 'setRefCountAuditing', 'setRelayEntries', 'setRemoteHighestReceivedSeq', 'setRemoteIdentityValue', @@ -206,31 +212,31 @@ describe('kernel store', () => { const ko2Owner = 'r23'; expect(ks.initKernelObject(ko1Owner)).toBe('ko1'); - // Check that the object is initialized with reachable=1, recognizable=1 - const refCounts = ks.getObjectRefCount('ko1'); - expect(refCounts.reachable).toBe(1); - expect(refCounts.recognizable).toBe(1); + expect(ks.getObjectRefCount('ko1')).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); // Increment the reference count ks.incrementRefCount('ko1', 'test'); - expect(ks.getObjectRefCount('ko1').reachable).toBe(2); - expect(ks.getObjectRefCount('ko1').recognizable).toBe(2); + expect(ks.getObjectRefCount('ko1')).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); // Increment again ks.incrementRefCount('ko1', 'test'); - expect(ks.getObjectRefCount('ko1').reachable).toBe(3); - expect(ks.getObjectRefCount('ko1').recognizable).toBe(3); - - // Decrement - ks.decrementRefCount('ko1', 'tess'); - expect(ks.getObjectRefCount('ko1').reachable).toBe(2); - expect(ks.getObjectRefCount('ko1').recognizable).toBe(2); + expect(ks.getObjectRefCount('ko1')).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); - // Decrement twice more to reach 0 ks.decrementRefCount('ko1', 'test'); ks.decrementRefCount('ko1', 'test'); - expect(ks.getObjectRefCount('ko1').reachable).toBe(0); - expect(ks.getObjectRefCount('ko1').recognizable).toBe(0); + expect(ks.getObjectRefCount('ko1')).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); // Create another object expect(ks.initKernelObject(ko2Owner)).toBe('ko2'); diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 5c8f49fc3d..9ac1085f18 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -38,9 +38,10 @@ * ${kpid}.decider = ${endid} // who decides on settlement * ${kpid}.value = JSON(CAPDATA) // value settled to, if settled * - * C-lists - * cle.${endid}.${eref} = ${kref} // ERef->KRef mapping - * clk.${endid}.${kref} = ${eref} // KRef->ERef mapping + * C-lists (both directions share one prefix; see `getCListPrefix`) + * ${endid}.c.${eref} = ${kref} // ERef->KRef mapping + * ${endid}.c.${kref} = R|_ ${eref} // KRef->ERef mapping, plus the + * // endpoint's reachable flag * * Vat bookkeeping * e.nextObjectId.${endid} = NN // allocation counter for imported object ERefs @@ -79,6 +80,7 @@ import { getPinMethods } from './methods/pinned.ts'; import { getPromiseMethods } from './methods/promise.ts'; import { getQueueMethods } from './methods/queue.ts'; import { getReachableMethods } from './methods/reachable.ts'; +import { getRefCountAuditMethods } from './methods/refcount-audit.ts'; import { getRefCountMethods } from './methods/refcount.ts'; import { getRelayMethods } from './methods/relay.ts'; import { getRemoteMethods } from './methods/remote.ts'; @@ -152,12 +154,14 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { subclusters: provideCachedStoredValue('subclusters', '[]'), nextSubclusterId: provideCachedStoredValue('nextSubclusterId', '1'), vatToSubclusterMap: provideCachedStoredValue('vatToSubclusterMap', '{}'), + auditRefCounts: false, // Logging logger: logger?.subLogger({ tags: ['kernel-store'] }), }; const id = getIdMethods(context); const refCount = getRefCountMethods(context); + const refCountAudit = getRefCountAuditMethods(context); const object = getObjectMethods(context); const promise = getPromiseMethods(context); const revocation = getRevocationMethods(context); @@ -291,6 +295,7 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { ...id, ...queue, ...refCount, + ...refCountAudit, ...object, ...promise, ...revocation, @@ -368,3 +373,4 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { export type KernelStore = ReturnType; export type { RelayEntry } from './types.ts'; +export type { RefCountViolation } from './methods/refcount-audit.ts'; diff --git a/packages/ocap-kernel/src/store/methods/base.ts b/packages/ocap-kernel/src/store/methods/base.ts index bd09b457a5..2ccd17f1f3 100644 --- a/packages/ocap-kernel/src/store/methods/base.ts +++ b/packages/ocap-kernel/src/store/methods/base.ts @@ -19,7 +19,18 @@ export function getBaseMethods(kv: KVStore) { * @returns The key for the reachable flag and vatSlot. */ function getSlotKey(endpointId: EndpointId, ref: Ref): string { - return `${endpointId}.c.${ref}`; + return `${getCListPrefix(endpointId)}${ref}`; + } + + /** + * Get the prefix shared by both directions of every entry in an endpoint's + * c-list, for iterating over the whole thing. + * + * @param endpointId - The endpoint whose c-list is of interest. + * @returns The prefix that all of that endpoint's c-list keys begin with. + */ + function getCListPrefix(endpointId: EndpointId): string { + return `${endpointId}.c.`; } /** @@ -206,6 +217,7 @@ export function getBaseMethods(kv: KVStore) { return { getSlotKey, + getCListPrefix, refCountKey, getOwnerKey, getRevokedKey, diff --git a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts new file mode 100644 index 0000000000..49d4d384e5 --- /dev/null +++ b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import type { VatConfig, VatId } from '../../types.ts'; +import { makeKernelStore } from '../index.ts'; + +/** + * Regressions for the asymmetry described in + * https://github.com/MetaMask/ocap-kernel/issues/1006: creating an import + * c-list entry changed no refcount while tearing one down decremented both, + * and `initKernelObject` compensated by minting every object at (1, 1). That + * constant came out right for exactly one importer, which is why nothing + * noticed. + */ +describe('c-list reference accounting', () => { + let kernelStore: ReturnType; + + /** + * Register and initialize an endpoint so it can hold c-list entries. + * + * @param vatIds - The vats to bring into existence. + */ + function givenVats(...vatIds: VatId[]): void { + for (const vatId of vatIds) { + kernelStore.setVatConfig(vatId, { sourceSpec: 'x' } as VatConfig); + kernelStore.initEndpoint(vatId); + } + } + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + kernelStore.markInitialized(); + kernelStore.setRefCountAuditing(true); + givenVats('v1', 'v2', 'v3'); + }); + + it('counts each importer separately', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + + kernelStore.translateRefKtoE('v2', kref, true); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + + kernelStore.translateRefKtoE('v3', kref, true); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); + }); + + it('keeps an object alive for a second importer after the first lets go', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.translateRefKtoE('v3', kref, true); + + kernelStore.clearReachableFlag('v2', kref); + kernelStore.forgetKref('v2', kref); + kernelStore.collectGarbage(); + + // v3 still holds it, so the owner must not be told to drop or retire + expect([...kernelStore.getGCActions()]).toStrictEqual([]); + expect(kernelStore.getReachableFlag('v3', kref)).toBe(true); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('drops an object once the last of several importers lets go', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + for (const vatId of ['v2', 'v3'] as VatId[]) { + kernelStore.translateRefKtoE(vatId, kref, true); + kernelStore.clearReachableFlag(vatId, kref); + kernelStore.forgetKref(vatId, kref); + } + kernelStore.collectGarbage(); + + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v1 dropExport ${kref}`, + `v1 retireExport ${kref}`, + ]); + }); + + it('cleans up a terminated owner whose importer had already dropped', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + kernelStore.markVatAsTerminated('v1'); + + // Previously the owner's baseline decrement drove this below zero and threw + // out of the middle of the export loop, leaving the vat half-cleaned + expect(kernelStore.cleanupTerminatedVat('v1')).toStrictEqual({ + exports: 1, + imports: 0, + promises: 0, + kv: 0, + }); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('restores reachability when a dropped import is handed over again', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + const eref = kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + + expect(kernelStore.getReachableFlag('v2', kref)).toBe(false); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + + expect(kernelStore.translateRefKtoE('v2', kref, true)).toBe(eref); + expect(kernelStore.getReachableFlag('v2', kref)).toBe(true); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + + it('does not inflate the count when the same import is translated twice', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.translateRefKtoE('v2', kref, true); + + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + + it('collects an object whose only reference went splat', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.incrementRefCount(kref, 'queue|slot'); + kernelStore.decrementRefCount(kref, 'deliver|splat|slot'); + + // Previously this settled at (1,1) with no holder, forever + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + kernelStore.collectGarbage(); + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v1 dropExport ${kref}`, + `v1 retireExport ${kref}`, + ]); + expect(kernelStore.getImporters(kref)).toStrictEqual([]); + }); + + describe('cleanupTerminatedVat', () => { + it('does nothing for a vat that is not terminated', () => { + expect(kernelStore.cleanupTerminatedVat('v1')).toStrictEqual({ + exports: 0, + imports: 0, + promises: 0, + kv: 0, + }); + }); + + it('orphans exports, releases imports, and forgets the vat', () => { + const mine = kernelStore.exportFromEndpoint('v1', 'o+1'); + const theirs = kernelStore.exportFromEndpoint('v2', 'o+1'); + kernelStore.translateRefKtoE('v1', theirs, true); + kernelStore.translateRefKtoE('v3', mine, true); + kernelStore.markVatAsTerminated('v1'); + + const work = kernelStore.cleanupTerminatedVat('v1'); + + expect(work).toMatchObject({ exports: 1, imports: 1, promises: 0 }); + // v1's export is orphaned but still recognized by v3 + expect(kernelStore.getOwner(mine)).toBeUndefined(); + expect(kernelStore.getObjectRefCount(mine)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + // v1's import of v2's object is released + expect(kernelStore.getObjectRefCount(theirs)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + expect(kernelStore.hasCListEntry('v1', mine)).toBe(false); + expect(kernelStore.hasCListEntry('v1', theirs)).toBe(false); + expect(kernelStore.isVatTerminated('v1')).toBe(false); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('releases the c-list entry of a promise the vat was deciding', () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.setPromiseDecider(kpid, 'v1'); + kernelStore.translateRefKtoE('v2', kpid, true); + // The caller rejects the orphans first, which is what releases the + // unsettled-promise reference and clears the decider + expect([...kernelStore.getPromisesByDecider('v1')]).toStrictEqual([kpid]); + kernelStore.resolveKernelPromise(kpid, true, { + body: '#"gone"', + slots: [], + }); + kernelStore.markVatAsTerminated('v1'); + + const work = kernelStore.cleanupTerminatedVat('v1'); + + expect(work).toMatchObject({ exports: 0, imports: 0, promises: 1 }); + expect(kernelStore.getRefCount(kpid)).toBe(1); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('leaves a live vat that shares the object untouched', () => { + const kref = kernelStore.exportFromEndpoint('v2', 'o+1'); + kernelStore.translateRefKtoE('v1', kref, true); + kernelStore.translateRefKtoE('v3', kref, true); + kernelStore.markVatAsTerminated('v1'); + + kernelStore.cleanupTerminatedVat('v1'); + kernelStore.collectGarbage(); + + expect(kernelStore.getReachableFlag('v3', kref)).toBe(true); + expect([...kernelStore.getGCActions()]).toStrictEqual([]); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + }); + + describe('three endpoints sharing one object', () => { + it('accounts for every hand-off and release in turn', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + const importers = ['v2', 'v3'] as VatId[]; + + for (const vatId of importers) { + kernelStore.translateRefKtoE(vatId, kref, true); + } + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); + expect(kernelStore.getImporters(kref)).toStrictEqual(importers); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + + // v2 drops but still recognizes + kernelStore.clearReachableFlag('v2', kref); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 2, + }); + kernelStore.collectGarbage(); + expect([...kernelStore.getGCActions()]).toStrictEqual([]); + + // v2 retires; v3 keeps it alive + kernelStore.forgetKref('v2', kref); + kernelStore.collectGarbage(); + expect([...kernelStore.getGCActions()]).toStrictEqual([]); + expect(kernelStore.getImporters(kref)).toStrictEqual(['v3']); + + // v3 lets go too, and only now is the owner told + kernelStore.clearReachableFlag('v3', kref); + kernelStore.forgetKref('v3', kref); + kernelStore.collectGarbage(); + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v1 dropExport ${kref}`, + `v1 retireExport ${kref}`, + ]); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + }); +}); diff --git a/packages/ocap-kernel/src/store/methods/clist.test.ts b/packages/ocap-kernel/src/store/methods/clist.test.ts index e43d6e428a..4df8841bf9 100644 --- a/packages/ocap-kernel/src/store/methods/clist.test.ts +++ b/packages/ocap-kernel/src/store/methods/clist.test.ts @@ -27,39 +27,58 @@ describe('clist-methods', () => { }); describe('addCListEntry', () => { - it('adds a bidirectional mapping between KRef and ERef', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - const eref: ERef = 'o-1'; + it.each([ + { what: 'an object import', kref: 'ko1', eref: 'o-1', flag: '_' }, + { what: 'an object export', kref: 'ko1', eref: 'o+1', flag: 'R' }, + { what: 'a promise import', kref: 'kp1', eref: 'p-2', flag: '_' }, + { what: 'a promise export', kref: 'kp1', eref: 'p+2', flag: 'R' }, + ] as { what: string; kref: KRef; eref: ERef; flag: string }[])( + 'adds a bidirectional mapping for $what', + ({ kref, eref, flag }) => { + const endpointId: EndpointId = 'v1'; + + clistMethods.addCListEntry(endpointId, kref, eref); + + // Only an export is born reachable; an import earns reachability when + // the reference is actually handed over + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`${flag} ${eref}`); + expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + }, + ); + + it('works with remote endpoints', () => { + const endpointId: EndpointId = 'r1'; + const kref: KRef = 'ko2'; + const eref: ERef = 'ro+3'; clistMethods.addCListEntry(endpointId, kref, eref); - // Check that both mappings are stored expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); }); - it('works with promise refs', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'kp1'; - const eref: ERef = 'p+2'; + it('takes a recognizable reference for an object import', () => { + clistMethods.addCListEntry('v1', 'ko1', 'o-1'); - clistMethods.addCListEntry(endpointId, kref, eref); + expect(kv.get('ko1.refCount')).toBe('0,1'); + }); - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); + it('takes no reference for an object export', () => { + clistMethods.addCListEntry('v1', 'ko1', 'o+1'); + + expect(kv.get('ko1.refCount')).toBeUndefined(); }); - it('works with remote endpoints', () => { - const endpointId: EndpointId = 'r1'; - const kref: KRef = 'ko2'; - const eref: ERef = 'ro+3'; + it.each(['p-1', 'p+1'] as ERef[])( + 'takes a reference for a promise entry (%s)', + (eref) => { + kv.set('kp1.refCount', '1'); - clistMethods.addCListEntry(endpointId, kref, eref); + clistMethods.addCListEntry('v1', 'kp1', eref); - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); - expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); - }); + expect(kv.get('kp1.refCount')).toBe('2'); + }, + ); }); describe('hasCListEntry', () => { @@ -96,7 +115,7 @@ describe('clist-methods', () => { expect(kv.get(`e.nextObjectId.${endpointId}`)).toBe('2'); // Check that the mapping was added - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`_ ${eref}`); expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); }); @@ -113,7 +132,7 @@ describe('clist-methods', () => { expect(kv.get(`e.nextPromiseId.${endpointId}`)).toBe('2'); // Check that the mapping was added - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`_ ${eref}`); expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); }); @@ -151,7 +170,7 @@ describe('clist-methods', () => { }); }); - describe('krefsToExistingErefs', () => { + describe('krefsToErefs', () => { it('returns the ERefs for existing KRefs', () => { const endpointId: EndpointId = 'v1'; const kref1: KRef = 'ko1'; @@ -163,25 +182,18 @@ describe('clist-methods', () => { clistMethods.addCListEntry(endpointId, kref2, eref2); expect( - clistMethods.krefsToExistingErefs(endpointId, [kref1, kref2]), + clistMethods.krefsToErefs(endpointId, [kref1, kref2]), ).toStrictEqual([eref1, eref2]); }); - it('returns an empty array for non-existent KRefs', () => { - const endpointId: EndpointId = 'v1'; - const kref: KRef = 'ko1'; - - expect( - clistMethods.krefsToExistingErefs(endpointId, [kref]), - ).toStrictEqual([]); + it('throws for an unmapped KRef', () => { + expect(() => clistMethods.krefsToErefs('v1', ['ko1'])).toThrow( + 'unmapped kref "ko1" in "v1" c-list', + ); }); it('returns an empty array for empty KRef array', () => { - const endpointId: EndpointId = 'v1'; - - expect(clistMethods.krefsToExistingErefs(endpointId, [])).toStrictEqual( - [], - ); + expect(clistMethods.krefsToErefs('v1', [])).toStrictEqual([]); }); }); @@ -191,7 +203,7 @@ describe('clist-methods', () => { const kref: KRef = 'ko1'; const eref: ERef = 'o-1'; clistMethods.addCListEntry(endpointId, kref, eref); - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`_ ${eref}`); expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); clistMethods.forgetEref(endpointId, eref); expect(kv.get(`${endpointId}.c.${kref}`)).toBeUndefined(); @@ -214,7 +226,7 @@ describe('clist-methods', () => { const kref: KRef = 'ko1'; const eref: ERef = 'o-1'; clistMethods.addCListEntry(endpointId, kref, eref); - expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`R ${eref}`); + expect(kv.get(`${endpointId}.c.${kref}`)).toBe(`_ ${eref}`); expect(kv.get(`${endpointId}.c.${eref}`)).toBe(kref); clistMethods.forgetKref(endpointId, kref); expect(kv.get(`${endpointId}.c.${kref}`)).toBeUndefined(); diff --git a/packages/ocap-kernel/src/store/methods/clist.ts b/packages/ocap-kernel/src/store/methods/clist.ts index 425f4a1295..d079100ae4 100644 --- a/packages/ocap-kernel/src/store/methods/clist.ts +++ b/packages/ocap-kernel/src/store/methods/clist.ts @@ -1,3 +1,5 @@ +import { Fail } from '@endo/errors'; + import { getBaseMethods } from './base.ts'; import { getReachableMethods } from './reachable.ts'; import { getRefCountMethods } from './refcount.ts'; @@ -20,23 +22,31 @@ import { export function getCListMethods(ctx: StoreContext) { const { getSlotKey } = getBaseMethods(ctx.kv); const { clearReachableFlag } = getReachableMethods(ctx); - const { decrementRefCount } = getRefCountMethods(ctx); + const { incrementRefCount, decrementRefCount } = getRefCountMethods(ctx); /** * Add an entry to an endpoint's c-list, creating a new bidirectional mapping * between an ERef belonging to the endpoint and a KRef belonging to the * kernel. * + * The entry is itself a reference, so creating one takes a count, mirroring + * {@link deleteCListEntry}. An import is born recognizing but not reaching: + * reachability is `setReachableFlag`'s job, when the reference is handed + * over. An export takes no count for an object — the owner is not one of its + * own referrers — and is born flagged. + * * @param endpointId - The endpoint whose c-list is to be added to. * @param kref - The KRef. * @param eref - The ERef. */ function addCListEntry(endpointId: EndpointId, kref: KRef, eref: ERef): void { + const isExport = parseRef(eref).direction === 'export'; ctx.kv.set( getSlotKey(endpointId, kref), - buildReachableAndVatSlot(true, eref), + buildReachableAndVatSlot(isExport, eref), ); ctx.kv.set(getSlotKey(endpointId, eref), kref); + incrementRefCount(kref, 'add|kref', { isExport, onlyRecognizable: true }); } /** @@ -133,16 +143,24 @@ export function getCListMethods(ctx: StoreContext) { } /** - * Look up the ERefs that an endpoint's c-list maps aa list of KRefs to. + * Look up the ERefs that an endpoint's c-list maps a list of KRefs to, + * without allocating entries or disturbing reachability. + * + * Every kref must already be mapped. Garbage collection is the only caller + * and has already established that each kref has an entry, so a missing one + * means the two disagree — worth hearing about rather than silently dropping + * the notification. * * @param endpointId - The endpoint in question. * @param krefs - The KRefs to look up. - * @returns The given endpoint's ERefs corresponding to `krefs` + * @returns The given endpoint's ERefs corresponding to `krefs`. */ - function krefsToExistingErefs(endpointId: EndpointId, krefs: KRef[]): ERef[] { - return krefs - .map((kref) => krefToEref(endpointId, kref)) - .filter((eref): eref is ERef => Boolean(eref)); + function krefsToErefs(endpointId: EndpointId, krefs: KRef[]): ERef[] { + return krefs.map( + (kref) => + krefToEref(endpointId, kref) ?? + Fail`unmapped kref ${kref} in ${endpointId} c-list`, + ); } /** @@ -182,6 +200,6 @@ export function getCListMethods(ctx: StoreContext) { krefToEref, forgetEref, forgetKref, - krefsToExistingErefs, + krefsToErefs, }; } diff --git a/packages/ocap-kernel/src/store/methods/gc.test.ts b/packages/ocap-kernel/src/store/methods/gc.test.ts index a294920e9a..e91fb29bb9 100644 --- a/packages/ocap-kernel/src/store/methods/gc.test.ts +++ b/packages/ocap-kernel/src/store/methods/gc.test.ts @@ -76,21 +76,6 @@ describe('GC methods', () => { }); }); - describe('reachability tracking', () => { - it('manages reachable flags', () => { - const v1Object = kernelStore.initKernelObject('v1'); - kernelStore.addCListEntry('v1', v1Object, 'o-1'); - - expect(kernelStore.getReachableFlag('v1', v1Object)).toBe(true); - - kernelStore.clearReachableFlag('v1', v1Object); - expect(kernelStore.getReachableFlag('v1', v1Object)).toBe(false); - - const refCounts = kernelStore.getObjectRefCount(v1Object); - expect(refCounts.reachable).toBe(0); - }); - }); - describe('reaping', () => { it('processes reap queue in order', () => { const vatIds = ['v1', 'v2', 'v3']; diff --git a/packages/ocap-kernel/src/store/methods/gc.ts b/packages/ocap-kernel/src/store/methods/gc.ts index 04f3c0cc0c..31b21294c8 100644 --- a/packages/ocap-kernel/src/store/methods/gc.ts +++ b/packages/ocap-kernel/src/store/methods/gc.ts @@ -165,8 +165,10 @@ export function getGCMethods(ctx: StoreContext) { actions.add(makeGCAction(ownerVatID, 'dropExport', kref)); } if (recognizable === 0) { - // TODO: rethink this assert - // assert.equal(vatConsidersReachable, false, `${kref} is reachable but not recognizable`); + // No assertion that the owner has stopped considering this + // reachable: when the last holder both drops and retires before + // we run, we queue dropExport and retireExport together and the + // owner's flag is still set until the first of them is delivered. actions.add(makeGCAction(ownerVatID, 'retireExport', kref)); } } else if (ownerVatID && terminated) { diff --git a/packages/ocap-kernel/src/store/methods/object.test.ts b/packages/ocap-kernel/src/store/methods/object.test.ts index def40adaef..26d4fb9b16 100644 --- a/packages/ocap-kernel/src/store/methods/object.test.ts +++ b/packages/ocap-kernel/src/store/methods/object.test.ts @@ -29,7 +29,7 @@ describe('object-methods', () => { }); describe('initKernelObject', () => { - it('creates a new kernel object with initial reference counts', () => { + it('creates a new kernel object, unreferenced', () => { const owner: EndpointId = 'v1'; const koId = objectStore.initKernelObject(owner); @@ -39,13 +39,13 @@ describe('object-methods', () => { // Check the owner is set correctly expect(kv.get(`${koId}.owner`)).toBe(owner); - // Check reference counts are initialized to 1,1 - expect(kv.get(`${koId}.refCount`)).toBe('1,1'); - - // Check via the API - const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts.reachable).toBe(1); - expect(refCounts.recognizable).toBe(1); + // A new object has no referrers yet; the owner's own export entry is + // not one of them + expect(kv.get(`${koId}.refCount`)).toBe('0,0'); + expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); }); it('initializes the revoked flag to false', () => { @@ -171,8 +171,10 @@ describe('object-methods', () => { it('returns reference counts for existing objects', () => { const koId = objectStore.initKernelObject('v1'); + objectStore.setObjectRefCount(koId, { reachable: 1, recognizable: 2 }); + const refCounts = objectStore.getObjectRefCount(koId); - expect(refCounts).toStrictEqual({ reachable: 1, recognizable: 1 }); + expect(refCounts).toStrictEqual({ reachable: 1, recognizable: 2 }); }); it('returns zero counts for non-existent objects', () => { @@ -276,8 +278,8 @@ describe('object-methods', () => { // Check initial state expect(objectStore.getOwner(koId)).toBe('v1'); expect(objectStore.getObjectRefCount(koId)).toStrictEqual({ - reachable: 1, - recognizable: 1, + reachable: 0, + recognizable: 0, }); // Update reference counts diff --git a/packages/ocap-kernel/src/store/methods/object.ts b/packages/ocap-kernel/src/store/methods/object.ts index 9e596f5e8f..57b7aaf88e 100644 --- a/packages/ocap-kernel/src/store/methods/object.ts +++ b/packages/ocap-kernel/src/store/methods/object.ts @@ -19,10 +19,11 @@ export function getObjectMethods(ctx: StoreContext) { getBaseMethods(ctx.kv); /** - * Create a new kernel object. The new object will be born with reference and - * recognizability counts of 1, on the assumption that the new object - * corresponds to an object that has just been imported from somewhere. The - * object is initially unrevoked. + * Create a new kernel object, born unreferenced at `(0, 0)`. Every unit of + * an object's counts is owed to a reference someone else holds — an + * importer's c-list entry, a queued message, a promise's resolution value, a + * pin — and the owner's own export entry is not one of them. The object is + * initially unrevoked. * * @param owner - The endpoint or 'kernel' that is the owner of the new object. * @returns The new object's KRef. @@ -30,7 +31,7 @@ export function getObjectMethods(ctx: StoreContext) { function initKernelObject(owner: EndpointId | 'kernel'): KRef { const koId = getNextObjectId(); ctx.kv.set(getOwnerKey(koId), owner); - setObjectRefCount(koId, { reachable: 1, recognizable: 1 }); + setObjectRefCount(koId, { reachable: 0, recognizable: 0 }); return koId; } diff --git a/packages/ocap-kernel/src/store/methods/promise.test.ts b/packages/ocap-kernel/src/store/methods/promise.test.ts index baeba6c2e5..d85e291ef5 100644 --- a/packages/ocap-kernel/src/store/methods/promise.test.ts +++ b/packages/ocap-kernel/src/store/methods/promise.test.ts @@ -58,6 +58,7 @@ describe('promise store methods', () => { }; let context: StoreContext; let promiseMethods: ReturnType; + const mockIncrementRefCount = vi.fn(); const mockDecrementRefCount = vi.fn(); beforeEach(() => { @@ -78,6 +79,7 @@ describe('promise store methods', () => { incCounter: mockIncCounter, provideStoredQueue: mockProvideStoredQueue, getPrefixedKeys: mockGetPrefixedKeys, + getCListPrefix: (endpointId: string) => `${endpointId}.c.`, }); (getQueueMethods as ReturnType).mockReturnValue({ @@ -85,6 +87,7 @@ describe('promise store methods', () => { }); (getRefCountMethods as ReturnType).mockReturnValue({ + incrementRefCount: mockIncrementRefCount, decrementRefCount: mockDecrementRefCount, }); @@ -304,11 +307,12 @@ describe('promise store methods', () => { slots: ['o+1', 'o+2'], }; const message1: KernelMessage = { - method: 'method1', - } as unknown as KernelMessage; + methargs: { body: 'method1', slots: ['ko7'] }, + result: 'kp8', + }; const message2: KernelMessage = { - method: 'method2', - } as unknown as KernelMessage; + methargs: { body: 'method2', slots: [] }, + }; mockKV.set(`${kpid}.state`, 'unresolved'); mockKV.set(`${kpid}.decider`, 'v1'); @@ -338,7 +342,15 @@ describe('promise store methods', () => { expect(mockKV.has(`${kpid}.decider`)).toBe(false); expect(mockKV.has(`${kpid}.subscribers`)).toBe(false); expect(mockQueue.delete).toHaveBeenCalled(); - expect(mockDecrementRefCount).toHaveBeenCalledTimes(1); + // Each dequeued message releases what its queue entry held, then the + // promise releases the decision it was owed + expect(mockDecrementRefCount.mock.calls).toStrictEqual([ + [kpid, 'resolve|dequeue|target'], + ['kp8', 'resolve|dequeue|result'], + ['ko7', 'resolve|dequeue|slot'], + [kpid, 'resolve|dequeue|target'], + [kpid, 'resolve|decider'], + ]); }); it('rejects a promise and enqueues pending messages', () => { @@ -372,6 +384,23 @@ describe('promise store methods', () => { expect(mockProvideStoredQueue).toHaveBeenCalledWith(kpid, false); expect(mockQueue.enqueue).toHaveBeenCalledWith(message); }); + + it('takes a reference on everything the queued message carries', () => { + const kpid = 'kp123'; + const message: KernelMessage = { + methargs: { body: 'test', slots: ['ko1', 'kp2'] }, + result: 'kp3', + }; + + promiseMethods.enqueuePromiseMessage(kpid, message); + + expect(mockIncrementRefCount.mock.calls).toStrictEqual([ + [kpid, 'promiseQueue|target'], + ['kp3', 'promiseQueue|result'], + ['ko1', 'promiseQueue|slot'], + ['kp2', 'promiseQueue|slot'], + ]); + }); }); describe('getKernelPromiseMessageQueue', () => { @@ -410,69 +439,71 @@ describe('promise store methods', () => { }); describe('getPromisesByDecider', () => { - it('yields promises decided by a specific vat', () => { - const vatId = 'v1' as VatId; - const kpid1 = 'kp101'; - const kpid2 = 'kp102'; - const kpid3 = 'kp103'; - - // Set up mock data - mockGetPrefixedKeys.mockReturnValue([ - `cle.${vatId}.p1`, - `cle.${vatId}.p2`, - `cle.${vatId}.p3`, - ]); - - mockKV.set(`cle.${vatId}.p1`, kpid1); - mockKV.set(`cle.${vatId}.p2`, kpid2); - mockKV.set(`cle.${vatId}.p3`, kpid3); - - // kpid1 is decided by vatId - mockKV.set(`${kpid1}.state`, 'unresolved'); - mockKV.set(`${kpid1}.decider`, vatId); - mockKV.set(`${kpid1}.subscribers`, '[]'); - - // kpid2 is also decided by vatId - mockKV.set(`${kpid2}.state`, 'unresolved'); - mockKV.set(`${kpid2}.decider`, vatId); - mockKV.set(`${kpid2}.subscribers`, '[]'); + /** + * Populate a c-list and an unresolved promise record, using the real key + * layout so the scan is exercised rather than mocked around. + * + * @param endpointId - The endpoint whose c-list to add to. + * @param eref - The endpoint's ref for the promise. + * @param kpid - The kernel promise. + * @param decider - The promise's decider, if it has one. + * @param state - The promise's state. + */ + function givenCListPromise( + endpointId: string, + eref: string, + kpid: string, + decider: string | undefined, + state = 'unresolved', + ): void { + mockKV.set(`${endpointId}.c.${eref}`, kpid); + mockKV.set(`${endpointId}.c.${kpid}`, `R ${eref}`); + mockKV.set(`${kpid}.state`, state); + mockKV.set(`${kpid}.subscribers`, '[]'); + if (state === 'unresolved') { + if (decider) { + mockKV.set(`${kpid}.decider`, decider); + } + } else { + mockKV.set(`${kpid}.value`, '{"body":"value","slots":[]}'); + } + mockGetPrefixedKeys.mockImplementation((prefix: string) => + [...mockKV.keys()].filter((key) => key.startsWith(prefix)).sort(), + ); + } - // kpid3 is unresolved but decided by a different vat - mockKV.set(`${kpid3}.state`, 'unresolved'); - mockKV.set(`${kpid3}.decider`, 'v2'); - mockKV.set(`${kpid3}.subscribers`, '[]'); + it.each([ + { context: 'a vat', endpointId: 'v1', erefs: ['p+1', 'p-2'] }, + { context: 'a remote', endpointId: 'r1', erefs: ['rp+1', 'rp-2'] }, + ])('yields promises decided by $context', ({ endpointId, erefs }) => { + givenCListPromise(endpointId, erefs[0] as string, 'kp101', endpointId); + givenCListPromise(endpointId, erefs[1] as string, 'kp102', endpointId); + givenCListPromise(endpointId, 'p+3', 'kp103', 'v2'); - const result = Array.from(promiseMethods.getPromisesByDecider(vatId)); + const result = Array.from( + promiseMethods.getPromisesByDecider(endpointId as VatId), + ); - expect(result).toStrictEqual([kpid1, kpid2]); - expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`cle.${vatId}.p`); + expect(result).toStrictEqual(['kp101', 'kp102']); }); it('does not yield resolved promises', () => { - const vatId = 'v1' as VatId; - const kpid1 = 'kp101'; - const kpid2 = 'kp102'; + givenCListPromise('v1', 'p+1', 'kp101', undefined, 'fulfilled'); + givenCListPromise('v1', 'p+2', 'kp102', 'v1'); - mockGetPrefixedKeys.mockReturnValue([ - `cle.${vatId}.p1`, - `cle.${vatId}.p2`, - ]); + const result = Array.from(promiseMethods.getPromisesByDecider('v1')); - mockKV.set(`cle.${vatId}.p1`, kpid1); - mockKV.set(`cle.${vatId}.p2`, kpid2); - - // kpid1 is fulfilled - mockKV.set(`${kpid1}.state`, 'fulfilled'); - mockKV.set(`${kpid1}.value`, '{"body":"value","slots":[]}'); + expect(result).toStrictEqual(['kp102']); + }); - // kpid2 is unresolved and decided by vatId - mockKV.set(`${kpid2}.state`, 'unresolved'); - mockKV.set(`${kpid2}.decider`, vatId); - mockKV.set(`${kpid2}.subscribers`, '[]'); + it('ignores object entries in the same c-list', () => { + givenCListPromise('v1', 'p+1', 'kp101', 'v1'); + mockKV.set('v1.c.o+1', 'ko1'); + mockKV.set('v1.c.ko1', 'R o+1'); - const result = Array.from(promiseMethods.getPromisesByDecider(vatId)); + const result = Array.from(promiseMethods.getPromisesByDecider('v1')); - expect(result).toStrictEqual([kpid2]); + expect(result).toStrictEqual(['kp101']); }); it('yields nothing if no promises are decided by the vat', () => { diff --git a/packages/ocap-kernel/src/store/methods/promise.ts b/packages/ocap-kernel/src/store/methods/promise.ts index c3aedbf6c3..40fa73b049 100644 --- a/packages/ocap-kernel/src/store/methods/promise.ts +++ b/packages/ocap-kernel/src/store/methods/promise.ts @@ -16,6 +16,9 @@ import { makeKernelSlot } from '../utils/kernel-slots.ts'; import { parseRef } from '../utils/parse-ref.ts'; import { isPromiseRef } from '../utils/promise-ref.ts'; +/** Matches the promise erefs in a c-list: `p+NN`/`p-NN`, or `rp+NN`/`rp-NN` for a remote. */ +const PROMISE_EREF = /^r?p[-+]\d+$/u; + /** * Create a promise store object that provides functionality for managing kernel promises. * @@ -25,14 +28,20 @@ import { isPromiseRef } from '../utils/promise-ref.ts'; */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getPromiseMethods(ctx: StoreContext) { - const { incCounter, provideStoredQueue, getPrefixedKeys, refCountKey } = - getBaseMethods(ctx.kv); - const { decrementRefCount } = getRefCountMethods(ctx); + const { + incCounter, + provideStoredQueue, + getPrefixedKeys, + getCListPrefix, + refCountKey, + } = getBaseMethods(ctx.kv); + const { incrementRefCount, decrementRefCount } = getRefCountMethods(ctx); /** - * Create a new, unresolved kernel promise. The new promise will be born with - * a reference count of 1 on the assumption that the promise has just been - * imported from somewhere. + * Create a new, unresolved kernel promise, born with a reference count of 1: + * an unsettled promise is owed a decision, and that obligation is itself a + * reference. Released, exactly once, when the promise settles in + * {@link resolveKernelPromise}. * * @returns A tuple of the new promise's KRef and an object describing the * new promise itself. @@ -159,16 +168,18 @@ export function getPromiseMethods(ctx: StoreContext) { value: CapData, ): [KRef, KernelMessage][] { const queue = provideStoredQueue(kpid, false); - // Collect messages that were queued on this promise + // Releasing each queue entry's references as we go: the caller re-enqueues + // these on the run queue, which takes its own. const queuedMessages: [KRef, KernelMessage][] = []; for (const message of getKernelPromiseMessageQueue(kpid)) { queuedMessages.push([kpid, message]); + releaseQueuedMessageRefs(kpid, message, 'resolve|dequeue'); } ctx.kv.set(`${kpid}.state`, rejected ? 'rejected' : 'fulfilled'); ctx.kv.set(`${kpid}.value`, JSON.stringify(value)); ctx.kv.delete(`${kpid}.decider`); ctx.kv.delete(`${kpid}.subscribers`); - // Drop the baseline "decider" refcount now that the promise is settled. + // The promise has been decided, so it is no longer owed a decision. decrementRefCount(kpid, 'resolve|decider'); queue.delete(); return queuedMessages; @@ -177,13 +188,46 @@ export function getPromiseMethods(ctx: StoreContext) { /** * Append a message to a promise's message queue. * + * The queue entry becomes the message's holder, so it takes references on + * everything the message carries, just as the run queue does. + * * @param kpid - The KRef of the promise to enqueue on. * @param message - The message to enqueue. */ function enqueuePromiseMessage(kpid: KRef, message: KernelMessage): void { + incrementRefCount(kpid, 'promiseQueue|target'); + if (message.result) { + incrementRefCount(message.result, 'promiseQueue|result'); + } + for (const slot of message.methargs.slots) { + incrementRefCount(slot, 'promiseQueue|slot'); + } provideStoredQueue(kpid, false).enqueue(message); } + /** + * Release the references a promise-queue entry held on the message it + * carried. + * + * @param kpid - The promise whose queue the message was on, and hence the + * message's target. + * @param message - The message being taken off the queue. + * @param tag - Tag for refcount logging. + */ + function releaseQueuedMessageRefs( + kpid: KRef, + message: KernelMessage, + tag: string, + ): void { + decrementRefCount(kpid, `${tag}|target`); + if (message.result) { + decrementRefCount(message.result, `${tag}|result`); + } + for (const slot of message.methargs.slots) { + decrementRefCount(slot, `${tag}|slot`); + } + } + /** * Fetch the messages in a kernel promise's message queue. * @@ -211,8 +255,13 @@ export function getPromiseMethods(ctx: StoreContext) { * @yields the kpids of all the unresolved promises decided by `decider`. */ function* getPromisesByDecider(decider: EndpointId): Generator { - const basePrefix = `cle.${decider}.`; - for (const key of getPrefixedKeys(`${basePrefix}p`)) { + const prefix = getCListPrefix(decider); + for (const key of getPrefixedKeys(prefix)) { + // A c-list holds both directions of each pair. Iterate by eref, and only + // the promise ones: `p+NN`/`p-NN` for a vat, `rp+NN`/`rp-NN` for a remote. + if (!PROMISE_EREF.test(key.slice(prefix.length))) { + continue; + } const kpid = ctx.kv.getRequired(key); const kp = getKernelPromise(kpid); if (kp.state === 'unresolved' && kp.decider === decider) { diff --git a/packages/ocap-kernel/src/store/methods/reachable.test.ts b/packages/ocap-kernel/src/store/methods/reachable.test.ts index fdaab477e5..1ec32bdb9f 100644 --- a/packages/ocap-kernel/src/store/methods/reachable.test.ts +++ b/packages/ocap-kernel/src/store/methods/reachable.test.ts @@ -15,13 +15,65 @@ describe('GC methods', () => { const ko1 = kernelStore.initKernelObject('v1'); kernelStore.addCListEntry('v1', ko1, 'o-1'); + // An import entry is born recognizing but not yet reaching + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + + kernelStore.setReachableFlag('v1', ko1); expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); kernelStore.clearReachableFlag('v1', ko1); expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + }); + + it.each(['setReachableFlag', 'clearReachableFlag'] as const)( + 'is idempotent: %s', + (method) => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addCListEntry('v1', ko1, 'o-1'); + kernelStore.setReachableFlag('v1', ko1); + + const before = kernelStore.getObjectRefCount(ko1); + kernelStore[method]('v1', ko1); + kernelStore[method]('v1', ko1); + const after = kernelStore.getObjectRefCount(ko1); + + expect(after).toStrictEqual( + method === 'setReachableFlag' + ? before + : { reachable: 0, recognizable: 1 }, + ); + }, + ); - const refCounts = kernelStore.getObjectRefCount(ko1); - expect(refCounts.reachable).toBe(0); + it('leaves an export entry alone: it carries no reachable count', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addCListEntry('v1', ko1, 'o+1'); + + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); + kernelStore.setReachableFlag('v1', ko1); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + + kernelStore.clearReachableFlag('v1', ko1); + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); }); }); }); diff --git a/packages/ocap-kernel/src/store/methods/reachable.ts b/packages/ocap-kernel/src/store/methods/reachable.ts index 4caa2d0960..05e4c5f986 100644 --- a/packages/ocap-kernel/src/store/methods/reachable.ts +++ b/packages/ocap-kernel/src/store/methods/reachable.ts @@ -54,6 +54,32 @@ export function getReachableMethods(ctx: StoreContext) { return parseReachableAndVatSlot(data); } + /** + * Set the reachable flag for a given endpoint and kref. + * + * The counterpart to {@link clearReachableFlag}: this is how an object + * regains reachability when a vat that dropped it is handed it again. + * Idempotent, so repeated translations don't inflate the count. + * + * @param endpointId - The endpoint for which the reachable flag is being set. + * @param kref - The kref. + */ + function setReachableFlag(endpointId: EndpointId, kref: KRef): void { + const key = getSlotKey(endpointId, kref); + const { isReachable, vatSlot } = getReachableAndVatSlot(endpointId, kref); + if (isReachable) { + return; + } + ctx.kv.set(key, buildReachableAndVatSlot(true, vatSlot)); + const { direction, isPromise } = parseRef(vatSlot); + // increment 'reachable' part of refcount, but only for object imports + if (!isPromise && direction === 'import' && kernelRefExists(kref)) { + const counts = getObjectRefCount(kref); + counts.reachable += 1; + setObjectRefCount(kref, counts); + } + } + /** * Clear the reachable flag for a given endpoint and kref. * @@ -84,6 +110,7 @@ export function getReachableMethods(ctx: StoreContext) { return { getReachableFlag, getReachableAndVatSlot, + setReachableFlag, clearReachableFlag, }; } diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts new file mode 100644 index 0000000000..70af7d4075 --- /dev/null +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import type { KRef, VatConfig, VatId } from '../../types.ts'; +import { makeKernelStore } from '../index.ts'; + +describe('reference count audit', () => { + let kernelStore: ReturnType; + + /** + * Register and initialize an endpoint so it can hold c-list entries. + * + * @param vatIds - The vats to bring into existence. + */ + function givenVats(...vatIds: VatId[]): void { + for (const vatId of vatIds) { + kernelStore.setVatConfig(vatId, { sourceSpec: 'x' } as VatConfig); + kernelStore.initEndpoint(vatId); + } + } + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + kernelStore.markInitialized(); + givenVats('v1', 'v2', 'v3'); + }); + + describe('auditRefCounts', () => { + it('finds nothing wrong in an empty store', () => { + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it.each([ + { + what: 'an export', + act: (kref: KRef) => kref, + }, + { + what: 'an export plus one importer', + act: (kref: KRef) => { + kernelStore.translateRefKtoE('v2', kref, true); + return kref; + }, + }, + { + what: 'an export plus two importers', + act: (kref: KRef) => { + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.translateRefKtoE('v3', kref, true); + return kref; + }, + }, + { + what: 'a dropped import', + act: (kref: KRef) => { + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + return kref; + }, + }, + { + what: 'a retired import', + act: (kref: KRef) => { + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + kernelStore.forgetKref('v2', kref); + return kref; + }, + }, + { + what: 'a pin', + act: (kref: KRef) => { + kernelStore.pinObject(kref); + return kref; + }, + }, + { + what: 'a queued message', + act: (kref: KRef) => { + kernelStore.enqueueRun({ + type: 'send', + target: kref, + message: { methargs: { body: '#[]', slots: [kref] }, result: null }, + }); + kernelStore.incrementRefCount(kref, 'queue|target'); + kernelStore.incrementRefCount(kref, 'queue|slot'); + return kref; + }, + }, + ])('holds for $what', ({ act }) => { + act(kernelStore.exportFromEndpoint('v1', 'o+1')); + + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('holds for an unsettled promise with importers', () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.translateRefKtoE('v2', kpid, true); + + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + expect(kernelStore.getRefCount(kpid)).toBe(3); + }); + + it('holds for a settled promise whose value carries a slot', () => { + const koid = kernelStore.exportFromEndpoint('v1', 'o+1'); + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.incrementRefCount(koid, 'resolve|slot'); + kernelStore.resolveKernelPromise(kpid, false, { + body: '#"$0"', + slots: [koid], + }); + + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('reports counts that are too low', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.setObjectRefCount(kref, { reachable: 0, recognizable: 0 }); + + expect(kernelStore.auditRefCounts()).toStrictEqual([ + { + kref, + stored: '0,0', + expected: '1,1', + holders: ['v2 c-list import o-1'], + }, + ]); + }); + + it('reports counts that are too high even though nothing underflowed', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); + + expect(kernelStore.auditRefCounts()).toStrictEqual([ + { kref, stored: '1,1', expected: '0,0', holders: [] }, + ]); + }); + + it('reports a reference to a kref that has been deleted', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.deleteKernelObject(kref); + + expect(kernelStore.auditRefCounts()).toStrictEqual([ + { + kref, + stored: '(deleted)', + expected: '1,1', + holders: ['v2 c-list import o-1'], + }, + ]); + }); + + it('does not mistake an owner for a referrer', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + }); + + describe('assertRefCountsIfAuditing', () => { + it('does nothing while auditing is off', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setObjectRefCount(kref, { reachable: 9, recognizable: 9 }); + + expect(() => kernelStore.assertRefCountsIfAuditing()).not.toThrow(); + }); + + it('throws with the offending krefs once auditing is on', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setObjectRefCount(kref, { reachable: 9, recognizable: 9 }); + kernelStore.setRefCountAuditing(true); + + expect(() => kernelStore.assertRefCountsIfAuditing()).toThrow( + `${kref}: stored 9,9, expected 0,0`, + ); + }); + + it('stays quiet when the counts agree', () => { + kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setRefCountAuditing(true); + + expect(() => kernelStore.assertRefCountsIfAuditing()).not.toThrow(); + }); + }); + + describe('recomputeRefCounts', () => { + it('rebuilds counts written under the old accounting', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.translateRefKtoE('v3', kref, true); + // As the pre-fix kernel would have left it: born (1,1), with neither + // importer's c-list entry taking a reference. Two importers is the + // smallest topology where that disagrees with the truth — with one, the + // phantom baseline happens to come out to the right number. + kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); + + const { corrected, unfixable } = kernelStore.recomputeRefCounts(); + + expect(corrected).toStrictEqual([ + { + kref, + stored: '1,1', + expected: '2,2', + holders: ['v2 c-list import o-1', 'v3 c-list import o-1'], + }, + ]); + expect(unfixable).toStrictEqual([]); + expect(kernelStore.getObjectRefCount(kref)).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('reports references it cannot repair', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.deleteKernelObject(kref); + + const { corrected, unfixable } = kernelStore.recomputeRefCounts(); + + expect(corrected).toStrictEqual([]); + expect(unfixable).toHaveLength(1); + expect(unfixable[0]?.kref).toBe(kref); + }); + + it('queues krefs it zeroes for collection', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); + + kernelStore.recomputeRefCounts(); + kernelStore.collectGarbage(); + + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v1 dropExport ${kref}`, + `v1 retireExport ${kref}`, + ]); + }); + }); + + describe('formatRefCountViolations', () => { + it('names the holders behind a mismatch', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.setObjectRefCount(kref, { reachable: 0, recognizable: 0 }); + + expect( + kernelStore.formatRefCountViolations(kernelStore.auditRefCounts()), + ).toBe( + `${kref}: stored 0,0, expected 1,1 (held by: v2 c-list import o-1)`, + ); + }); + + it('says so when a mismatch has no holders at all', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); + + expect( + kernelStore.formatRefCountViolations(kernelStore.auditRefCounts()), + ).toBe(`${kref}: stored 1,1, expected 0,0 (held by: nothing)`); + }); + }); +}); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts new file mode 100644 index 0000000000..c43fd904de --- /dev/null +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -0,0 +1,352 @@ +import type { CapData } from '@endo/marshal'; + +import { getBaseMethods } from './base.ts'; +import { getObjectMethods } from './object.ts'; +import { getPinMethods } from './pinned.ts'; +import type { KRef, KernelMessage, RunQueueItem } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; +import { parseRef } from '../utils/parse-ref.ts'; +import { isPromiseRef } from '../utils/promise-ref.ts'; +import { parseReachableAndVatSlot } from '../utils/reachable.ts'; + +/** + * A kref whose stored reference counts disagree with the counts implied by the + * references the kernel can actually be seen to hold. + */ +export type RefCountViolation = { + kref: KRef; + /** + * The counts as stored, in the store's own encoding: `"reachable,recognizable"` + * for objects, a single number for promises, or `"(deleted)"` if the kref has + * no refcount entry at all. + */ + stored: string; + /** The counts implied by `holders`, in the same encoding as `stored`. */ + expected: string; + /** One entry per reference found, so a mismatch can be traced to its source. */ + holders: string[]; +}; + +/** + * The running total of references found for one kref. For a promise, which has + * only a single count, that count accumulates in `reachable`. + */ +type Tally = { + reachable: number; + recognizable: number; + holders: string[]; +}; + +/** Matches the kref-keyed half of a c-list entry, e.g. `v1.c.ko3`. */ +const CLIST_KREF_KEY = /^([vr]\d+)\.c\.(k[op]\d+)$/u; + +/** Matches a queue entry (but not the queue's `head`/`tail` bookkeeping). */ +const QUEUE_ENTRY_KEY = /^queue\.([^.]+)\.(\d+)$/u; + +/** Matches the state record that exists for every live kernel promise. */ +const PROMISE_STATE_KEY = /^(kp\d+)\.state$/u; + +/** Matches the refcount record that exists for every live kernel object or promise. */ +const REFCOUNT_KEY = /^(k[op]\d+)\.refCount$/u; + +/** + * Get the methods that audit reference counts against ground truth. + * + * The kernel's reference counts are a cache: every unit of every count is owed + * to some reference the kernel is holding somewhere else in the store — a + * c-list entry, a queued message, a promise's resolution value, a pin. This + * module recomputes those counts from the references themselves and reports + * where the cache has drifted, in either direction. Counts that are too low + * let a live capability be collected; counts that are too high leak it. + * + * @param ctx - The store context. + * @returns The reference count audit methods. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function getRefCountAuditMethods(ctx: StoreContext) { + const { getPrefixedKeys, refCountKey } = getBaseMethods(ctx.kv); + const { getObjectRefCount } = getObjectMethods(ctx); + const { getPinnedObjects } = getPinMethods(ctx); + + /** + * Render a tally the way the store encodes it, so expected and stored values + * can be compared and reported as like for like. + * + * @param kref - The kref the counts belong to. + * @param counts - The counts to render. + * @param counts.reachable - The reachable count (the only count, for a promise). + * @param counts.recognizable - The recognizable count (ignored for a promise). + * @returns The encoded counts. + */ + function renderCounts( + kref: KRef, + counts: { reachable: number; recognizable: number }, + ): string { + return isPromiseRef(kref) + ? `${counts.reachable}` + : `${counts.reachable},${counts.recognizable}`; + } + + /** + * Walk the whole store and total up, for every kref, the references the + * kernel is holding to it. + * + * The credits below mirror `incrementRefCount` case for case; when that + * function's rules change, these have to change with it. + * + * @returns A tally per kref that anything refers to. + */ + function computeExpectedRefCounts(): Map { + const tallies = new Map(); + + const credit = ( + kref: KRef, + holder: string, + { onlyRecognizable = false }: { onlyRecognizable?: boolean } = {}, + ): void => { + let tally = tallies.get(kref); + if (!tally) { + tally = { reachable: 0, recognizable: 0, holders: [] }; + tallies.set(kref, tally); + } + tally.holders.push(holder); + if (isPromiseRef(kref)) { + // Promises have a single count and no reachable/recognizable split. + tally.reachable += 1; + return; + } + if (!onlyRecognizable) { + tally.reachable += 1; + } + tally.recognizable += 1; + }; + + /** + * A queued message holds its result promise and every slot it carries. + * + * @param message - The queued message. + * @param holder - Description of the queue entry holding it. + */ + const creditMessage = (message: KernelMessage, holder: string): void => { + if (message.result) { + credit(message.result, `${holder} result`); + } + for (const slot of message.methargs.slots) { + credit(slot, `${holder} slot`); + } + }; + + for (const key of getPrefixedKeys('')) { + const clistMatch = CLIST_KREF_KEY.exec(key); + if (clistMatch) { + const [, endpointId, kref] = clistMatch as unknown as [ + string, + string, + KRef, + ]; + const { isReachable, vatSlot } = parseReachableAndVatSlot( + ctx.kv.getRequired(key), + ); + const { direction } = parseRef(vatSlot); + const holder = `${endpointId} c-list ${direction} ${vatSlot}`; + if (isPromiseRef(kref)) { + // Both directions count for a promise. + credit(kref, holder); + } else if (direction === 'import') { + // An object export is the owner's own entry and carries no count; + // an object import always recognizes and, while flagged, reaches. + credit(kref, holder, { onlyRecognizable: !isReachable }); + } + continue; + } + + const queueMatch = QUEUE_ENTRY_KEY.exec(key); + if (queueMatch) { + const [, queueName, seq] = queueMatch as unknown as [ + string, + string, + string, + ]; + const entry = ctx.kv.getRequired(key); + if (queueName === 'run') { + const item = JSON.parse(entry) as RunQueueItem; + if (item.type === 'send') { + credit(item.target, `run queue #${seq} send target`); + creditMessage(item.message, `run queue #${seq} send`); + } else if (item.type === 'notify') { + credit(item.kpid, `run queue #${seq} notify`); + } + } else { + const kpid = queueName as KRef; + const message = JSON.parse(entry) as KernelMessage; + credit(kpid, `${kpid} queue #${seq} target`); + creditMessage(message, `${kpid} queue #${seq}`); + } + continue; + } + + const promiseMatch = PROMISE_STATE_KEY.exec(key); + if (promiseMatch) { + const kpid = promiseMatch[1] as KRef; + if (ctx.kv.getRequired(key) === 'unresolved') { + // The unit `initKernelPromise` mints, released when the promise settles. + credit(kpid, 'unsettled promise'); + } else { + const value = JSON.parse( + ctx.kv.getRequired(`${kpid}.value`), + ) as CapData; + for (const slot of value.slots) { + credit(slot, `${kpid} resolution slot`); + } + } + } + } + + for (const kref of getPinnedObjects()) { + credit(kref, 'pin'); + } + + return tallies; + } + + /** + * Collect every kref the store has a refcount entry for. + * + * @returns The krefs with refcount entries. + */ + function getCountedKrefs(): KRef[] { + const krefs: KRef[] = []; + for (const key of getPrefixedKeys('')) { + const match = REFCOUNT_KEY.exec(key); + if (match) { + krefs.push(match[1] as KRef); + } + } + return krefs; + } + + /** + * Compare every kref's stored reference counts against the references the + * kernel can be seen to hold. + * + * @returns The krefs whose counts disagree with ground truth, in kref order. + */ + function auditRefCounts(): RefCountViolation[] { + const expected = computeExpectedRefCounts(); + const violations: RefCountViolation[] = []; + const krefs = new Set([...expected.keys(), ...getCountedKrefs()]); + + for (const kref of [...krefs].sort()) { + const tally = expected.get(kref) ?? { + reachable: 0, + recognizable: 0, + holders: [], + }; + const expectedText = renderCounts(kref, tally); + const raw = ctx.kv.get(refCountKey(kref)); + if (raw === undefined) { + // The kref has been deleted from the kernel, so anything still + // pointing at it is a dangling reference. + if (tally.holders.length > 0) { + violations.push({ + kref, + stored: '(deleted)', + expected: expectedText, + holders: tally.holders, + }); + } + continue; + } + const storedText = isPromiseRef(kref) + ? raw + : renderCounts(kref, getObjectRefCount(kref)); + if (storedText !== expectedText) { + violations.push({ + kref, + stored: storedText, + expected: expectedText, + holders: tally.holders, + }); + } + } + return violations; + } + + /** + * Overwrite stored reference counts with the counts implied by ground truth. + * + * This is how a store written under the pre-fix accounting is brought onto + * the current scheme: the references themselves are authoritative, so the + * counts can simply be rebuilt from them. Krefs that are referenced but have + * already been deleted cannot be repaired this way and are reported instead. + * + * @returns The violations that were corrected and those that could not be. + */ + function recomputeRefCounts(): { + corrected: RefCountViolation[]; + unfixable: RefCountViolation[]; + } { + const corrected: RefCountViolation[] = []; + const unfixable: RefCountViolation[] = []; + for (const violation of auditRefCounts()) { + if (violation.stored === '(deleted)') { + unfixable.push(violation); + continue; + } + ctx.kv.set(refCountKey(violation.kref), violation.expected); + if (violation.expected.startsWith('0')) { + ctx.maybeFreeKrefs.add(violation.kref); + } + corrected.push(violation); + } + return { corrected, unfixable }; + } + + /** + * Render violations as a human-readable report. + * + * @param violations - The violations to describe. + * @returns A multi-line description, one paragraph per violation. + */ + function formatRefCountViolations(violations: RefCountViolation[]): string { + return violations + .map(({ kref, stored, expected, holders }) => { + const held = holders.length > 0 ? holders.join(', ') : 'nothing'; + return `${kref}: stored ${stored}, expected ${expected} (held by: ${held})`; + }) + .join('\n'); + } + + /** + * Audit reference counts and throw if any have drifted. Enabled per kernel + * via the `auditRefCounts` option, and run at the end of every crank. + */ + function assertRefCountsIfAuditing(): void { + if (!ctx.auditRefCounts) { + return; + } + const violations = auditRefCounts(); + if (violations.length > 0) { + throw Error( + `reference count invariant violated:\n${formatRefCountViolations(violations)}`, + ); + } + } + + /** + * Turn the per-crank reference count audit on or off. + * + * @param enabled - Whether to audit after every crank. + */ + function setRefCountAuditing(enabled: boolean): void { + ctx.auditRefCounts = enabled; + } + + return { + auditRefCounts, + recomputeRefCounts, + formatRefCountViolations, + assertRefCountsIfAuditing, + setRefCountAuditing, + }; +} diff --git a/packages/ocap-kernel/src/store/methods/translators.test.ts b/packages/ocap-kernel/src/store/methods/translators.test.ts index 0ac95eafb0..fd1b5e1df6 100644 --- a/packages/ocap-kernel/src/store/methods/translators.test.ts +++ b/packages/ocap-kernel/src/store/methods/translators.test.ts @@ -14,6 +14,7 @@ import type { } from '../../types.ts'; import type { StoreContext } from '../types.ts'; import * as clistModule from './clist.ts'; +import * as reachableModule from './reachable.ts'; import { getTranslators } from './translators.ts'; import * as vatModule from './vat.ts'; @@ -22,6 +23,7 @@ describe('getTranslators', () => { const mockErefToKref = vi.fn(); const mockAllocateErefForKref = vi.fn(); const mockExportFromEndpoint = vi.fn(); + const mockSetReachableFlag = vi.fn(); const mockCtx = {} as StoreContext; beforeEach(() => { @@ -33,6 +35,10 @@ describe('getTranslators', () => { allocateErefForKref: mockAllocateErefForKref, } as unknown as ReturnType); + vi.spyOn(reachableModule, 'getReachableMethods').mockReturnValue({ + setReachableFlag: mockSetReachableFlag, + } as unknown as ReturnType); + vi.spyOn(vatModule, 'getVatMethods').mockReturnValue({ exportFromEndpoint: mockExportFromEndpoint, } as unknown as ReturnType); diff --git a/packages/ocap-kernel/src/store/methods/translators.ts b/packages/ocap-kernel/src/store/methods/translators.ts index c5948fedff..b7d418a1c3 100644 --- a/packages/ocap-kernel/src/store/methods/translators.ts +++ b/packages/ocap-kernel/src/store/methods/translators.ts @@ -21,6 +21,7 @@ import type { } from '../../types.ts'; import type { StoreContext } from '../types.ts'; import { getCListMethods } from './clist.ts'; +import { getReachableMethods } from './reachable.ts'; import { getVatMethods } from './vat.ts'; import { Fail, assert } from '../../utils/assert.ts'; @@ -35,6 +36,7 @@ import { Fail, assert } from '../../utils/assert.ts'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getTranslators(ctx: StoreContext) { const { krefToEref, erefToKref, allocateErefForKref } = getCListMethods(ctx); + const { setReachableFlag } = getReachableMethods(ctx); const { exportFromEndpoint } = getVatMethods(ctx); /** @@ -54,6 +56,11 @@ export function getTranslators(ctx: StoreContext) { /** * Translate a reference from kernel space into endpoint space. * + * Translating is how the kernel hands an endpoint a reference, so it also + * re-establishes reachability: a vat given an object it previously dropped + * holds it live again. Garbage collection deliveries must not do this, and + * don't — they map through `krefsToErefs`, which never touches the flag. + * * @param endpointId - The endpoint for whom translation is desired. * @param kref - The KRef of the entity of interest. * @param importIfNeeded - If true, allocate a new clist entry if necessary; @@ -74,6 +81,7 @@ export function getTranslators(ctx: StoreContext) { throw Fail`unmapped kref ${kref} endpoint=${endpointId}`; } } + setReachableFlag(endpointId, kref); if (isRemoteId(endpointId)) { // The import/export relationship between a vat and the kernel is // asymmetric -- the vat always exports to the kernel and imports from the diff --git a/packages/ocap-kernel/src/store/methods/vat.test.ts b/packages/ocap-kernel/src/store/methods/vat.test.ts index d04c609896..e2a3914f6e 100644 --- a/packages/ocap-kernel/src/store/methods/vat.test.ts +++ b/packages/ocap-kernel/src/store/methods/vat.test.ts @@ -118,6 +118,7 @@ describe('vat store methods', () => { (getBaseMethods as ReturnType).mockReturnValue({ getPrefixedKeys: mockGetPrefixedKeys, getSlotKey: mockGetSlotKey, + getCListPrefix: (endpointId: string) => `${endpointId}.c.`, getOwnerKey: mockGetOwnerKey, }); @@ -273,33 +274,26 @@ describe('vat store methods', () => { it('deletes all keys related to the endpoint', () => { const endpointId = 'e1'; - // Setup mock data - mockKV.set(`cle.${endpointId}.obj1`, 'data1'); - mockKV.set(`cle.${endpointId}.obj2`, 'data2'); - mockKV.set(`clk.${endpointId}.prom1`, 'data3'); + // The c-list holds both directions of each pair under one prefix + mockKV.set(`${endpointId}.c.o-1`, 'ko1'); + mockKV.set(`${endpointId}.c.ko1`, 'R o-1'); + mockKV.set(`${endpointId}.c.p+1`, 'kp1'); mockKV.set(`e.nextObjectId.${endpointId}`, '10'); mockKV.set(`e.nextPromiseId.${endpointId}`, '5'); - mockGetPrefixedKeys.mockImplementation((prefix: string) => { - if (prefix === `cle.${endpointId}.`) { - return [`cle.${endpointId}.obj1`, `cle.${endpointId}.obj2`]; - } - if (prefix === `clk.${endpointId}.`) { - return [`clk.${endpointId}.prom1`]; - } - return []; - }); + mockGetPrefixedKeys.mockImplementation((prefix: string) => + [...mockKV.keys()].filter((key) => key.startsWith(prefix)), + ); vatMethods.deleteEndpoint(endpointId); - expect(mockKV.has(`cle.${endpointId}.obj1`)).toBe(false); - expect(mockKV.has(`cle.${endpointId}.obj2`)).toBe(false); - expect(mockKV.has(`clk.${endpointId}.prom1`)).toBe(false); + expect(mockKV.has(`${endpointId}.c.o-1`)).toBe(false); + expect(mockKV.has(`${endpointId}.c.ko1`)).toBe(false); + expect(mockKV.has(`${endpointId}.c.p+1`)).toBe(false); expect(mockKV.has(`e.nextObjectId.${endpointId}`)).toBe(false); expect(mockKV.has(`e.nextPromiseId.${endpointId}`)).toBe(false); - expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`cle.${endpointId}.`); - expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`clk.${endpointId}.`); + expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`${endpointId}.c.`); }); it('does nothing if endpoint has no associated keys', () => { @@ -309,8 +303,7 @@ describe('vat store methods', () => { expect(() => vatMethods.deleteEndpoint(endpointId)).not.toThrow(); - expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`cle.${endpointId}.`); - expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`clk.${endpointId}.`); + expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`${endpointId}.c.`); }); }); @@ -470,11 +463,10 @@ describe('vat store methods', () => { expect(result).toBe('kp123'); expect(mockInitKernelPromise).toHaveBeenCalled(); expect(mockSetPromiseDecider).toHaveBeenCalledWith('kp123', vatId); + // addCListEntry takes the entry's reference; exportFromEndpoint no + // longer takes one of its own expect(mockAddCListEntry).toHaveBeenCalledWith(vatId, 'kp123', vref); - expect(mockIncrementRefCount).toHaveBeenCalledWith('kp123', 'export', { - isExport: true, - onlyRecognizable: true, - }); + expect(mockIncrementRefCount).not.toHaveBeenCalled(); }); it('creates a kernel object for an exported object', () => { @@ -486,10 +478,7 @@ describe('vat store methods', () => { expect(result).toBe('ko456'); expect(mockInitKernelObject).toHaveBeenCalledWith(vatId); expect(mockAddCListEntry).toHaveBeenCalledWith(vatId, 'ko456', vref); - expect(mockIncrementRefCount).toHaveBeenCalledWith('ko456', 'export', { - isExport: true, - onlyRecognizable: true, - }); + expect(mockIncrementRefCount).not.toHaveBeenCalled(); }); it('throws an error for non-export reference', () => { @@ -566,38 +555,22 @@ describe('vat store methods', () => { }); } - it("decrements the decider refcount for the peer's promise exports", () => { - seedClist([['rp+1', 'kp123']]); - mockGetKernelPromise.mockReturnValue({ decider: endpointId }); - - vatMethods.forgetEndpointImports(endpointId); - - expect(mockDeleteCListEntry).toHaveBeenCalledWith( - endpointId, - 'kp123', - 'rp+1', - ); - expect(mockDecrementRefCount).toHaveBeenCalledWith( - 'kp123', - 'cleanup|peerRestart|promise|decider', - ); - }); - - it('skips the decider decrement when the peer is no longer the decider', () => { + it("releases the peer's promise exports through the c-list", () => { seedClist([['rp+1', 'kp123']]); - mockGetKernelPromise.mockReturnValue({ decider: 'someoneElse' }); vatMethods.forgetEndpointImports(endpointId); + // The caller rejected the promises the peer was deciding first, which + // released the unsettled-promise reference; the entry's own reference is + // all that is left, and deleteCListEntry releases it. expect(mockDeleteCListEntry).toHaveBeenCalledWith( endpointId, 'kp123', 'rp+1', ); - expect(mockDecrementRefCount).not.toHaveBeenCalled(); }); - it("releases the peer's object exports: owner, c-list, baseline refcount, GC", () => { + it("releases the peer's object exports: owner, c-list, GC", () => { seedClist([['ro+7', 'ko42']]); mockKV.set(`owner.ko42`, endpointId); mockGetReachableAndVatSlot.mockReturnValue({ vatSlot: 'ro+7' }); @@ -607,33 +580,26 @@ describe('vat store methods', () => { expect(mockKV.has(`owner.ko42`)).toBe(false); expect(mockKV.has(`slot.${endpointId}.ko42`)).toBe(false); expect(mockKV.has(`slot.${endpointId}.ro+7`)).toBe(false); - expect(mockDecrementRefCount).toHaveBeenCalledWith( - 'ko42', - 'cleanup|peerRestart|export|baseline', - ); expect(mockMaybeFreeKrefs.add).toHaveBeenCalledWith('ko42'); - // Object-export tear-down handles the c-list pair directly; we don't - // also call deleteCListEntry (which uses the recognizable-only path - // and would corrupt the count). + // An export entry carries no reference, so tearing it down changes no + // count; the object is simply orphaned for GC to retire. + expect(mockDecrementRefCount).not.toHaveBeenCalled(); expect(mockDeleteCListEntry).not.toHaveBeenCalled(); }); - it('preserves baseline refcount when ownership has migrated', () => { + it('leaves the owner mapping alone when ownership has migrated', () => { seedClist([['ro+7', 'ko42']]); mockKV.set(`owner.ko42`, 'someoneElse'); mockGetReachableAndVatSlot.mockReturnValue({ vatSlot: 'ro+7' }); vatMethods.forgetEndpointImports(endpointId); - // Foreign owner survives — the baseline reference is theirs now. expect(mockKV.get(`owner.ko42`)).toBe('someoneElse'); - // Our c-list pair is still torn down (the peer can't reach the kref - // through us anymore), but the refcount stays untouched so we don't - // corrupt the new owner's accounting. + // Our c-list pair is still torn down: the peer can't reach the kref + // through us anymore. expect(mockKV.has(`slot.${endpointId}.ko42`)).toBe(false); expect(mockKV.has(`slot.${endpointId}.ro+7`)).toBe(false); expect(mockDecrementRefCount).not.toHaveBeenCalled(); - expect(mockMaybeFreeKrefs.add).not.toHaveBeenCalled(); }); it('preserves our exports to the peer (import-direction entries)', () => { diff --git a/packages/ocap-kernel/src/store/methods/vat.ts b/packages/ocap-kernel/src/store/methods/vat.ts index 48d431dc10..29e2a85dff 100644 --- a/packages/ocap-kernel/src/store/methods/vat.ts +++ b/packages/ocap-kernel/src/store/methods/vat.ts @@ -5,7 +5,6 @@ import { getCListMethods } from './clist.ts'; import { getObjectMethods } from './object.ts'; import { getPromiseMethods } from './promise.ts'; import { getReachableMethods } from './reachable.ts'; -import { getRefCountMethods } from './refcount.ts'; import type { EndpointId, KRef, @@ -35,18 +34,14 @@ const VAT_CONFIG_BASE_LEN = VAT_CONFIG_BASE.length; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getVatMethods(ctx: StoreContext) { const { kv } = ctx; - const { getPrefixedKeys, getSlotKey, getOwnerKey } = getBaseMethods(ctx.kv); + const { getPrefixedKeys, getSlotKey, getCListPrefix, getOwnerKey } = + getBaseMethods(ctx.kv); const { deleteCListEntry } = getCListMethods(ctx); const { getReachableAndVatSlot } = getReachableMethods(ctx); - const { - initKernelPromise, - setPromiseDecider, - getKernelPromise, - addPromiseSubscriber, - } = getPromiseMethods(ctx); - const { initKernelObject, getObjectRefCount } = getObjectMethods(ctx); + const { initKernelPromise, setPromiseDecider, addPromiseSubscriber } = + getPromiseMethods(ctx); + const { initKernelObject } = getObjectMethods(ctx); const { addCListEntry } = getCListMethods(ctx); - const { incrementRefCount, decrementRefCount } = getRefCountMethods(ctx); /** * Delete all persistent state associated with an endpoint. @@ -54,10 +49,7 @@ export function getVatMethods(ctx: StoreContext) { * @param endpointId - The endpoint whose state is to be deleted. */ function deleteEndpoint(endpointId: EndpointId): void { - for (const key of getPrefixedKeys(`cle.${endpointId}.`)) { - kv.delete(key); - } - for (const key of getPrefixedKeys(`clk.${endpointId}.`)) { + for (const key of getPrefixedKeys(getCListPrefix(endpointId))) { kv.delete(key); } kv.delete(`e.nextObjectId.${endpointId}`); @@ -261,11 +253,8 @@ export function getVatMethods(ctx: StoreContext) { const { vatSlot } = getReachableAndVatSlot(vatID, kref); ctx.kv.delete(getSlotKey(vatID, kref)); ctx.kv.delete(getSlotKey(vatID, vatSlot)); - // Skip baseline decrement if GC already zeroed reachable via dropImports. - const { reachable } = getObjectRefCount(kref); - if (reachable > 0) { - decrementRefCount(kref, 'cleanup|export|baseline'); - } + // An export entry holds no count, so there is nothing to release; the + // object is now orphaned, and GC retires it once importers let go. ctx.maybeFreeKrefs.add(kref); work.exports += 1; } @@ -282,20 +271,15 @@ export function getVatMethods(ctx: StoreContext) { work.imports += 1; } - // The caller used enumeratePromisesByDecider() before calling us, - // so they have already rejected the orphan promises, but those - // kpids are still present in the dead vat's c-list. Clean those up now. + // The caller rejected the orphan promises via getPromisesByDecider() before + // calling us, which is what released each promise's unsettled reference, + // but their kpids are still in the dead vat's c-list. Clean those up now. for (const key of getPrefixedKeys(promisePrefix)) { const krefStr = ctx.kv.get(key) ?? Fail`getNextKey ensures get`; assert(key.startsWith(clistPrefix), key); const vref = key.slice(clistPrefix.length) as ERef; // the following will also delete both db keys deleteCListEntry(vatID, krefStr, vref); - // If the dead vat was still the decider, drop the decider’s refcount, too. - const kp = getKernelPromise(krefStr); - if (kp.decider === vatID) { - decrementRefCount(krefStr, 'cleanup|promise|decider'); - } work.promises += 1; } @@ -377,49 +361,28 @@ export function getVatMethods(ctx: StoreContext) { } const { isPromise } = parseRef(eref); if (isPromise) { - // deleteCListEntry decrements the promise refcount via the - // recognizable path. Additionally, if the endpoint was still - // recorded as decider, drop the decider's reference too. + // The caller already rejected the promises this endpoint was deciding, + // so only the c-list entry's own reference is left. deleteCListEntry(endpointId, kref, eref); - const kp = getKernelPromise(kref); - if (kp.decider === endpointId) { - decrementRefCount(kref, 'cleanup|peerRestart|promise|decider'); - } } else { // Object exports: drop the owner mapping if it still names the - // restarting endpoint, decrement the baseline refcount the kernel - // implicitly held while the endpoint owned the object, and queue - // it for GC. Then tear down the c-list pair. - // - // We deliberately do NOT call deleteCListEntry here: that path uses - // `onlyRecognizable: true`, which is the right semantics for an - // endpoint dropping its imports but the wrong semantics for - // releasing an export the endpoint owned. The baseline decrement - // below corresponds to the implicit reference exportFromEndpoint - // installed when the kernel object was first created. + // restarting endpoint, tear down the c-list pair, and queue the object + // for GC. An export entry holds no count, so this changes none. If + // ownership has migrated (e.g. a kernel-internal handoff), leave the + // new owner's mapping alone: the kref is theirs from here. const ownerKey = getOwnerKey(kref); const currentOwner = ctx.kv.get(ownerKey); - const stillOwned = currentOwner === endpointId; - if (stillOwned) { + if (currentOwner === endpointId) { ctx.kv.delete(ownerKey); } else if (currentOwner !== undefined) { - // Ownership has migrated (e.g. via a kernel-internal handoff). - // The baseline reference is now owed to the new owner; do not - // decrement against their accounting. Tear down our c-list pair - // and stop — the new owner is responsible for the kref's lifetime. ctx.logger?.warn( `forgetEndpointImports: kref ${kref} was exported by ${endpointId} ` + - `but is now owned by ${currentOwner}; preserving baseline refcount`, + `but is now owned by ${currentOwner}`, ); - const { vatSlot } = getReachableAndVatSlot(endpointId, kref); - ctx.kv.delete(getSlotKey(endpointId, kref)); - ctx.kv.delete(getSlotKey(endpointId, vatSlot)); - continue; } const { vatSlot } = getReachableAndVatSlot(endpointId, kref); ctx.kv.delete(getSlotKey(endpointId, kref)); ctx.kv.delete(getSlotKey(endpointId, vatSlot)); - decrementRefCount(kref, 'cleanup|peerRestart|export|baseline'); ctx.maybeFreeKrefs.add(kref); } } @@ -444,11 +407,9 @@ export function getVatMethods(ctx: StoreContext) { } else { kref = initKernelObject(endpointId); } + // addCListEntry takes the entry's reference: none for an object, since the + // owner is not one of its referrers, and one for a promise. addCListEntry(endpointId, kref, eref); - incrementRefCount(kref, 'export', { - isExport: true, - onlyRecognizable: true, - }); ctx.logger?.debug('exportFromEndpoint', endpointId, eref, kref); if (context === 'remote' && isPromise) { addPromiseSubscriber(endpointId, kref); diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index 1ea54d19cc..3bf54862fa 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -27,6 +27,7 @@ export type StoreContext = { subclusters: StoredValue; // Holds Subcluster[] nextSubclusterId: StoredValue; // Holds string vatToSubclusterMap: StoredValue; // Holds Record + auditRefCounts: boolean; // If set, verify refcounts against ground truth every crank logger?: Logger | undefined; }; diff --git a/packages/ocap-kernel/src/vats/VatManager.ts b/packages/ocap-kernel/src/vats/VatManager.ts index b90f6f30c8..314bfa87fb 100644 --- a/packages/ocap-kernel/src/vats/VatManager.ts +++ b/packages/ocap-kernel/src/vats/VatManager.ts @@ -128,6 +128,11 @@ export class VatManager { vatId, ROOT_OBJECT_VREF, ); + // A root is addressable for as long as its vat lives, whether or not + // anyone currently imports it: the kernel's own API hands out root krefs + // and `getRootObject` resolves them through this c-list entry. Without a + // pin, GC would retire the entry the moment the last importer let go. + this.#kernelStore.pinObject(rootRef); this.#kernelStore.setVatConfig(vatId, vatConfig); return rootRef; } @@ -186,6 +191,14 @@ export class VatManager { } else if (terminating) { terminationError = new VatDeletedError(vatId); } + if (terminating) { + // Release the pin `launchVat` took, so the root can be collected once + // its importers let go. A restart keeps it: the same root comes back. + const rootRef = this.#kernelStore.getRootObject(vatId); + if (rootRef) { + this.#kernelStore.unpinObject(rootRef); + } + } await this.#platformServices .terminate(vatId, terminationError) .catch(this.#logger.error); From d921e9fe6be37b2d4ec72b359c757fc00dfeac0b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:09:30 +0200 Subject: [PATCH 02/34] fix(ocap-kernel): format the changelog and cite this PR Prettier wanted a blank line before the entry following a nested bullet, and the entries still cited #1010, which this PR replaces. --- packages/ocap-kernel/CHANGELOG.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 0b9328172a..9a7e0b3050 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -37,10 +37,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Purely an RPC-boundary concern: internal callers of `Kernel.queueMessage` are unaffected - Caveat: a legitimate string argument that begins with `@@` followed by alphanumerics will be misinterpreted as a marker; wrap such literals inside an object -- Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high with no holder (a leak) - Exports the `RefCountViolation` type -- Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) ### Changed @@ -71,21 +71,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Report the database error when an aborted crank cannot be rolled back, instead of a spurious "no such savepoint" ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - The abort path recorded the rollback only after it succeeded, so a throwing rollback had the run loop try again against the savepoint `rollbackCrank` had already discarded. The second attempt's "no such savepoint" then became the reported cause of death — and since only `error.message` crosses the wire, the real failure reached neither `getStatus` nor the daemon log - Reject the run loop's promise with the same `Error` its status reports, rather than re-throwing a non-`Error` for the embedder to normalize a second time ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) -- **BREAKING:** Make c-list reference accounting symmetric: creating an import c-list entry now takes a reference, as tearing one down has always released one ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- **BREAKING:** Make c-list reference accounting symmetric: creating an import c-list entry now takes a reference, as tearing one down has always released one ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - `initKernelObject` now births objects at `(0, 0)` instead of `(1, 1)`. The old constant made the arithmetic come out right for exactly one importer, masking the missing increment; with two importers a live capability could be dropped and retired out from under a holder - Removes the owner-side baseline decrements in `cleanupTerminatedVat` and `forgetEndpointImports`, which double-claimed the same unit an importer's drop also spent — the source of `"koNN" underflow -1,0` escaping mid-cleanup and leaving a vat half-cleaned - `translateRefKtoE` now re-establishes reachability, so a vat handed an object it previously dropped is counted as holding it live again - Renames `krefsToExistingErefs` to `krefsToErefs`, which now throws on an unmapped kref instead of silently dropping it -- Pin vat root objects for the lifetime of their vat, and release the pin on termination ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Pin vat root objects for the lifetime of their vat, and release the pin on termination ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - A root is addressable while its vat lives whether or not anyone imports it; the old `(1, 1)` birth baseline was standing in for this -- Garbage-collection action delivery now moves the kernel's own c-list: `dropExports` clears the owner's reachable flag and `retireExports`/`retireImports` tear the entry down ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Garbage-collection action delivery now moves the kernel's own c-list: `dropExports` clears the owner's reachable flag and `retireExports`/`retireImports` tear the entry down ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Previously the owner's flag never cleared, so the same action could be re-derived, and retired entries outlived the objects they named -- Charge a delivered message's target reference against the run-queue item's own target rather than the routed target, so a message routed through a resolved promise no longer decrements an object nobody charged while leaking the promise ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) -- Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) -- Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) -- Fix the stale `cle.`/`clk.` key prefixes in `getPromisesByDecider` and `deleteEndpoint`, which no longer matched the `${endpointId}.c.` c-list layout ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010)) +- Charge a delivered message's target reference against the run-queue item's own target rather than the routed target, so a message routed through a resolved promise no longer decrements an object nobody charged while leaking the promise ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Fix the stale `cle.`/`clk.` key prefixes in `getPromisesByDecider` and `deleteEndpoint`, which no longer matched the `${endpointId}.c.` c-list layout ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - `getPromisesByDecider` matched nothing, so promises a terminating vat or restarting peer was deciding were never rejected + - Deserialize CapData rejections in `Kernel.queueMessage` so vat errors surface as plain `Error` objects to all callers ([#928](https://github.com/MetaMask/ocap-kernel/pull/928)) - Detect peer restart across receiver state loss so the receiving kernel no longer silently drops a restarted peer's `seq=1` messages ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) - Persist the peer's last-observed incarnation and compare it on every successful handshake; on a detected restart, clear the peer's c-list contributions and reject the promises it was deciding before the new incarnation reuses any erefs From 1f9c8881327296af70ab0b56408696aaffb1d0f9 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:20:12 +0200 Subject: [PATCH 03/34] fix(ocap-kernel): don't audit an importer entry the collector is retiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `retireKernelObjects` deletes an object and queues a `retireImport` for each importer in the same breath, so until that action is delivered an importer's c-list entry names a kref the kernel has already dropped. The audit counted those entries as holders and reported a violation against the collector's own output — and since `assertRefCountsIfAuditing` throws from inside the crank, that killed the run loop for good. Reachable from an ordinary `terminateVat` while a surviving vat holds the dying vat's export in liveslots' dropped-but-recognizable state. No current test produced it; found by Cursor Bugbot on #1020 and reproduced against the real store. Co-Authored-By: Claude Opus 5 (1M context) --- .../store/methods/clist-accounting.test.ts | 20 +++++++++++++++++++ .../src/store/methods/refcount-audit.ts | 14 ++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts index 49d4d384e5..c7d1cf22d1 100644 --- a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts +++ b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts @@ -216,6 +216,26 @@ describe('c-list reference accounting', () => { expect(kernelStore.auditRefCounts()).toStrictEqual([]); }); + it('tolerates an importer entry that outlives the object it names', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v3', kref, true); + // v3 has let go of the object but can still recognize the name + kernelStore.clearReachableFlag('v3', kref); + kernelStore.markVatAsTerminated('v1'); + kernelStore.cleanupTerminatedVat('v1'); + + kernelStore.collectGarbage(); + + // The collector deletes the object and queues the retirement together, so + // v3's entry names a kref the kernel has already dropped until that action + // is delivered. Counting it as a holder fails the end-of-crank audit on a + // state the collector itself just created, which kills the run loop. + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v3 retireImport ${kref}`, + ]); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + it('leaves a live vat that shares the object untouched', () => { const kref = kernelStore.exportFromEndpoint('v2', 'o+1'); kernelStore.translateRefKtoE('v1', kref, true); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index c43fd904de..e79a63e946 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -98,6 +98,15 @@ export function getRefCountAuditMethods(ctx: StoreContext) { */ function computeExpectedRefCounts(): Map { const tallies = new Map(); + // `retireKernelObjects` deletes an object and queues a `retireImport` for + // each importer in the same breath, so between then and the delivery an + // importer's c-list entry legitimately names a kref the kernel has already + // dropped. Those entries are scheduled for teardown and are not holders. + const retiring = new Set( + (JSON.parse(ctx.gcActions.get() ?? '[]') as string[]).filter((action) => + action.includes(' retireImport '), + ), + ); const credit = ( kref: KRef, @@ -152,7 +161,10 @@ export function getRefCountAuditMethods(ctx: StoreContext) { if (isPromiseRef(kref)) { // Both directions count for a promise. credit(kref, holder); - } else if (direction === 'import') { + } else if ( + direction === 'import' && + !retiring.has(`${endpointId} retireImport ${kref}`) + ) { // An object export is the owner's own entry and carries no count; // an object import always recognizes and, while flagged, reaches. credit(kref, holder, { onlyRecognizable: !isReachable }); From e26c92924a4d55d68b3632fb4dedb13bee71805b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 14 Aug 2026 16:26:06 +0200 Subject: [PATCH 04/34] fix(ocap-kernel): retain the holders the accounting cannot see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebasing the baseline to (0, 0) made every reference explicit, which exposed the holders that were never references at all. An ocap URL carries its kref inside an encrypted bearer token and nothing else, so the kernel cannot discover from its own state that a holder exists: `issueOcapURL` took no reference of any kind. Under the old baseline nothing exported was collectable and it never showed; at (0, 0) the target is collected as soon as the message that carried it to the issuer is delivered, and the URL names a dead capability. The audit is silent on it by construction — the object genuinely has no holder it can see. Retain the target when the URL is issued, before the token exists, since the token is unretractable once it does. One pin per kref however many URLs name it, and no release: the token is persistent and unexpiring, so `revoke` is how the capability dies. Pinning also puts the holder inside the reference graph, so the audit can see it rather than being taught to excuse it. The same shape had a second door. `incrementRefCount` has no `kernelRefExists` guard where `decrementRefCount` does, so importing a deleted kref read its missing counts as (0, 0) and wrote them back, resurrecting a live-looking object with no owner — deliverable to by nobody, and endorsed by the audit, since the new c-list entry is a legitimate holder for exactly the count it finds. Reached by redeeming a URL issued for an object since collected. Guard the point of corruption, `translateRefKtoE`, rather than `incrementRefCount` itself: creating an entry for a deleted kref is the invariant, and releasing a reference to something already gone is how GC teardown is allowed to race deletion. Also release a vat's root pin when `deleteSubcluster` retires vats that never ran here. It bypasses `stopVat`, so nothing released the pin `launchVat` took in the incarnation that did run them, leaving the root's count permanently above zero and `pinnedObjects` naming a vat that no longer exists. `stopVat` and `deleteSubcluster` now share `releaseVatRootPin`. Vat root pinning had no unit coverage at all, so pin-on-launch, release-on-terminate and keep-across-restart are asserted now; the last is what the comment claims and what would break silently. Restores the `maybeFreeKrefs` assertion on `forgetEndpointImports`' ownership-migrated branch, which lost its `not.toHaveBeenCalled` when that branch stopped returning early. Corrects three claims that the (0, 0) birth falsified and that shipped as documentation: both `KernelServiceManager` comments asserting its delete branch cannot fire, when it now does, and a changelog entry asserting (1, 1) birth two dozen lines above one asserting (0, 0). `recomputeRefCounts` no longer describes itself as a migration; nothing calls it, and opening an existing store does not migrate one. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 17 ++++- packages/ocap-kernel/src/Kernel.ts | 1 + .../ocap-kernel/src/KernelServiceManager.ts | 12 ++-- .../src/remotes/kernel/OcapURLManager.test.ts | 63 ++++++++++++++++--- .../src/remotes/kernel/OcapURLManager.ts | 12 +++- .../src/remotes/kernel/RemoteHandle.test.ts | 7 ++- packages/ocap-kernel/src/store/index.test.ts | 2 + packages/ocap-kernel/src/store/index.ts | 31 +++++++++ .../src/store/methods/refcount-audit.ts | 20 ++++-- .../ocap-kernel/src/store/methods/refcount.ts | 5 ++ .../src/store/methods/translators.test.ts | 19 ++++++ .../src/store/methods/translators.ts | 11 ++++ .../ocap-kernel/src/store/methods/vat.test.ts | 4 ++ .../src/vats/SubclusterManager.test.ts | 6 ++ .../ocap-kernel/src/vats/SubclusterManager.ts | 5 ++ .../ocap-kernel/src/vats/VatManager.test.ts | 45 +++++++++++++ packages/ocap-kernel/src/vats/VatManager.ts | 34 +++++++--- packages/ocap-kernel/test/remotes-mocks.ts | 1 + 18 files changed, 263 insertions(+), 32 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 9a7e0b3050..1079dd9db7 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -38,9 +38,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Caveat: a legitimate string argument that begins with `@@` followed by alphanumerics will be misinterpreted as a marker; wrap such literals inside an object - Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high with no holder (a leak) + - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high (a leak) + - Only references visible in the kernel's own state are checkable, so a holder that keeps a kref outside them has to take a pin to be counted at all + - `recomputeRefCounts` is a repair tool for a drifted store, offered to embedders and never run automatically: opening an existing store does not migrate it - Exports the `RefCountViolation` type - Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Add `getOcapURLObjects` and `retainForOcapURL` to the kernel store, and `VatManager.releaseVatRootPin` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) ### Changed @@ -55,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A message delivered to a kernel-owned kref with no registered service now rejects the caller with `ENDPOINT_UNREACHABLE` instead of throwing, which escaped the crank and killed the run loop — turning one unreachable reference into a dead kernel ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007)) - - Reachable without any kernel bug: an anonymous kernel object hosts something that cannot outlive the process, such as an accepted socket connection, so a vat holding one across a restart or a message to one still queued from the previous incarnation lands here. Kernel objects are born with a `(1, 1)` refcount, so the init sweep cannot delete such an object and its `kernel` owner survives (see [#1006](https://github.com/MetaMask/ocap-kernel/issues/1006)) + - Reachable without any kernel bug: an anonymous kernel object hosts something that cannot outlive the process, such as an accepted socket connection, so a vat holding one across a restart or a message to one still queued from the previous incarnation lands here. That surviving reference is exactly what stops the init sweep deleting the object, so its `kernel` owner survives with it - Matches what `KernelRouter` already does for a delivery whose endpoint has vanished. A message sent with no result promise has nobody to report to, so it is logged instead - The kernel run queue no longer strands messages, going quiet with no error, no log, and no crash. `runQueueLengthCache` uses a negative value to mean "unknown, re-read from the database", but `enqueueRun`/`dequeueRun` adjusted it arithmetically without materializing it first — so an enqueue while the cache held `-1` produced `0` for a queue that actually held an item, and because `0` is not negative it was never re-read again. The run loop then saw an empty queue, went to sleep, and stranded everything behind it. Two paths reach that `-1`: kernel startup, and `rollbackCrank`, which invalidates the cache because a rollback may have restored dequeued items. The rollback path is the more likely of the two in practice, since a rollback is normally followed immediately by enqueueing an error or termination message. The run loop is also now woken by any non-empty queue rather than only by the empty-to-one transition, so a drifted count cannot silently lose the wakeup either ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007)) - Stop reporting a healthy kernel after the run loop dies ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) @@ -84,8 +87,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Fix the stale `cle.`/`clk.` key prefixes in `getPromisesByDecider` and `deleteEndpoint`, which no longer matched the `${endpointId}.c.` c-list layout ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - - `getPromisesByDecider` matched nothing, so promises a terminating vat or restarting peer was deciding were never rejected +- Issuing an ocap URL now retains its target, so the URL stays redeemable ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) + - A URL carries its kref inside an encrypted bearer token and nothing else, so the kernel cannot see from its own state that a holder exists. Under the old `(1, 1)` birth baseline nothing exported was ever collectable and this went unnoticed; at `(0, 0)` the target is collected as soon as the message that carried it to the issuer is delivered, and the URL names a dead capability + - One pin per kref however many URLs name it, and no release: the token is persistent and unexpiring, so `revoke` is the way to kill the capability + - Issuing a URL for a kref the kernel has already deleted is now refused rather than resurrecting its counts +- Refuse to import a kref the kernel has deleted into an endpoint's c-list ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) + - `getObjectRefCount` reads a missing entry as `(0, 0)`, so the new entry's own increment wrote it back and resurrected a live-looking object with no owner — deliverable to by nobody, and endorsed by the audit, since the entry is a legitimate holder for exactly the count it finds +- Release a vat's root pin when a subcluster is deleted without its vats having run ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) + + - `deleteSubcluster` bypasses `stopVat`, so nothing released the pin `launchVat` took in the incarnation that did run them, leaving the root's count permanently above zero and `pinnedObjects` naming a vat that no longer exists - Deserialize CapData rejections in `Kernel.queueMessage` so vat errors surface as plain `Error` objects to all callers ([#928](https://github.com/MetaMask/ocap-kernel/pull/928)) - Detect peer restart across receiver state loss so the receiving kernel no longer silently drops a restarted peer's `seq=1` messages ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) diff --git a/packages/ocap-kernel/src/Kernel.ts b/packages/ocap-kernel/src/Kernel.ts index 94865cb049..c041d060be 100644 --- a/packages/ocap-kernel/src/Kernel.ts +++ b/packages/ocap-kernel/src/Kernel.ts @@ -176,6 +176,7 @@ export class Kernel { this.#ocapURLManager = new OcapURLManager({ remoteManager: this.#remoteManager, + kernelStore: this.#kernelStore, }); this.#kernelServiceManager = new KernelServiceManager({ diff --git a/packages/ocap-kernel/src/KernelServiceManager.ts b/packages/ocap-kernel/src/KernelServiceManager.ts index bc236e6113..6744c98c74 100644 --- a/packages/ocap-kernel/src/KernelServiceManager.ts +++ b/packages/ocap-kernel/src/KernelServiceManager.ts @@ -162,11 +162,10 @@ export class KernelServiceManager { * harmful if left: still pinned, so they accumulate with every restart. * * Note what this does *not* guarantee. The kernel object is deleted only - * once nothing references it, which with the current `(1, 1)` refcount - * baseline (see #1006) is never; a survivor therefore keeps its `'kernel'` - * owner, and a delivery to it still routes to `invokeKernelService`. That - * case is made survivable there, by rejecting the caller's promise rather - * than throwing, and not here. + * once nothing references it, so a survivor that a vat import or a still + * queued message holds keeps its `'kernel'` owner, and a delivery to it + * still routes to `invokeKernelService`. That case is made survivable there, + * by rejecting the caller's promise rather than throwing, and not here. * * Runs before the run queue starts, so the unpinning is complete before * anything can address one of these krefs. @@ -196,8 +195,7 @@ export class KernelServiceManager { * * The kernel object itself is deleted here once nothing references it, * rather than being left to `collectGarbage`, which skips kernel-owned - * objects. With the current refcount baseline this branch does not fire; - * it is the correct place for the deletion once that changes (see #1006). + * objects. * * @param kref - The kref of the object to release. */ diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts index b97b79369c..8c59ab9389 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts @@ -7,6 +7,8 @@ import type { RemoteManager } from './RemoteManager.ts'; import { createMockRemotesFactory } from '../../../test/remotes-mocks.ts'; import type { SlotValue } from '../../liveslots/kernel-marshal.ts'; import { kslot } from '../../liveslots/kernel-marshal.ts'; +import type { KernelStore } from '../../store/index.ts'; +import type { KRef } from '../../types.ts'; import type { RemoteComms } from '../types.ts'; type RedeemService = { @@ -23,6 +25,12 @@ describe('OcapURLManager', () => { let mockRemoteComms: RemoteComms; let mockRemoteHandle: RemoteHandle; let mockFactory: ReturnType; + let mockKernelStore: KernelStore; + // Issuing retains the target, which requires it to exist, so mint real + // entities rather than naming krefs the kernel never had. + let objectKRef: KRef; + let otherObjectKRef: KRef; + let promiseKRef: KRef; beforeEach(() => { mockFactory = createMockRemotesFactory({ @@ -33,6 +41,10 @@ describe('OcapURLManager', () => { const mocks = mockFactory.makeOcapURLManagerMocks(); mockRemoteComms = mocks.remoteComms; mockRemoteHandle = mocks.remoteHandle; + mockKernelStore = mocks.kernelStore; + objectKRef = mockKernelStore.initKernelObject('kernel'); + otherObjectKRef = mockKernelStore.initKernelObject('kernel'); + [promiseKRef] = mockKernelStore.initKernelPromise(); mockRemoteManager = mocks.remoteManager as unknown as RemoteManager; // Override specific mock behaviors for this test @@ -48,6 +60,7 @@ describe('OcapURLManager', () => { ocapURLManager = new OcapURLManager({ remoteManager: mockRemoteManager, + kernelStore: mockKernelStore, }); }); @@ -92,8 +105,42 @@ describe('OcapURLManager', () => { }); describe('issueOcapURL', () => { + it('retains the target, so garbage collection cannot take it', async () => { + // The URL is the only holder, and it lives outside the store's reference + // graph, so without the pin the target collects and the URL goes dead. + mockKernelStore.incrementRefCount(objectKRef, 'queue|slot'); + await ocapURLManager.issueOcapURL(objectKRef); + mockKernelStore.decrementRefCount(objectKRef, 'deliver|send|slot'); + + expect(mockKernelStore.isObjectPinned(objectKRef)).toBe(true); + expect(mockKernelStore.getOcapURLObjects()).toStrictEqual([objectKRef]); + mockKernelStore.collectGarbage(); + expect([...mockKernelStore.getGCActions()]).toStrictEqual([]); + expect(mockKernelStore.kernelRefExists(objectKRef)).toBe(true); + }); + + it('retains a target named by several URLs only once', async () => { + await ocapURLManager.issueOcapURL(objectKRef); + await ocapURLManager.issueOcapURL(objectKRef); + + expect(mockKernelStore.getPinnedObjects()).toStrictEqual([objectKRef]); + expect(mockKernelStore.getObjectRefCount(objectKRef)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + + it('refuses to issue a URL for a kref the kernel has deleted', async () => { + mockKernelStore.deleteKernelObject(objectKRef); + + await expect(ocapURLManager.issueOcapURL(objectKRef)).rejects.toThrow( + `cannot issue an ocap URL for deleted kref "${objectKRef}"`, + ); + expect(mockRemoteComms.issueOcapURL).not.toHaveBeenCalled(); + }); + it('issues OCAP URL for a kref', async () => { - const kref = 'ko123'; + const kref = objectKRef; const url = await ocapURLManager.issueOcapURL(kref); expect(url).toBe('ocap:abc123@local-peer-id'); @@ -183,7 +230,7 @@ describe('OcapURLManager', () => { describe('issuer service', () => { it('issues URL through issuer service with valid remotable', async () => { // Create a valid remotable object that krefOf can extract a kref from - const kref = 'ko777'; + const kref = objectKRef; const remotableObj = kslot(kref, 'TestInterface'); vi.spyOn(mockRemoteComms, 'issueOcapURL').mockResolvedValue( @@ -201,7 +248,7 @@ describe('OcapURLManager', () => { it('issues URL through issuer service with promise kref', async () => { // Create a promise-type kref (starts with 'p', 'kp', or 'rp') - const promiseKref = 'kp888'; + const promiseKref = promiseKRef; const promiseObj = kslot(promiseKref); vi.spyOn(mockRemoteComms, 'issueOcapURL').mockResolvedValue( @@ -223,7 +270,7 @@ describe('OcapURLManager', () => { // The issuer service is already tested implicitly through other tests. // Test that issueOcapURL is called correctly directly - const kref = 'ko777'; + const kref = objectKRef; vi.spyOn(mockRemoteComms, 'issueOcapURL').mockResolvedValue( `ocap:issued@local-peer-id`, ); @@ -291,7 +338,7 @@ describe('OcapURLManager', () => { describe('integration scenarios', () => { it('handles round-trip issue and redeem', async () => { // Issue a URL - const kref = 'ko789'; + const kref = objectKRef; vi.spyOn(mockRemoteComms, 'issueOcapURL').mockResolvedValue( 'ocap:xyz789@local-peer-id', ); @@ -306,8 +353,8 @@ describe('OcapURLManager', () => { it('handles multiple simultaneous operations', async () => { const promises = [ - ocapURLManager.issueOcapURL('ko1'), - ocapURLManager.issueOcapURL('ko2'), + ocapURLManager.issueOcapURL(objectKRef), + ocapURLManager.issueOcapURL(otherObjectKRef), ocapURLManager.redeemOcapURL('ocap:abc@local-peer-id'), ocapURLManager.redeemOcapURL('ocap:def@remote-peer-id'), ]; @@ -327,7 +374,7 @@ describe('OcapURLManager', () => { new Error('Issue failed'), ); - await expect(ocapURLManager.issueOcapURL('ko123')).rejects.toThrow( + await expect(ocapURLManager.issueOcapURL(objectKRef)).rejects.toThrow( 'Issue failed', ); }); diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts index eec66cae1b..2a2b97f3cc 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts @@ -4,6 +4,7 @@ import { parseOcapURL } from './remote-comms.ts'; import type { RemoteManager } from './RemoteManager.ts'; import { kslot, krefOf } from '../../liveslots/kernel-marshal.ts'; import type { SlotValue } from '../../liveslots/kernel-marshal.ts'; +import type { KernelStore } from '../../store/index.ts'; import type { KRef } from '../../types.ts'; /** @@ -25,6 +26,7 @@ export type OcapURLRedemptionService = { type OcapURLManagerConstructorProps = { remoteManager: RemoteManager; + kernelStore: KernelStore; }; /** @@ -34,6 +36,9 @@ export class OcapURLManager { /** Remote manager for handling remote connections */ readonly #remoteManager: RemoteManager; + /** The kernel's store, for retaining the objects issued URLs name */ + readonly #kernelStore: KernelStore; + /** OCAP URL issuer service object */ readonly #ocapURLIssuerService: object; @@ -45,9 +50,11 @@ export class OcapURLManager { * * @param options - Constructor options. * @param options.remoteManager - The remote manager for handling cross-kernel communications. + * @param options.kernelStore - The kernel's store. */ - constructor({ remoteManager }: OcapURLManagerConstructorProps) { + constructor({ remoteManager, kernelStore }: OcapURLManagerConstructorProps) { this.#remoteManager = remoteManager; + this.#kernelStore = kernelStore; // Create the OCAP URL issuer service this.#ocapURLIssuerService = Far('ocapURLIssuerService', { @@ -119,6 +126,9 @@ export class OcapURLManager { */ async issueOcapURL(kref: KRef): Promise { const identity = this.#remoteManager.getRemoteIdentity(); + // Before minting the token, not after: the URL is unretractable once it + // exists, so the target must already be retained. See `retainForOcapURL`. + this.#kernelStore.retainForOcapURL(kref); return identity.issueOcapURL(kref); } diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index 009834a65b..4f53572c0d 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -660,7 +660,12 @@ describe('RemoteHandle', () => { const remote = makeRemote(); const mockOcapURL = 'as if it was a URL'; const mockReplyKey = 'replyKey'; - const replyKRef = 'ko100'; + // A URL only ever names an object the kernel still has, so redeem one that + // exists: importing a deleted kref is refused outright. + const replyKRef = mockKernelStore.initKernelObject('kernel'); + vi.spyOn(mockRemoteComms, 'redeemLocalOcapURL').mockResolvedValue( + replyKRef, + ); const replyRRef = 'ro+1'; // Include seq for incoming message const request = JSON.stringify({ diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index d4e68387cf..83b27dea79 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -103,6 +103,7 @@ describe('kernel store', () => { 'getNextRemoteId', 'getNextVatId', 'getObjectRefCount', + 'getOcapURLObjects', 'getOwner', 'getPeerIncarnation', 'getPendingMessage', @@ -159,6 +160,7 @@ describe('kernel store', () => { 'removeVatFromSubcluster', 'reset', 'resolveKernelPromise', + 'retainForOcapURL', 'retireKernelObjects', 'revoke', 'rollbackCrank', diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 9ac1085f18..00a8d591f2 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -65,6 +65,7 @@ * kernelService.${serviceName} = ${koid} // kref of kernel service object ${serviceName} */ +import { Fail } from '@endo/errors'; import type { KernelDatabase, KVStore, VatStore } from '@metamask/kernel-store'; import { Logger } from '@metamask/logger'; @@ -368,6 +369,36 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { kv.set('anonymousKernelObjects', [...krefs].sort().join(',')); } }, + + // Objects named by an issued ocap URL + // + // An ocap URL is a durable bearer token: it carries an encrypted kref and + // nothing else, so the kernel cannot discover from its own state that a + // holder exists. Retaining the target is therefore the only thing keeping + // the URL redeemable, and it has to outlive every other reference — the + // token is persistent, unexpiring, and may be redeemed by a peer that was + // not running when it was issued. `revoke` is the way to kill the + // capability; there is deliberately no release here. + getOcapURLObjects(): KRef[] { + const raw = kv.get('ocapURLObjects'); + return raw ? (raw.split(',') as KRef[]) : []; + }, + retainForOcapURL(kref: KRef): void { + // Refuse to mint a token for something already collected: the pin would + // resurrect a `(1, 1)` row for an object with no owner, and the URL would + // name a capability that can never be delivered to. + this.kernelRefExists(kref) || + Fail`cannot issue an ocap URL for deleted kref ${kref}`; + const krefs = new Set(this.getOcapURLObjects()); + // One pin per kref, however many URLs name it: pins are a multiset, and + // a second pin here would be one nothing could ever release. + if (krefs.has(kref)) { + return; + } + krefs.add(kref); + kv.set('ocapURLObjects', [...krefs].sort().join(',')); + this.pinObject(kref); + }, }); } diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index e79a63e946..b4178e5c6f 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -59,6 +59,12 @@ const REFCOUNT_KEY = /^(k[op]\d+)\.refCount$/u; * where the cache has drifted, in either direction. Counts that are too low * let a live capability be collected; counts that are too high leak it. * + * Only references the kernel can see in its own state are checkable, so a + * holder that keeps a kref outside them is invisible here and the audit will + * pronounce its target unreferenced. Anything of that shape has to take a pin + * to be counted at all — see `retainForOcapURL`, where an issued URL's + * encrypted kref does exactly that. + * * @param ctx - The store context. * @returns The reference count audit methods. */ @@ -287,10 +293,16 @@ export function getRefCountAuditMethods(ctx: StoreContext) { /** * Overwrite stored reference counts with the counts implied by ground truth. * - * This is how a store written under the pre-fix accounting is brought onto - * the current scheme: the references themselves are authoritative, so the - * counts can simply be rebuilt from them. Krefs that are referenced but have - * already been deleted cannot be repaired this way and are reported instead. + * A repair tool for a store whose counts have drifted, offered to embedders + * and never run automatically: nothing calls it, and opening an existing + * store does not migrate it. Krefs that are referenced but have already been + * deleted cannot be repaired this way and are reported instead. + * + * Ground truth here means the references the kernel can see in its own + * state. A holder the store cannot see — an issued ocap URL names its target + * only inside an encrypted bearer token — is not among them, which is why + * such a target is pinned when the URL is issued rather than left for this to + * infer. * * @returns The violations that were corrected and those that could not be. */ diff --git a/packages/ocap-kernel/src/store/methods/refcount.ts b/packages/ocap-kernel/src/store/methods/refcount.ts index b87d13c253..6cea54268b 100644 --- a/packages/ocap-kernel/src/store/methods/refcount.ts +++ b/packages/ocap-kernel/src/store/methods/refcount.ts @@ -71,6 +71,11 @@ export function getRefCountMethods(ctx: StoreContext) { * have only a "reachable" count, whereas objects track both "reachable" * and "recognizable" counts. * + * Every rule below has a mirror in `computeExpectedRefCounts` + * (`refcount-audit.ts`), which recomputes these counts from the references + * themselves; the two have to change together or the audit starts reporting + * violations against correct accounting. + * * @param kref - The kernel slot whose refcount is to be incremented. * @param tag - The tag of the kernel slot. * @param options - Options for the increment. diff --git a/packages/ocap-kernel/src/store/methods/translators.test.ts b/packages/ocap-kernel/src/store/methods/translators.test.ts index fd1b5e1df6..b50f21b4a0 100644 --- a/packages/ocap-kernel/src/store/methods/translators.test.ts +++ b/packages/ocap-kernel/src/store/methods/translators.test.ts @@ -15,6 +15,7 @@ import type { import type { StoreContext } from '../types.ts'; import * as clistModule from './clist.ts'; import * as reachableModule from './reachable.ts'; +import * as refCountModule from './refcount.ts'; import { getTranslators } from './translators.ts'; import * as vatModule from './vat.ts'; @@ -24,10 +25,12 @@ describe('getTranslators', () => { const mockAllocateErefForKref = vi.fn(); const mockExportFromEndpoint = vi.fn(); const mockSetReachableFlag = vi.fn(); + const mockKernelRefExists = vi.fn(); const mockCtx = {} as StoreContext; beforeEach(() => { vi.clearAllMocks(); + mockKernelRefExists.mockReturnValue(true); vi.spyOn(clistModule, 'getCListMethods').mockReturnValue({ krefToEref: mockKrefToEref, @@ -39,6 +42,10 @@ describe('getTranslators', () => { setReachableFlag: mockSetReachableFlag, } as unknown as ReturnType); + vi.spyOn(refCountModule, 'getRefCountMethods').mockReturnValue({ + kernelRefExists: mockKernelRefExists, + } as unknown as ReturnType); + vi.spyOn(vatModule, 'getVatMethods').mockReturnValue({ exportFromEndpoint: mockExportFromEndpoint, } as unknown as ReturnType); @@ -70,6 +77,18 @@ describe('getTranslators', () => { expect(result).toStrictEqual(expectedEref); }); + it('refuses to import a kref the kernel has deleted', () => { + const vatId: VatId = 'v1'; + const kref: KRef = 'ko1' as KRef; + mockKrefToEref.mockReturnValue(null); + mockKernelRefExists.mockReturnValue(false); + const { translateRefKtoE } = getTranslators(mockCtx); + expect(() => translateRefKtoE(vatId, kref, true)).toThrow( + `cannot import deleted kref "${kref}" into "${vatId}"`, + ); + expect(mockAllocateErefForKref).not.toHaveBeenCalled(); + }); + it('throws error when not found and importIfNeeded is false', () => { const vatId: VatId = 'v1'; const kref: KRef = 'ko1' as KRef; diff --git a/packages/ocap-kernel/src/store/methods/translators.ts b/packages/ocap-kernel/src/store/methods/translators.ts index b7d418a1c3..3ba551d28f 100644 --- a/packages/ocap-kernel/src/store/methods/translators.ts +++ b/packages/ocap-kernel/src/store/methods/translators.ts @@ -22,6 +22,7 @@ import type { import type { StoreContext } from '../types.ts'; import { getCListMethods } from './clist.ts'; import { getReachableMethods } from './reachable.ts'; +import { getRefCountMethods } from './refcount.ts'; import { getVatMethods } from './vat.ts'; import { Fail, assert } from '../../utils/assert.ts'; @@ -37,6 +38,7 @@ import { Fail, assert } from '../../utils/assert.ts'; export function getTranslators(ctx: StoreContext) { const { krefToEref, erefToKref, allocateErefForKref } = getCListMethods(ctx); const { setReachableFlag } = getReachableMethods(ctx); + const { kernelRefExists } = getRefCountMethods(ctx); const { exportFromEndpoint } = getVatMethods(ctx); /** @@ -76,6 +78,15 @@ export function getTranslators(ctx: StoreContext) { let eref = krefToEref(endpointId, kref); if (!eref) { if (importIfNeeded) { + // A kref the kernel has already deleted must not acquire a new c-list + // entry. `getObjectRefCount` reads a missing row as `(0, 0)`, so the + // entry's own increment would write it back and resurrect a + // live-looking object that nobody owns — one the audit then endorses, + // since the entry is a legitimate holder for exactly the count it + // finds. Reached by redeeming an ocap URL issued for an object that + // has since been collected. + kernelRefExists(kref) || + Fail`cannot import deleted kref ${kref} into ${endpointId}`; eref = allocateErefForKref(endpointId, kref); } else { throw Fail`unmapped kref ${kref} endpoint=${endpointId}`; diff --git a/packages/ocap-kernel/src/store/methods/vat.test.ts b/packages/ocap-kernel/src/store/methods/vat.test.ts index e2a3914f6e..fd1f05402e 100644 --- a/packages/ocap-kernel/src/store/methods/vat.test.ts +++ b/packages/ocap-kernel/src/store/methods/vat.test.ts @@ -600,6 +600,10 @@ describe('vat store methods', () => { expect(mockKV.has(`slot.${endpointId}.ko42`)).toBe(false); expect(mockKV.has(`slot.${endpointId}.ro+7`)).toBe(false); expect(mockDecrementRefCount).not.toHaveBeenCalled(); + // Queued for GC even though the new owner keeps the kref: tearing our + // pair down may have been what took its last reference, and the new + // owner's accounting decides the outcome. + expect(mockMaybeFreeKrefs.add).toHaveBeenCalledWith('ko42'); }); it('preserves our exports to the peer (import-direction entries)', () => { diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index bea731738c..23af0e0402 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -83,6 +83,7 @@ describe('SubclusterManager', () => { collectGarbage: vi.fn(), terminateAllVats: vi.fn().mockResolvedValue(undefined), hasVat: vi.fn().mockReturnValue(false), + releaseVatRootPin: vi.fn(), } as unknown as Mocked; mockGetKernelService = vi.fn().mockReturnValue(undefined) as unknown as ( @@ -768,6 +769,11 @@ describe('SubclusterManager', () => { expect( mockKernelStore.deleteSystemSubclusterMapping, ).toHaveBeenCalledWith('orphan'); + // These vats never ran here, so nothing else releases the root pin the + // incarnation that did run them took. + for (const vatId of Object.values(subcluster.vats)) { + expect(mockVatManager.releaseVatRootPin).toHaveBeenCalledWith(vatId); + } }); it('restores valid persisted system subclusters', () => { diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index 2686fda5f3..1b21b6c26a 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -295,6 +295,11 @@ export class SubclusterManager { // Delete vat configs and mark vats as terminated so their data will be cleaned up for (const vatId of Object.values(subcluster.vats)) { + // These vats are not running, so `stopVat` never gets to release the pin + // its `launchVat` took in the incarnation that did run them. Without this + // the root's count never reaches zero and `pinnedObjects` keeps naming a + // vat that no longer exists. + this.#vatManager.releaseVatRootPin(vatId); this.#kernelStore.deleteVatConfig(vatId); this.#kernelStore.markVatAsTerminated(vatId); } diff --git a/packages/ocap-kernel/src/vats/VatManager.test.ts b/packages/ocap-kernel/src/vats/VatManager.test.ts index d5e92b1aac..21361f942c 100644 --- a/packages/ocap-kernel/src/vats/VatManager.test.ts +++ b/packages/ocap-kernel/src/vats/VatManager.test.ts @@ -160,6 +160,14 @@ describe('VatManager', () => { expect(kref).toBe('ko1'); }); + it('pins the root for the vat lifetime', async () => { + // A root is addressable while its vat lives whether or not anyone + // imports it, so without this GC retires it as the last importer lets go. + await vatManager.launchVat(createMockVatConfig(), 'test'); + + expect(mockKernelStore.pinObject).toHaveBeenCalledWith('ko1'); + }); + it('launches a new vat with subcluster', async () => { const config = createMockVatConfig(); const kref = await vatManager.launchVat(config, 'test', 's1'); @@ -236,6 +244,24 @@ describe('VatManager', () => { expect(vatManager.hasVat('v1')).toBe(false); }); + it('keeps the root pin across a restart', async () => { + await vatManager.runVat('v1', createMockVatConfig()); + + await vatManager.stopVat('v1', false); + + // The same root comes back, so releasing the pin would let GC retire it + // in the window where the vat has no handle. + expect(mockKernelStore.unpinObject).not.toHaveBeenCalled(); + }); + + it('releases the root pin on termination', async () => { + await vatManager.runVat('v1', createMockVatConfig()); + + await vatManager.stopVat('v1', true); + + expect(mockKernelStore.unpinObject).toHaveBeenCalledWith('ko1'); + }); + it('stops a vat for termination with reason', async () => { const config = createMockVatConfig(); await vatManager.runVat('v1', config); @@ -429,6 +455,25 @@ describe('VatManager', () => { }); }); + describe('releaseVatRootPin', () => { + it('releases the pin on a vat root', async () => { + await vatManager.runVat('v1', createMockVatConfig()); + + vatManager.releaseVatRootPin('v1'); + + expect(mockKernelStore.unpinObject).toHaveBeenCalledWith('ko1'); + }); + + it('does nothing for a vat with no root', () => { + // Teardown can outlive the kernel's knowledge of the vat, and there is + // no pin to release in that case. + mockKernelStore.getRootObject.mockReturnValue(undefined); + + expect(() => vatManager.releaseVatRootPin('v1')).not.toThrow(); + expect(mockKernelStore.unpinObject).not.toHaveBeenCalled(); + }); + }); + describe('pinVatRoot', () => { it('pins vat root successfully', async () => { const config = createMockVatConfig(); diff --git a/packages/ocap-kernel/src/vats/VatManager.ts b/packages/ocap-kernel/src/vats/VatManager.ts index 314bfa87fb..5da2bebe2d 100644 --- a/packages/ocap-kernel/src/vats/VatManager.ts +++ b/packages/ocap-kernel/src/vats/VatManager.ts @@ -192,12 +192,8 @@ export class VatManager { terminationError = new VatDeletedError(vatId); } if (terminating) { - // Release the pin `launchVat` took, so the root can be collected once - // its importers let go. A restart keeps it: the same root comes back. - const rootRef = this.#kernelStore.getRootObject(vatId); - if (rootRef) { - this.#kernelStore.unpinObject(rootRef); - } + // A restart keeps the pin: the same root comes back. + this.releaseVatRootPin(vatId); } await this.#platformServices .terminate(vatId, terminationError) @@ -299,7 +295,26 @@ export class VatManager { } /** - * Pin a vat root. + * Release the pin `launchVat` took on a vat's root, so the root can be + * collected once its importers let go. + * + * For paths that end a vat's life. Tolerant of a root that is already gone, + * since a vat can be torn down after the kernel has lost track of it. + * + * @param vatId - The ID of the vat whose life is ending. + */ + releaseVatRootPin(vatId: VatId): void { + const rootRef = this.#kernelStore.getRootObject(vatId); + if (rootRef) { + this.#kernelStore.unpinObject(rootRef); + } + } + + /** + * Pin a vat root, on behalf of an embedder that wants to keep it addressable. + * + * Pins are counted, and `launchVat` already holds one for the vat's lifetime, + * so this adds to that rather than replacing it. * * @param vatId - The ID of the vat. * @returns The KRef of the vat root. @@ -314,7 +329,10 @@ export class VatManager { } /** - * Unpin a vat root. + * Release one embedder pin on a vat root. + * + * Removes a single pin, so a root still pinned for its vat's lifetime stays + * addressable: this does not make it collectable while the vat lives. * * @param vatId - The ID of the vat. */ diff --git a/packages/ocap-kernel/test/remotes-mocks.ts b/packages/ocap-kernel/test/remotes-mocks.ts index 0e37b4a77c..3d3e332e13 100644 --- a/packages/ocap-kernel/test/remotes-mocks.ts +++ b/packages/ocap-kernel/test/remotes-mocks.ts @@ -164,6 +164,7 @@ export class MockRemotesFactory { }, remoteComms, remoteHandle, + kernelStore: this.config.kernelStore as KernelStore, }; } From 051d772dd0310b125db5f75b5e15259fbfeca70c Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 14 Aug 2026 16:47:19 +0200 Subject: [PATCH 05/34] fix(ocap-kernel): release the ocap URL retention a failed mint took Retaining before minting is right: minting awaits, so a collection crank can run in that window. But nothing undid the retention when minting then failed. A rejected kernel-service call is reported to the caller rather than thrown out of the crank, so the crank commits and the pin outlives the kernel that took it, naming a URL that never existed. retainForOcapURL now reports whether this call took the pin, and undoOcapURLRetention unwinds one that never backed a URL. Guarded on the ledger rather than the pin list, so it can only remove the pin it put there: a kref some live URL already names keeps the pin that URL depends on, and a vat root keeps its lifetime pin. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 1 + .../src/remotes/kernel/OcapURLManager.test.ts | 33 +++++++++++ .../src/remotes/kernel/OcapURLManager.ts | 19 ++++++- packages/ocap-kernel/src/store/index.test.ts | 57 +++++++++++++++++++ packages/ocap-kernel/src/store/index.ts | 20 ++++++- 5 files changed, 124 insertions(+), 6 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 1079dd9db7..c48411a0ff 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -92,6 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A URL carries its kref inside an encrypted bearer token and nothing else, so the kernel cannot see from its own state that a holder exists. Under the old `(1, 1)` birth baseline nothing exported was ever collectable and this went unnoticed; at `(0, 0)` the target is collected as soon as the message that carried it to the issuer is delivered, and the URL names a dead capability - One pin per kref however many URLs name it, and no release: the token is persistent and unexpiring, so `revoke` is the way to kill the capability - Issuing a URL for a kref the kernel has already deleted is now refused rather than resurrecting its counts + - The retention is taken before the token is minted, since minting awaits and a collection crank can run in that window; it is released again if minting fails, and only when that call was the one that took it - Refuse to import a kref the kernel has deleted into an endpoint's c-list ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - `getObjectRefCount` reads a missing entry as `(0, 0)`, so the new entry's own increment wrote it back and resurrected a live-looking object with no owner — deliverable to by nobody, and endorsed by the audit, since the entry is a legitimate holder for exactly the count it finds - Release a vat's root pin when a subcluster is deleted without its vats having run ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts index 8c59ab9389..5bba9896a0 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts @@ -130,6 +130,39 @@ describe('OcapURLManager', () => { }); }); + it('releases the retention when minting the URL fails', async () => { + // No URL exists to depend on the pin, and nothing else would ever release + // it: the rejection is reported to the caller, not thrown out of a crank. + vi.spyOn(mockRemoteComms, 'issueOcapURL').mockRejectedValue( + new Error('Issue failed'), + ); + + await expect(ocapURLManager.issueOcapURL(objectKRef)).rejects.toThrow( + 'Issue failed', + ); + + expect(mockKernelStore.isObjectPinned(objectKRef)).toBe(false); + expect(mockKernelStore.getOcapURLObjects()).toStrictEqual([]); + expect(mockKernelStore.getObjectRefCount(objectKRef)).toStrictEqual({ + reachable: 0, + recognizable: 0, + }); + }); + + it('keeps the retention a URL already issued depends on', async () => { + await ocapURLManager.issueOcapURL(objectKRef); + vi.spyOn(mockRemoteComms, 'issueOcapURL').mockRejectedValue( + new Error('Issue failed'), + ); + + await expect(ocapURLManager.issueOcapURL(objectKRef)).rejects.toThrow( + 'Issue failed', + ); + + expect(mockKernelStore.isObjectPinned(objectKRef)).toBe(true); + expect(mockKernelStore.getOcapURLObjects()).toStrictEqual([objectKRef]); + }); + it('refuses to issue a URL for a kref the kernel has deleted', async () => { mockKernelStore.deleteKernelObject(objectKRef); diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts index 2a2b97f3cc..aee9b1710f 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts @@ -127,9 +127,22 @@ export class OcapURLManager { async issueOcapURL(kref: KRef): Promise { const identity = this.#remoteManager.getRemoteIdentity(); // Before minting the token, not after: the URL is unretractable once it - // exists, so the target must already be retained. See `retainForOcapURL`. - this.#kernelStore.retainForOcapURL(kref); - return identity.issueOcapURL(kref); + // exists, so the target must already be retained. Minting also awaits, and + // a collection crank can run in that window. See `retainForOcapURL`. + const retained = this.#kernelStore.retainForOcapURL(kref); + try { + return await identity.issueOcapURL(kref); + } catch (error) { + // Nothing else undoes this. A rejected kernel-service call is reported to + // the caller rather than thrown out of the crank, so the crank commits + // and the pin outlives the kernel that took it, naming a URL that never + // existed. Only the pin this call took: a kref some live URL already + // names keeps the pin that URL depends on. + if (retained) { + this.#kernelStore.undoOcapURLRetention(kref); + } + throw error; + } } /** diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index 83b27dea79..af06ace09c 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -192,6 +192,7 @@ describe('kernel store', () => { 'translateRefEtoK', 'translateRefKtoE', 'translateSyscallVtoK', + 'undoOcapURLRetention', 'unpinObject', 'waitForCrank', ]); @@ -366,6 +367,62 @@ describe('kernel store', () => { }); }); + describe('ocap URL retention', () => { + it('pins the target and records it', () => { + const ks = makeKernelStore(mockKernelDatabase); + const kref = ks.initKernelObject('v1'); + + expect(ks.retainForOcapURL(kref)).toBe(true); + expect(ks.isObjectPinned(kref)).toBe(true); + expect(ks.getOcapURLObjects()).toStrictEqual([kref]); + }); + + it('reports that a second URL for the same target took no pin', () => { + const ks = makeKernelStore(mockKernelDatabase); + const kref = ks.initKernelObject('v1'); + ks.retainForOcapURL(kref); + + expect(ks.retainForOcapURL(kref)).toBe(false); + expect(ks.getPinnedObjects()).toStrictEqual([kref]); + }); + + it('refuses to retain a kref the kernel has deleted', () => { + const ks = makeKernelStore(mockKernelDatabase); + + expect(() => ks.retainForOcapURL('ko99')).toThrow( + 'cannot issue an ocap URL for deleted kref "ko99"', + ); + }); + + it('undoing the last retention clears the record entirely', () => { + const ks = makeKernelStore(mockKernelDatabase); + const kref = ks.initKernelObject('v1'); + const otherKref = ks.initKernelObject('v1'); + ks.retainForOcapURL(kref); + ks.retainForOcapURL(otherKref); + + ks.undoOcapURLRetention(kref); + expect(ks.isObjectPinned(kref)).toBe(false); + expect(ks.getOcapURLObjects()).toStrictEqual([otherKref]); + + ks.undoOcapURLRetention(otherKref); + expect(ks.getOcapURLObjects()).toStrictEqual([]); + expect( + mockKernelDatabase.kernelKVStore.get('ocapURLObjects'), + ).toBeUndefined(); + }); + + it('undoing a retention that was never taken leaves the pin alone', () => { + const ks = makeKernelStore(mockKernelDatabase); + const kref = ks.initKernelObject('v1'); + ks.pinObject(kref); + + ks.undoOcapURLRetention(kref); + + expect(ks.isObjectPinned(kref)).toBe(true); + }); + }); + describe('reset', () => { it('clears store and resets counters', () => { const ks = makeKernelStore(mockKernelDatabase); diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 00a8d591f2..6ba7c51775 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -378,12 +378,13 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { // the URL redeemable, and it has to outlive every other reference — the // token is persistent, unexpiring, and may be redeemed by a peer that was // not running when it was issued. `revoke` is the way to kill the - // capability; there is deliberately no release here. + // capability; `undoOcapURLRetention` is not a release, only an unwind of a + // retention whose URL was never minted. getOcapURLObjects(): KRef[] { const raw = kv.get('ocapURLObjects'); return raw ? (raw.split(',') as KRef[]) : []; }, - retainForOcapURL(kref: KRef): void { + retainForOcapURL(kref: KRef): boolean { // Refuse to mint a token for something already collected: the pin would // resurrect a `(1, 1)` row for an object with no owner, and the URL would // name a capability that can never be delivered to. @@ -393,11 +394,24 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { // One pin per kref, however many URLs name it: pins are a multiset, and // a second pin here would be one nothing could ever release. if (krefs.has(kref)) { - return; + return false; } krefs.add(kref); kv.set('ocapURLObjects', [...krefs].sort().join(',')); this.pinObject(kref); + return true; + }, + undoOcapURLRetention(kref: KRef): void { + const krefs = new Set(this.getOcapURLObjects()); + if (!krefs.delete(kref)) { + return; + } + if (krefs.size === 0) { + kv.delete('ocapURLObjects'); + } else { + kv.set('ocapURLObjects', [...krefs].sort().join(',')); + } + this.unpinObject(kref); }, }); } From ee9c894fa398fc53c91b671da27731da61f56722 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 17 Aug 2026 11:45:09 +0200 Subject: [PATCH 06/34] fix(ocap-kernel): take an ocap URL retention per issuance, not per kref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retention was deduplicated by kref, and the failure path undid it if this call was the one that took it. Minting awaits, though, so issuances for the same target overlap: a second `issue` can mint a URL while the first is still in flight, having taken no retention of its own because the ledger already named the kref. If the first then fails it unwinds the retention the second's live URL depends on, and collection can take the capability out from under it. The ledger is a multiset now, one entry and one pin per issuance, so a failed mint releases only what it took. Pins were already a multiset, and each pin here is either released by its own failure or held by its own live URL, so none is left unreleasable — the concern that motivated deduplicating. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 4 +-- .../src/remotes/kernel/OcapURLManager.test.ts | 35 ++++++++++++++++--- .../src/remotes/kernel/OcapURLManager.ts | 11 +++--- packages/ocap-kernel/src/store/index.test.ts | 31 +++++++++++++--- packages/ocap-kernel/src/store/index.ts | 31 ++++++++-------- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index c48411a0ff..1cab9b17ef 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -90,9 +90,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `getPromisesByDecider` matched nothing, so promises a terminating vat or restarting peer was deciding were never rejected - Issuing an ocap URL now retains its target, so the URL stays redeemable ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - A URL carries its kref inside an encrypted bearer token and nothing else, so the kernel cannot see from its own state that a holder exists. Under the old `(1, 1)` birth baseline nothing exported was ever collectable and this went unnoticed; at `(0, 0)` the target is collected as soon as the message that carried it to the issuer is delivered, and the URL names a dead capability - - One pin per kref however many URLs name it, and no release: the token is persistent and unexpiring, so `revoke` is the way to kill the capability + - One pin per URL, and no release: the token is persistent and unexpiring, so `revoke` is the way to kill the capability - Issuing a URL for a kref the kernel has already deleted is now refused rather than resurrecting its counts - - The retention is taken before the token is minted, since minting awaits and a collection crank can run in that window; it is released again if minting fails, and only when that call was the one that took it + - The retention is taken before the token is minted, since minting awaits and a collection crank can run in that window, and released again if minting fails. It is a retention per issuance rather than per kref because that window lets issuances for the same target overlap: sharing one would let a failed mint release the retention a URL minted alongside it depends on - Refuse to import a kref the kernel has deleted into an endpoint's c-list ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - `getObjectRefCount` reads a missing entry as `(0, 0)`, so the new entry's own increment wrote it back and resurrected a live-looking object with no owner — deliverable to by nobody, and endorsed by the audit, since the entry is a legitimate holder for exactly the count it finds - Release a vat's root pin when a subcluster is deleted without its vats having run ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts index 5bba9896a0..79bd75222a 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.test.ts @@ -1,3 +1,4 @@ +import { makePromiseKit } from '@endo/promise-kit'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import type { Mock } from 'vitest'; @@ -119,14 +120,17 @@ describe('OcapURLManager', () => { expect(mockKernelStore.kernelRefExists(objectKRef)).toBe(true); }); - it('retains a target named by several URLs only once', async () => { + it('retains a target once per URL naming it', async () => { await ocapURLManager.issueOcapURL(objectKRef); await ocapURLManager.issueOcapURL(objectKRef); - expect(mockKernelStore.getPinnedObjects()).toStrictEqual([objectKRef]); + expect(mockKernelStore.getPinnedObjects()).toStrictEqual([ + objectKRef, + objectKRef, + ]); expect(mockKernelStore.getObjectRefCount(objectKRef)).toStrictEqual({ - reachable: 1, - recognizable: 1, + reachable: 2, + recognizable: 2, }); }); @@ -163,6 +167,29 @@ describe('OcapURLManager', () => { expect(mockKernelStore.getOcapURLObjects()).toStrictEqual([objectKRef]); }); + it('keeps the retention of a URL minted while a failing issuance was in flight', async () => { + // Minting awaits, so a second issuance for the same target can run to + // completion inside the first's window. The first's failure may only + // release its own retention, not the one the second's live URL needs. + const firstMint = makePromiseKit(); + vi.spyOn(mockRemoteComms, 'issueOcapURL') + .mockImplementationOnce(async () => await firstMint.promise) + .mockImplementationOnce(async () => 'ocap:def456@local-peer-id'); + + const failing = ocapURLManager.issueOcapURL(objectKRef); + const url = await ocapURLManager.issueOcapURL(objectKRef); + firstMint.reject(new Error('Issue failed')); + + await expect(failing).rejects.toThrow('Issue failed'); + expect(url).toBe('ocap:def456@local-peer-id'); + expect(mockKernelStore.isObjectPinned(objectKRef)).toBe(true); + expect(mockKernelStore.getOcapURLObjects()).toStrictEqual([objectKRef]); + expect(mockKernelStore.getObjectRefCount(objectKRef)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + it('refuses to issue a URL for a kref the kernel has deleted', async () => { mockKernelStore.deleteKernelObject(objectKRef); diff --git a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts index aee9b1710f..8575bbe33d 100644 --- a/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/OcapURLManager.ts @@ -129,18 +129,17 @@ export class OcapURLManager { // Before minting the token, not after: the URL is unretractable once it // exists, so the target must already be retained. Minting also awaits, and // a collection crank can run in that window. See `retainForOcapURL`. - const retained = this.#kernelStore.retainForOcapURL(kref); + this.#kernelStore.retainForOcapURL(kref); try { return await identity.issueOcapURL(kref); } catch (error) { // Nothing else undoes this. A rejected kernel-service call is reported to // the caller rather than thrown out of the crank, so the crank commits // and the pin outlives the kernel that took it, naming a URL that never - // existed. Only the pin this call took: a kref some live URL already - // names keeps the pin that URL depends on. - if (retained) { - this.#kernelStore.undoOcapURLRetention(kref); - } + // existed. This unwinds exactly the one retention above took, so the + // retentions of URLs that were minted — including any issued for this + // same kref while this one was in flight — are left alone. + this.#kernelStore.undoOcapURLRetention(kref); throw error; } } diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index af06ace09c..0344da12d4 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -372,18 +372,41 @@ describe('kernel store', () => { const ks = makeKernelStore(mockKernelDatabase); const kref = ks.initKernelObject('v1'); - expect(ks.retainForOcapURL(kref)).toBe(true); + ks.retainForOcapURL(kref); + expect(ks.isObjectPinned(kref)).toBe(true); expect(ks.getOcapURLObjects()).toStrictEqual([kref]); }); - it('reports that a second URL for the same target took no pin', () => { + it('takes a retention of its own for each URL naming the same target', () => { const ks = makeKernelStore(mockKernelDatabase); const kref = ks.initKernelObject('v1'); + + ks.retainForOcapURL(kref); ks.retainForOcapURL(kref); - expect(ks.retainForOcapURL(kref)).toBe(false); - expect(ks.getPinnedObjects()).toStrictEqual([kref]); + expect(ks.getOcapURLObjects()).toStrictEqual([kref, kref]); + expect(ks.getPinnedObjects()).toStrictEqual([kref, kref]); + expect(ks.getObjectRefCount(kref)).toStrictEqual({ + reachable: 2, + recognizable: 2, + }); + }); + + it('undoes one retention of a target that several URLs name', () => { + const ks = makeKernelStore(mockKernelDatabase); + const kref = ks.initKernelObject('v1'); + ks.retainForOcapURL(kref); + ks.retainForOcapURL(kref); + + ks.undoOcapURLRetention(kref); + + expect(ks.getOcapURLObjects()).toStrictEqual([kref]); + expect(ks.isObjectPinned(kref)).toBe(true); + expect(ks.getObjectRefCount(kref)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); }); it('refuses to retain a kref the kernel has deleted', () => { diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 6ba7c51775..8e082aa8b1 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -380,36 +380,39 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { // not running when it was issued. `revoke` is the way to kill the // capability; `undoOcapURLRetention` is not a release, only an unwind of a // retention whose URL was never minted. + // + // The ledger is a multiset, one entry and one pin per issuance, because + // issuances for the same kref overlap: minting awaits, so a second `issue` + // can run to completion while the first is still in flight. Deduplicating + // by kref would leave the second issuance holding no retention of its own + // and the first free to unwind, on failure, the one the second's live URL + // depends on. getOcapURLObjects(): KRef[] { const raw = kv.get('ocapURLObjects'); return raw ? (raw.split(',') as KRef[]) : []; }, - retainForOcapURL(kref: KRef): boolean { + retainForOcapURL(kref: KRef): void { // Refuse to mint a token for something already collected: the pin would // resurrect a `(1, 1)` row for an object with no owner, and the URL would // name a capability that can never be delivered to. this.kernelRefExists(kref) || Fail`cannot issue an ocap URL for deleted kref ${kref}`; - const krefs = new Set(this.getOcapURLObjects()); - // One pin per kref, however many URLs name it: pins are a multiset, and - // a second pin here would be one nothing could ever release. - if (krefs.has(kref)) { - return false; - } - krefs.add(kref); - kv.set('ocapURLObjects', [...krefs].sort().join(',')); + const krefs = this.getOcapURLObjects(); + krefs.push(kref); + kv.set('ocapURLObjects', krefs.sort().join(',')); this.pinObject(kref); - return true; }, undoOcapURLRetention(kref: KRef): void { - const krefs = new Set(this.getOcapURLObjects()); - if (!krefs.delete(kref)) { + const krefs = this.getOcapURLObjects(); + const index = krefs.indexOf(kref); + if (index === -1) { return; } - if (krefs.size === 0) { + krefs.splice(index, 1); + if (krefs.length === 0) { kv.delete('ocapURLObjects'); } else { - kv.set('ocapURLObjects', [...krefs].sort().join(',')); + kv.set('ocapURLObjects', krefs.join(',')); } this.unpinObject(kref); }, From e0573b7e32ac0f275ebff8534afc02bf71ea97a0 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:17:49 -0400 Subject: [PATCH 07/34] test: pin the transaction invariants #1005 left broken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight tests, all currently failing, for three defects that landed with #1005. They change no production code: each one states the invariant the fix has to restore, so the diff that repairs them is the specification being met rather than a claim about it. `releaseSavepoint` was never hardened the way `rollbackSavepoint` was in that PR. A RELEASE that throws leaves the savepoint on the stack and the transaction open with nothing that will ever commit or abort it, so every later write on the connection joins it, reports success, and vanishes on close — verbatim the failure mode #1005 documents for the other door. The driver tests sit beside their rollback counterparts so the asymmetry is visible in place. `endCrank` gets the companion case: it now settles its waiters in a `finally`, which is right, but it also leaves the savepoint listed, so the next crank numbers its savepoint `t1` against a database that still has `t0`. `#processCrankResult` does fallible work after the crank's transactional boundary has already been crossed. On the success path `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external caller, and only then can `#terminateVat` throw and have the new catch roll the crank back — so the caller keeps an answer computed from state the store discarded, and a restart delivers the message again. On the abort path the rollback ends the transaction, so `#terminateVat` and `collectGarbage` autocommit piecemeal and the second rollback the flag correctly suppresses would have had nothing left to undo either way. The invariant is stated as "the rollback is the last thing the crank asks of the store", which leaves the choice of remedy open. The wasm driver tracks `_inTx` itself rather than reading it from SQLite, so a failed abort inside the new catch is the one case that can leave it disagreeing with the database. Left true, `beginIfNeeded` is a no-op from then on and the next `createSavepoint` runs in autocommit mode, where the matching RELEASE commits (Agoric/agoric-sdk#8423, already cited two lines above the code) and no rollback can undo the delivery. The second test runs that next `createSavepoint` and asserts the BEGIN, so the corruption path is observable instead of argued. Co-Authored-By: Claude Opus 5 --- .../kernel-store/src/sqlite/nodejs.test.ts | 23 ++++ packages/kernel-store/src/sqlite/wasm.test.ts | 69 ++++++++++++ packages/ocap-kernel/src/KernelQueue.test.ts | 106 ++++++++++++++++++ .../src/store/methods/crank.test.ts | 17 +++ 4 files changed, 215 insertions(+) diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index a62392fe19..3bf0db5f68 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -360,6 +360,29 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._spStack).toStrictEqual([]); }); + // The same hazard `rollbackSavepoint` guards against, by the other door: a + // RELEASE that throws leaves the savepoint on the stack and the transaction + // open with nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. + it('releaseSavepoint discards the transaction when the release fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['point1']; + mockStatement.run.mockClear(); + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + // The abort is the only prepared statement this path runs. + expect(mockStatement.run).toHaveBeenCalledOnce(); + mockDb.inTransaction = false; + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 2cbc96d658..e0e6be1c45 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -518,6 +518,75 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); + // The same hazard `rollbackSavepoint` guards against, by the other door: a + // RELEASE that throws leaves the savepoint on the stack and the transaction + // open with nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. + it('releaseSavepoint discards the transaction when the release fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + expect(mockDb._inTx).toBe(false); + }); + + // `_inTx` is tracked here rather than read from SQLite, so a failed abort is + // the one case that can leave it disagreeing with the database. Left true, + // `beginIfNeeded` becomes a no-op forever after. + it('stops believing it is in a transaction when the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.rollbackSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + }); + + // The consequence of the above, and the reason it is worth asserting: a + // savepoint created outside a transaction autocommits when released + // (Agoric/agoric-sdk#8423), so no later rollback can undo the delivery — an + // aborted crank would silently keep its writes. + it('begins a transaction for the next savepoint after a failed abort', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + expect(() => db.rollbackSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + mockDb.exec.mockClear(); + mockStatement.step.mockClear(); + db.createSavepoint('next'); + + // BEGIN is the only prepared statement `createSavepoint` runs; the + // SAVEPOINT itself goes through `exec`. + expect(mockStatement.step).toHaveBeenCalledOnce(); + expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 1b3bd4a35a..0aafea68d5 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -195,6 +195,112 @@ describe('KernelQueue', () => { expect(kernelStore.collectGarbage).toHaveBeenCalled(); expect(kernelStore.endCrank).toHaveBeenCalled(); }); + + // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external + // caller, reading the resolution out of the store on the way. Rolling the + // crank back afterwards un-resolves that promise in the store and restores + // the run queue item, so a restart delivers the message a second time and + // notifies every other subscriber again — while the original caller has + // already been told the first answer. + it('does not roll back a crank whose result the caller already received', async () => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp1' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + // A caller is awaiting this message's result. + const resolve = vi.fn(); + const reject = vi.fn(); + kernelQueue.subscriptions.set('kp1', { resolve, reject }); + + // The crank succeeds, so the flush hands that caller its answer... + ( + kernelStore.flushCrankBuffer as unknown as MockInstance + ).mockReturnValueOnce([ + { type: 'notify', endpointId: 'v1', kpid: 'kp1' }, + ]); + (kernelStore.getKernelPromise as unknown as MockInstance).mockReturnValue( + { + state: 'fulfilled', + value: { body: '"answer"', slots: [] }, + }, + ); + + // ...and only then does the kernel die, in work that runs after the flush. + const terminationError = new Error('vat worker already gone'); + (terminateVat as unknown as MockInstance).mockRejectedValueOnce( + terminationError, + ); + const deliver = vi.fn().mockResolvedValue({ + terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, + }); + + await expect(kernelQueue.run(deliver)).rejects.toBe(terminationError); + + expect(resolve).toHaveBeenCalledWith({ body: '"answer"', slots: [] }); + expect(kernelStore.rollbackCrank).not.toHaveBeenCalled(); + }); + + // `rollbackCrank('start')` rolls back the crank's outermost savepoint, which + // ends the transaction — so anything the crank does to the store afterwards + // autocommits piecemeal and no rollback can reach it. Whatever the ordering, + // the rollback has to be the last thing the crank asks of the store. + it.each([ + { label: 'an abort', crankResult: { abort: true } }, + { + label: 'an abort that also terminates', + crankResult: { + abort: true, + terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, + }, + }, + ])( + 'does no store work after rolling back $label', + async ({ crankResult }) => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp99' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + const storeCalls: string[] = []; + ( + kernelStore.rollbackCrank as unknown as MockInstance + ).mockImplementation(() => { + storeCalls.push('rollbackCrank'); + }); + (terminateVat as unknown as MockInstance).mockImplementation( + async () => { + storeCalls.push('terminateVat'); + }, + ); + ( + kernelStore.collectGarbage as unknown as MockInstance + ).mockImplementation(() => { + storeCalls.push('collectGarbage'); + throw new Error(STOP_RUN_LOOP); + }); + + const deliver = vi.fn().mockResolvedValue(crankResult); + await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); + + expect(storeCalls).toContain('rollbackCrank'); + expect(storeCalls.at(-1)).toBe('rollbackCrank'); + }, + ); }); describe('getRunLoopStatus', () => { diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index db3de86450..8fd6cd1fec 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -206,6 +206,23 @@ describe('crank methods', () => { expect(context.resolveCrank).toBeUndefined(); expect(await waiter).toBeUndefined(); }); + + // What `rollbackCrank` already does in its own `finally`. Settling the crank + // regardless means callers proceed, so a savepoint left listed here has the + // next crank number its savepoint `t1` while the database still has `t0`: + // from then on `releaseAllSavepoints` releases the wrong one and every + // rollback aims past the crank it meant to undo. + it('forgets its savepoints even if releasing them fails', () => { + crankMethods.startCrank(); + context.savepoints = ['test']; + vi.mocked(kdb.releaseSavepoint).mockImplementationOnce(() => { + throw new Error('database is gone'); + }); + + expect(() => crankMethods.endCrank()).toThrow('database is gone'); + + expect(context.savepoints).toStrictEqual([]); + }); }); describe('releaseAllSavepoints', () => { From c8ce039764a0fce7405c2118f74a4c6b43c83d4b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:23:28 +0200 Subject: [PATCH 08/34] fix: keep a crank's store work inside one transaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three transaction-integrity defects, all in the same family: a store call fails, and the layer above goes on as though its bookkeeping still matched the database. - `releaseSavepoint` (both SQLite drivers) discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already does when `ROLLBACK TO` fails. Left as it was, the savepoint stayed on the stack and the transaction open with nothing to ever commit or abort it, so every later write on the connection joined it, reported success, and vanished on `close()`. - `releaseAllSavepoints` forgets its savepoints even if the release throws, as `rollbackCrank` already does. A savepoint left listed had the next crank number its savepoint `t1` while the database still had `t0`, from which point every release and rollback aimed one crank past the one it meant to end. - The wasm driver stops believing it is in a transaction when an abort fails. `_inTx` is tracked in the driver rather than read from SQLite, and an abort usually fails because SQLite already rolled back on its own. Left true, `beginIfNeeded` was a no-op from then on and the next `createSavepoint` ran in autocommit mode, where its `RELEASE` commits (Agoric/agoric-sdk#8423) and no later rollback could undo the delivery. And the crank boundary itself, in two parts: - A crank now takes two savepoints. Rolling back to the outermost one discards the enclosing transaction, so the work an aborted crank still owes — terminating the vat whose delivery failed, collecting garbage — was autocommitting statement by statement, beyond the reach of any later rollback. That work has to follow the rollback, since the worker is gone and the store must not go on believing the vat is alive, so it is the rollback that spares the transaction. Releasing the outer savepoint in `endCrank` is now a crank's one commit point. - `#flushCrankBuffer` runs last, after everything that can still fail. It settles the promise `enqueueMessage` handed an external caller, reading the result out of the store; rolling the crank back after that left the caller holding an answer computed from state the store had discarded, and a restart would deliver the message again. Tests for the first three defects are Ryan's, from #1011. The two crank tests there specify the remedy as "the rollback is the last thing the crank asks of the store", which reordering the fallible work before it would satisfy — but that rollback would then undo the vat termination. They are restated here as the invariant the fix does hold. Co-authored-by: Claude Opus 5 (1M context) --- .../kernel-store/src/sqlite/nodejs.test.ts | 19 ++++ packages/kernel-store/src/sqlite/nodejs.ts | 17 +++- packages/kernel-store/src/sqlite/wasm.test.ts | 19 +++- packages/kernel-store/src/sqlite/wasm.ts | 29 +++++- packages/ocap-kernel/src/KernelQueue.test.ts | 99 ++++++++++++------- packages/ocap-kernel/src/KernelQueue.ts | 49 ++++++--- .../ocap-kernel/src/store/methods/crank.ts | 13 ++- 7 files changed, 189 insertions(+), 56 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index 3bf0db5f68..e38655d9a9 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -383,6 +383,25 @@ describe('makeSQLKernelDatabase', () => { mockDb.inTransaction = false; }); + it('releaseSavepoint reports the release failure even if the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb.inTransaction = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.run.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._spStack).toStrictEqual([]); + mockDb.inTransaction = false; + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index ec863edc7c..d0a6442442 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -321,7 +321,22 @@ export async function makeSQLKernelDatabase({ throw new Error(`No such savepoint: ${name}`); } const query = SQL_QUERIES.RELEASE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // The hazard `rollbackSavepoint` guards against, by the other door: left as + // it was, the savepoint stays on the stack and the transaction open with + // nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. There is no + // committing this transaction now, so discard it. + db._spStack.length = 0; + try { + rollbackIfNeeded(); + } catch { + // The release failure below is the one worth reporting. + } + throw error; + } db._spStack.splice(idx); if (db._spStack.length === 0) { commitIfNeeded(); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index e0e6be1c45..25403caa47 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -488,7 +488,6 @@ describe('makeSQLKernelDatabase', () => { ); expect(mockDb._spStack).toStrictEqual([]); - mockDb._inTx = false; }); it('releaseSavepoint validates savepoint exists', async () => { @@ -538,6 +537,24 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); + it('releaseSavepoint reports the release failure even if the abort fails too', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + mockDb.exec.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + mockStatement.step.mockImplementationOnce(() => { + throw new Error('cannot rollback'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + }); + // `_inTx` is tracked here rather than read from SQLite, so a failed abort is // the one case that can leave it disagreeing with the database. Left true, // `beginIfNeeded` becomes a no-op forever after. diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index c0c32b8a72..beda148117 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -210,10 +210,18 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - sqlAbortTransaction.step(); - sqlAbortTransaction.reset(); + // Out of the transaction as far as this driver is concerned before the + // abort is even attempted. Unlike the nodejs driver, which reads + // `inTransaction` from SQLite, `_inTx` is tracked here — and an abort + // typically fails because SQLite already rolled back on its own as part of + // whatever went wrong. Left true, `beginIfNeeded` is a no-op from then on + // and the next `createSavepoint` runs in autocommit mode, where its + // `RELEASE` commits (see `createSavepoint`) and no later rollback can undo + // the delivery. db._inTx = false; db._spStack.length = 0; + sqlAbortTransaction.step(); + sqlAbortTransaction.reset(); } } @@ -403,7 +411,22 @@ export async function makeSQLKernelDatabase({ throw new Error(`No such savepoint: ${name}`); } const query = SQL_QUERIES.RELEASE_SAVEPOINT.replace('%NAME%', name); - db.exec(query); + try { + db.exec(query); + } catch (error) { + // The hazard `rollbackSavepoint` guards against, by the other door: left as + // it was, the savepoint stays on the stack and the transaction open with + // nothing to ever commit or abort it, so every later write on this + // connection joins it, reports success, and vanishes on close. There is no + // committing this transaction now, so discard it. + db._spStack.length = 0; + try { + rollbackIfNeeded(); + } catch { + // The release failure below is the one worth reporting. + } + throw error; + } db._spStack.splice(idx); if (db._spStack.length === 0) { commitIfNeeded(); diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 0aafea68d5..dd3bd2d6d2 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -92,6 +92,23 @@ describe('KernelQueue', () => { }; }; + /** + * Stop the run loop by failing the *next* crank's start, so that the crank + * under test runs to completion. Throwing from one of a crank's own store calls + * cuts it short, which hides everything the crank does after that call. + */ + const stopAfterOneCrank = (): void => { + let cranks = 0; + (kernelStore.startCrank as unknown as MockInstance).mockImplementation( + () => { + cranks += 1; + if (cranks > 1) { + throw new Error(STOP_RUN_LOOP); + } + }, + ); + }; + /** * Run a single crank whose delivery blows up, killing the run loop. * @@ -128,7 +145,8 @@ describe('KernelQueue', () => { const deliver = vi.fn().mockRejectedValue(deliverError); await expect(kernelQueue.run(deliver)).rejects.toBe(deliverError); expect(kernelStore.startCrank).toHaveBeenCalled(); - expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('start'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('crank'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('delivery'); expect(processGCActionSetSpy).toHaveBeenCalled(); expect(kernelStore.nextReapAction).toHaveBeenCalled(); expect(kernelStore.nextTerminatedVatCleanup).toHaveBeenCalled(); @@ -156,9 +174,9 @@ describe('KernelQueue', () => { }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(kernelStore.startCrank).toHaveBeenCalled(); - expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('start'); + expect(kernelStore.createCrankSavepoint).toHaveBeenCalledWith('delivery'); expect(deliver).toHaveBeenCalledWith(mockItem); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(kernelStore.collectGarbage).toHaveBeenCalled(); expect(kernelStore.endCrank).toHaveBeenCalled(); }); @@ -197,12 +215,13 @@ describe('KernelQueue', () => { }); // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external - // caller, reading the resolution out of the store on the way. Rolling the - // crank back afterwards un-resolves that promise in the store and restores - // the run queue item, so a restart delivers the message a second time and - // notifies every other subscriber again — while the original caller has - // already been told the first answer. - it('does not roll back a crank whose result the caller already received', async () => { + // caller, reading the resolution out of the store on the way. Were the crank + // rolled back after that, the store would un-resolve the promise and restore + // the run queue item, so a restart would deliver the message a second time + // and notify every other subscriber again — while the original caller had + // already been told the first answer. So the flush comes last, after + // everything that could still fail. + it('answers no caller from a crank it then rolls back', async () => { const mockItem: RunQueueItem = { type: 'send', target: 'ko123', @@ -220,7 +239,7 @@ describe('KernelQueue', () => { const reject = vi.fn(); kernelQueue.subscriptions.set('kp1', { resolve, reject }); - // The crank succeeds, so the flush hands that caller its answer... + // The delivery succeeds and its result is there for the flush to hand over... ( kernelStore.flushCrankBuffer as unknown as MockInstance ).mockReturnValueOnce([ @@ -233,7 +252,7 @@ describe('KernelQueue', () => { }, ); - // ...and only then does the kernel die, in work that runs after the flush. + // ...but the crank still has fallible work left, and it dies there. const terminationError = new Error('vat worker already gone'); (terminateVat as unknown as MockInstance).mockRejectedValueOnce( terminationError, @@ -244,26 +263,39 @@ describe('KernelQueue', () => { await expect(kernelQueue.run(deliver)).rejects.toBe(terminationError); - expect(resolve).toHaveBeenCalledWith({ body: '"answer"', slots: [] }); - expect(kernelStore.rollbackCrank).not.toHaveBeenCalled(); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); + expect(resolve).not.toHaveBeenCalled(); + // Told the result will never come, rather than left waiting on a crank the + // store no longer has any record of. + expect(reject).toHaveBeenCalledWith( + expect.objectContaining({ cause: terminationError }), + ); }); - // `rollbackCrank('start')` rolls back the crank's outermost savepoint, which - // ends the transaction — so anything the crank does to the store afterwards - // autocommits piecemeal and no rollback can reach it. Whatever the ordering, - // the rollback has to be the last thing the crank asks of the store. + // Rolling back to the crank's *outermost* savepoint discards the enclosing + // transaction (see `rollbackSavepoint`), which would leave the work an aborted + // crank still owes — terminating the vat whose delivery failed, collecting + // garbage — autocommitting statement by statement, beyond the reach of any + // later rollback. That work has to follow the rollback, since the worker is + // already gone and the store must not go on believing the vat is alive, so it + // is the rollback that has to spare the transaction. it.each([ - { label: 'an abort', crankResult: { abort: true } }, + { + label: 'an abort', + crankResult: { abort: true }, + storeOrder: ['rollbackCrank', 'collectGarbage'], + }, { label: 'an abort that also terminates', crankResult: { abort: true, terminate: { vatId: 'v1', info: { body: '"exit"', slots: [] } }, }, + storeOrder: ['rollbackCrank', 'terminateVat', 'collectGarbage'], }, ])( - 'does no store work after rolling back $label', - async ({ crankResult }) => { + 'keeps the crank transactional after rolling back $label', + async ({ crankResult, storeOrder }) => { const mockItem: RunQueueItem = { type: 'send', target: 'ko123', @@ -297,8 +329,13 @@ describe('KernelQueue', () => { const deliver = vi.fn().mockResolvedValue(crankResult); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(storeCalls).toContain('rollbackCrank'); - expect(storeCalls.at(-1)).toBe('rollbackCrank'); + expect(storeCalls).toStrictEqual(storeOrder); + expect( + ( + kernelStore.createCrankSavepoint as unknown as MockInstance + ).mock.calls.flat(), + ).toStrictEqual(['crank', 'delivery']); + expect(kernelStore.rollbackCrank).not.toHaveBeenCalledWith('crank'); }, ); }); @@ -393,7 +430,7 @@ describe('KernelQueue', () => { await killRunLoop(new Error('crank exploded')); // Without this, endCrank's savepoint release commits the half-finished // crank and the dequeued item is lost. - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); }); it('does not roll back when the savepoint was never created', async () => { @@ -945,7 +982,7 @@ describe('KernelQueue', () => { throw new Error(STOP_RUN_LOOP); }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(rejectSpy).toHaveBeenCalledWith(terminateInfo); expect(kernelQueue.subscriptions.has('kp99')).toBe(false); }); @@ -981,7 +1018,7 @@ describe('KernelQueue', () => { throw new Error(STOP_RUN_LOOP); }); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); - expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('start'); + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); expect(rejectedAfterAbort).toBe(false); expect(resolveSpy).not.toHaveBeenCalled(); expect(subscribedAfterAbort).toBe(true); @@ -1051,11 +1088,7 @@ describe('KernelQueue', () => { mockItem, ); const deliver = vi.fn().mockResolvedValue(undefined); - ( - kernelStore.collectGarbage as unknown as MockInstance - ).mockImplementation(() => { - throw new Error(STOP_RUN_LOOP); - }); + stopAfterOneCrank(); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(rejectSpy).toHaveBeenCalledWith(rejectedValue); expect(resolveSpy).not.toHaveBeenCalled(); @@ -1090,11 +1123,7 @@ describe('KernelQueue', () => { mockItem, ); const deliver = vi.fn().mockResolvedValue(undefined); - ( - kernelStore.collectGarbage as unknown as MockInstance - ).mockImplementation(() => { - throw new Error(STOP_RUN_LOOP); - }); + stopAfterOneCrank(); await expect(kernelQueue.run(deliver)).rejects.toThrow(STOP_RUN_LOOP); expect(resolveSpy).toHaveBeenCalledWith(fulfilledValue); expect(rejectSpy).not.toHaveBeenCalled(); diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index afda8139c7..cd3b7435e3 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -130,7 +130,15 @@ export class KernelQueue { this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; try { - this.#kernelStore.createCrankSavepoint('start'); + // Two savepoints, because the crank's transaction has to outlive the + // delivery's rollback. Rolling back to the outermost savepoint discards + // the enclosing transaction (see `rollbackSavepoint`), and the work an + // aborted crank still owes — terminating the vat whose delivery failed, + // collecting garbage — would then autocommit statement by statement, + // beyond the reach of any later rollback. Only `delivery` is ever rolled + // back; releasing `crank` in `endCrank` is this crank's one commit point. + this.#kernelStore.createCrankSavepoint('crank'); + this.#kernelStore.createCrankSavepoint('delivery'); // The savepoint exists from here on, so a throw can be undone. Without // this, `endCrank`'s savepoint release commits the half-finished crank: @@ -158,7 +166,7 @@ export class KernelQueue { // savepoint" over the real error. if (!this.#crankRollbackAttempted) { try { - this.#kernelStore.rollbackCrank('start'); + this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { // The original failure stays the `cause`, since that is the root // cause an operator needs; the rollback failure is named here. @@ -304,7 +312,7 @@ export class KernelQueue { // For active vats, this allows the message to be retried in a future crank. // For terminated vats, the message will just go splat. try { - this.#kernelStore.rollbackCrank('start'); + this.#kernelStore.rollbackCrank('delivery'); } finally { // Set even when the rollback threw. `rollbackCrank` forgets the // savepoint in its own `finally`, so "attempted" and "the savepoint is @@ -333,17 +341,27 @@ export class KernelQueue { // TODO: Currently all errors terminate the vat, but instead we could // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. - } else { - // Upon on successful crank completion, enqueue buffered vat outputs for delivery. - this.#flushCrankBuffer(); } // Vat termination during delivery is triggered by an illegal syscall - // or by syscall.exit(). + // or by syscall.exit(). Its store writes have to survive the rollback above: + // the worker is already gone, so a store that still believed the vat was + // alive would relaunch it after a restart and redeliver what killed it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); + if (!crankResult?.abort) { + // The crank survived, so hand its buffered outputs on — last, once nothing + // fallible remains. The flush settles the promise `enqueueMessage` gave an + // external caller, reading the result out of the store; were the crank + // rolled back after that, the caller would keep an answer computed from + // state the store discarded, and a restart would deliver the message again. + this.#flushCrankBuffer(); + } + // After the flush, because the audit reads the run queue as ground truth + // while a buffered item's references were already counted when it was + // enqueued: audited mid-flush, every buffered item reads as a leak. this.#kernelStore.assertRefCountsIfAuditing(); } @@ -371,21 +389,24 @@ export class KernelQueue { */ #flushCrankBuffer(): void { const items = this.#kernelStore.flushCrankBuffer(); + const resolved: KRef[] = []; for (const item of items) { this.#enqueueRun(item); if (item.type === 'notify') { - // Invoke kernel subscription callback if any, reading resolution - // data from the (now committed) promise state - this.#invokeKernelSubscription(item.kpid); + resolved.push(item.kpid); } } + // Also promises resolved during this crank that don't have kernel-level + // subscribers (e.g., promises from enqueueMessage). + resolved.push(...this.#resolvedWithKernelSubscription); + this.#resolvedWithKernelSubscription = []; - // Invoke kernel subscriptions for promises resolved during this crank - // that don't have kernel-level subscribers (e.g., promises from enqueueMessage) - for (const kpid of this.#resolvedWithKernelSubscription) { + // Callbacks only once every store write is done. Each hands an external + // caller a result read out of the store, and a write that threw in between + // would have the crank rolled back underneath answers already given. + for (const kpid of resolved) { this.#invokeKernelSubscription(kpid); } - this.#resolvedWithKernelSubscription = []; } /** diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 87d2bc65b8..24957bce4b 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -77,8 +77,17 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { */ function releaseAllSavepoints(): void { if (ctx.savepoints.length > 0) { - kdb.releaseSavepoint('t0'); - ctx.savepoints.length = 0; + try { + kdb.releaseSavepoint('t0'); + } finally { + // Forget the savepoints even if the release failed, as `rollbackCrank` + // does. A failed release discards the whole transaction (see + // `releaseSavepoint`), so the database has no savepoints left either; + // leaving them listed here would have the next crank number its savepoint + // `t1`, and from then on every release and rollback would aim one crank + // past the one it meant to end. + ctx.savepoints.length = 0; + } } } From 58b00b760efdd9aca14d1a5f3a52f019d10ed24a Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:23:44 +0200 Subject: [PATCH 09/34] test(kernel-test): reap until the vat's GC is visible, not three times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `should trigger GC syscalls through bringOutYourDead` scheduled one reap and then ran three cranks. `scheduleReap` dedupes, so that bought one `bringOutYourDead`, not three — and an import is only reported as dropped once the engine has collected the vat's presence and run its finalizer, which the forced GC pass inside `bringOutYourDead` cannot guarantee on the first attempt. When it hadn't, no further reap was ever scheduled and the refcount stayed where it was: `expected 2 to be 1`, as on main in 31081630878. Each attempt now schedules its own reap and stops as soon as the kernel's bookkeeping catches up, so the common case is one crank rather than three. Co-authored-by: Claude Opus 5 (1M context) --- .../src/garbage-collection.test.ts | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 67055414f1..45f21f12d2 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -117,6 +117,28 @@ describe('Garbage Collection', () => { expect(parseReplyBody(useResult.body)).toBe(objectId); }); + /** + * Reap the importer vat until the kernel's bookkeeping catches up with the + * vat's own garbage collection, or the attempts run out. + * + * `bringOutYourDead` can only report an import as dropped once the engine has + * collected the vat's presence and run its finalizer, which forcing a GC pass + * does not guarantee on the first attempt. Reaping once and then cranking + * buys one attempt rather than several, because `scheduleReap` dedupes — so + * each attempt schedules its own reap, and a message to the vat wakes the run + * loop to consume it. + * + * @param settled - Whether the state under test has arrived. + */ + async function reapImporterUntil(settled: () => boolean): Promise { + const isImporter = (vatId: VatId): boolean => vatId === importerVatId; + for (let attempt = 0; attempt < 5 && !settled(); attempt += 1) { + kernel.reapVats(isImporter); + await kernel.queueMessage(importerKRef, 'noop', []); + await waitUntilQuiescent(500); + } + } + it('should trigger GC syscalls through bringOutYourDead', async () => { // Create an object in the exporter vat with a known ID const objectId = 'test-object'; @@ -161,14 +183,10 @@ describe('Garbage Collection', () => { await kernel.queueMessage(importerKRef, 'makeWeak', [objectId]); await waitUntilQuiescent(); - // Schedule reap to trigger bringOutYourDead on next crank - kernel.reapVats((vatId) => vatId === importerVatId); - - // Run 3 cranks to allow bringOutYourDead to be processed - for (let i = 0; i < 3; i++) { - await kernel.queueMessage(importerKRef, 'noop', []); - await waitUntilQuiescent(500); - } + // Reap until the importer reports the drop + await reapImporterUntil( + () => kernelStore.getObjectRefCount(createObjectRef).reachable === 1, + ); // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); @@ -180,13 +198,10 @@ describe('Garbage Collection', () => { await kernel.queueMessage(importerKRef, 'forgetImport', []); await waitUntilQuiescent(); - // Schedule another reap - kernel.reapVats((vatId) => vatId === importerVatId); - - for (let i = 0; i < 3; i++) { - await kernel.queueMessage(importerKRef, 'noop', []); - await waitUntilQuiescent(500); - } + // Reap until the importer reports the retirement + await reapImporterUntil( + () => kernelStore.getObjectRefCount(createObjectRef).recognizable === 1, + ); // Check reference counts after retireImports const afterForgetRefCounts = kernelStore.getObjectRefCount(createObjectRef); From 29e4dbfd6ce98542bf606689a2e5fc7138e21b09 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 16:33:20 +0200 Subject: [PATCH 10/34] test: fix rollback crank test --- .../kernel-test/src/crank-rollback.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index cd2a708aa0..dce56b3227 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -138,6 +138,46 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('second')).toBe('yes'); }); + // An aborted crank rolls its delivery back and then still has work to do — + // terminating the vat whose delivery failed, collecting garbage — whose writes + // have to survive that rollback, since the vat's worker is already gone. + it('keeps the writes a crank makes after rolling its delivery back', async () => { + const { kernelStore, kdb } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + kdb.kernelKVStore.set('delivered', 'yes'); + + kernelStore.rollbackCrank('delivery'); + kdb.kernelKVStore.set('terminated', 'yes'); + kernelStore.endCrank(); + + expect(kdb.kernelKVStore.get('delivered')).toBeUndefined(); + expect(kdb.kernelKVStore.get('terminated')).toBe('yes'); + }); + + // And they survive it *as part of the crank's transaction*, which is why the + // delivery gets a savepoint of its own rather than rolling back the crank's: + // rolling back the outermost savepoint discards the transaction, after which + // those writes would autocommit one statement at a time. Nothing in the run + // loop rolls the crank's own savepoint back — it is the only way from here to + // observe that the writes are still undoable at all. + it('holds those writes in the transaction rather than autocommitting them', async () => { + const { kernelStore, kdb } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + kernelStore.rollbackCrank('delivery'); + kdb.kernelKVStore.set('terminated', 'yes'); + + kernelStore.rollbackCrank('crank'); + kernelStore.endCrank(); + + expect(kdb.kernelKVStore.get('terminated')).toBeUndefined(); + }); + // `createCrankSavepoint` records the name only once the database has the // savepoint. Asking to roll back one that was never created must therefore say // so, rather than releasing someone else's savepoint. From f3b0d4acf40dcbd678adca94ba070030fbb23f49 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:37:12 +0200 Subject: [PATCH 11/34] fix(ocap-kernel): forget every savepoint when a crank rollback fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed `ROLLBACK TO` discards the whole transaction, taking every savepoint with it — not just the one rolled back to. `rollbackCrank` truncated `ctx.savepoints` to the rolled-back ordinal regardless, which was correct while a crank took one savepoint at ordinal 0 and cleared the list, but leaves `['crank']` listed now that the delivery sits at ordinal 1. `endCrank` then releases a `t0` the database no longer has, and throws "No such savepoint: t0" from the run loop's `finally` — replacing the failure that actually killed the kernel, with no `cause`. That is the masking this branch's own error-preservation exists to prevent. Clear the list on the throwing path, truncate to the ordinal only on success. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/store/methods/crank.test.ts | 33 ++++++++++++++++--- .../ocap-kernel/src/store/methods/crank.ts | 28 +++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 8fd6cd1fec..389afd7a03 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -153,6 +153,29 @@ describe('crank methods', () => { expect(mockCrankBuffer).toHaveLength(0); }); + + // A failed rollback discards the whole transaction, so the enclosing + // savepoints are gone from the database too — not just the one rolled back + // to. Truncating to the ordinal would leave `endCrank` releasing a `t0` the + // database no longer has, and that "No such savepoint" would be thrown from + // the run loop's `finally`, over whatever really killed the kernel. + it('forgets every savepoint when the rollback fails', () => { + context.inCrank = true; + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + 'disk I/O error', + ); + + expect(context.savepoints).toStrictEqual([]); + // The release `endCrank` would otherwise attempt, and throw over. + crankMethods.endCrank(); + expect(kdb.releaseSavepoint).not.toHaveBeenCalled(); + }); }); describe('endCrank', () => { @@ -207,11 +230,11 @@ describe('crank methods', () => { expect(await waiter).toBeUndefined(); }); - // What `rollbackCrank` already does in its own `finally`. Settling the crank - // regardless means callers proceed, so a savepoint left listed here has the - // next crank number its savepoint `t1` while the database still has `t0`: - // from then on `releaseAllSavepoints` releases the wrong one and every - // rollback aims past the crank it meant to undo. + // What `rollbackCrank` already does when its own rollback fails. Settling + // the crank regardless means callers proceed, so a savepoint left listed + // here has the next crank number its savepoint `t1` while the database still + // has `t0`: from then on `releaseAllSavepoints` releases the wrong one and + // every rollback aims past the crank it meant to undo. it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); context.savepoints = ['test']; diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 24957bce4b..b26219cbb6 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -51,13 +51,19 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { if (ctx.savepoints[ordinal] === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); - } finally { - // Forget the savepoint even if the rollback failed. Leaving it listed - // would have `endCrank`'s release commit the crank we just abandoned — - // the half-finished state this rollback exists to discard. A failed - // rollback discards the whole transaction instead (see - // `rollbackSavepoint`), which for a crank is the same boundary. + // Forget the savepoint. Leaving it listed would have `endCrank`'s + // release commit the crank we just abandoned — the half-finished state + // this rollback exists to discard. ctx.savepoints.length = ordinal; + } catch (error) { + // A failed rollback discards the whole transaction (see + // `rollbackSavepoint`), taking every savepoint with it, not just this + // one. Truncating to `ordinal` would leave the enclosing savepoints + // listed against a database that no longer has them, and `endCrank` + // would then throw "No such savepoint: t0" from the run loop's + // `finally` — burying the failure that actually killed the kernel. + ctx.savepoints.length = 0; + throw error; } // The rollback reverted DB state but in-memory caches are stale. // Recreate the run queue so its cached head/tail are re-read from DB. @@ -81,11 +87,11 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { kdb.releaseSavepoint('t0'); } finally { // Forget the savepoints even if the release failed, as `rollbackCrank` - // does. A failed release discards the whole transaction (see - // `releaseSavepoint`), so the database has no savepoints left either; - // leaving them listed here would have the next crank number its savepoint - // `t1`, and from then on every release and rollback would aim one crank - // past the one it meant to end. + // does when its own rollback fails. A failed release discards the whole + // transaction (see `releaseSavepoint`), so the database has no savepoints + // left either; leaving them listed here would have the next crank number + // its savepoint `t1`, and from then on every release and rollback would + // aim one crank past the one it meant to end. ctx.savepoints.length = 0; } } From d6ab74cf8c876be8fdedc20f077507b584468d46 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:37:23 +0200 Subject: [PATCH 12/34] fix(kernel-store): log an abort that fails while discarding a transaction Both drivers recover from a failed savepoint operation by discarding the enclosing transaction, and swallow any error from that abort so the savepoint failure stays the one reported. That part is right, but it left the abandoned transaction entirely silent: on the nodejs driver, where `inTransaction` is read from SQLite, the next crank's `beginIfNeeded` sees the transaction still open, skips its `BEGIN`, and commits the dead crank's writes alongside the new crank's. Nothing here can repair that, so at least record it. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/src/sqlite/nodejs.ts | 22 ++++++++++++++++++---- packages/kernel-store/src/sqlite/wasm.ts | 22 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index d0a6442442..5b7e777b0c 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -298,8 +298,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The rollback failure below is the one worth reporting. + } catch (abortError) { + // The rollback failure below is the one worth reporting, but a failed + // abort leaves SQLite holding a transaction this driver has stopped + // tracking, which the next crank would silently write into. Nothing here + // can repair that, so at least say so. + logger?.error( + 'failed to discard transaction after rollback', + abortError, + ); } throw error; } @@ -332,8 +339,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The release failure below is the one worth reporting. + } catch (abortError) { + // The release failure below is the one worth reporting, but a failed + // abort leaves SQLite holding a transaction this driver has stopped + // tracking, which the next crank would silently write into. Nothing here + // can repair that, so at least say so. + logger?.error( + 'failed to discard transaction after release', + abortError, + ); } throw error; } diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index beda148117..65d9915e48 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -388,8 +388,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The rollback failure below is the one worth reporting. + } catch (abortError) { + // The rollback failure below is the one worth reporting. `_inTx` is + // already false by then, so the next `beginIfNeeded` issues its `BEGIN` + // and SQLite says loudly if it really is still in a transaction — but + // that is a crank away, and this is where the evidence is. + logger?.error( + 'failed to discard transaction after rollback', + abortError, + ); } throw error; } @@ -422,8 +429,15 @@ export async function makeSQLKernelDatabase({ db._spStack.length = 0; try { rollbackIfNeeded(); - } catch { - // The release failure below is the one worth reporting. + } catch (abortError) { + // The release failure below is the one worth reporting. `_inTx` is + // already false by then, so the next `beginIfNeeded` issues its `BEGIN` + // and SQLite says loudly if it really is still in a transaction — but + // that is a crank away, and this is where the evidence is. + logger?.error( + 'failed to discard transaction after release', + abortError, + ); } throw error; } From dcf1db7a063e2c8a941bbae22de8ba2f01565979 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:38:10 +0200 Subject: [PATCH 13/34] test(ocap-kernel): pin the flush's ordering against a failing enqueue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving `#invokeKernelSubscription` out of the enqueue loop and after it was the one production change on this branch with no test: reverting `#flushCrankBuffer` to its interleaved form left all 2412 ocap-kernel tests passing. Same hazard as the crank-level ordering a few tests up, one level down — `#enqueueRun` is store work and can fail part-way, so answering the first caller while the second enqueue is still ahead hands out a result the crank's rollback then discards. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelQueue.test.ts | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index dd3bd2d6d2..02336d1b6d 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -272,6 +272,57 @@ describe('KernelQueue', () => { ); }); + // The same invariant one level down, inside the flush itself: moving every + // buffered item onto the run queue is store work too, and it can fail + // part-way. Answering the first caller while the second enqueue is still + // ahead would hand out a result the crank's rollback then discards. + it('answers no caller until every buffered item is enqueued', async () => { + const mockItem: RunQueueItem = { + type: 'send', + target: 'ko123', + message: { result: 'kp1' } as KernelMessage, + }; + (kernelStore.runQueueLength as unknown as MockInstance) + .mockReturnValueOnce(1) + .mockReturnValue(0); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce( + mockItem, + ); + + const resolve = vi.fn(); + const reject = vi.fn(); + kernelQueue.subscriptions.set('kp1', { resolve, reject }); + + // Two resolutions to hand over, the caller's first. + ( + kernelStore.flushCrankBuffer as unknown as MockInstance + ).mockReturnValueOnce([ + { type: 'notify', endpointId: 'v1', kpid: 'kp1' }, + { type: 'notify', endpointId: 'v2', kpid: 'kp2' }, + ]); + (kernelStore.getKernelPromise as unknown as MockInstance).mockReturnValue( + { state: 'fulfilled', value: { body: '"answer"', slots: [] } }, + ); + + // The second enqueue is the write that fails. + const enqueueError = new Error('database is gone'); + let enqueued = 0; + (kernelStore.enqueueRun as unknown as MockInstance).mockImplementation( + () => { + enqueued += 1; + if (enqueued > 1) { + throw enqueueError; + } + }, + ); + + const deliver = vi.fn().mockResolvedValue(undefined); + await expect(kernelQueue.run(deliver)).rejects.toBe(enqueueError); + + expect(kernelStore.rollbackCrank).toHaveBeenCalledWith('delivery'); + expect(resolve).not.toHaveBeenCalled(); + }); + // Rolling back to the crank's *outermost* savepoint discards the enclosing // transaction (see `rollbackSavepoint`), which would leave the work an aborted // crank still owes — terminating the vat whose delivery failed, collecting From 97f161b0ff5c7320232430e3451544ab4468caed Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:38:24 +0200 Subject: [PATCH 14/34] docs: correct the transaction claims review found wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five comments on this branch asserted more than the code holds: - `wasm.ts` claimed a stale `_inTx` meant "no later rollback can undo the delivery". False: a savepoint created in autocommit mode does open a transaction, and an inner savepoint still rolls back. The real cost is that writes outside a savepoint autocommit one statement at a time, and the outermost `RELEASE` commits. The "an abort typically fails because SQLite already rolled back" premise was unsupported and isn't the reason for the reorder — the reason is simply that the abort can throw. - `#processCrankResult` said "the worker is already gone" ahead of the call that kills the worker. - The flush was described as running "once nothing fallible remains". It doesn't: `#terminateVat` resolves the dying vat's promises through `resolvePromises`, which defaults to `immediate` and invokes their kernel subscriptions before `collectGarbage`. Reachable without an abort, via a clean `exitVat`. Recorded rather than fixed — closing it changes termination semantics, not crank ordering. - "Only `delivery` is ever rolled back" is true of the run loop but not of the tests. Scoped, and the ordinal coupling it depends on is now stated: `endCrank` releases `t0` by position, so `crank` must stay first. - `reapImporterUntil` credited `scheduleReap` deduping for the old one-BOYD behaviour; it was `nextReapAction` shifting the single entry off, leaving the later cranks nothing to do. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 6 +++ packages/kernel-store/src/sqlite/wasm.ts | 16 ++++---- .../src/garbage-collection.test.ts | 15 ++++--- packages/ocap-kernel/CHANGELOG.md | 5 +++ packages/ocap-kernel/src/KernelQueue.ts | 39 ++++++++++++------- 5 files changed, 56 insertions(+), 25 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 5cfe39eb5e..5e1957b5e8 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` itself fails, the same way `rollbackSavepoint` already did when `ROLLBACK TO` failed ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - The same hazard by the other door: the savepoint stayed on the stack and the transaction stayed open with nothing left that would ever commit or abort it. The release failure is what gets thrown, even if aborting fails too +- The wasm driver leaves `_inTx` false when aborting a transaction throws, rather than believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - It tracks `_inTx` itself instead of reading it from SQLite, so a throwing abort was the one case that could leave the two disagreeing. Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted a statement at a time; left false, the next `BEGIN` fails loudly if SQLite really is still in a transaction. The nodejs driver reads `db.inTransaction` and was never affected +- An abort that fails while recovering from a failed savepoint operation is logged, in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - The savepoint failure is still the one thrown, but the abandoned transaction it leaves behind was previously silent ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 65d9915e48..45313869fa 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -211,13 +211,15 @@ export async function makeSQLKernelDatabase({ function rollbackIfNeeded(): void { if (db._inTx) { // Out of the transaction as far as this driver is concerned before the - // abort is even attempted. Unlike the nodejs driver, which reads - // `inTransaction` from SQLite, `_inTx` is tracked here — and an abort - // typically fails because SQLite already rolled back on its own as part of - // whatever went wrong. Left true, `beginIfNeeded` is a no-op from then on - // and the next `createSavepoint` runs in autocommit mode, where its - // `RELEASE` commits (see `createSavepoint`) and no later rollback can undo - // the delivery. + // abort is even attempted, because the abort can throw. Unlike the nodejs + // driver, which reads `inTransaction` from SQLite, `_inTx` is tracked here, + // so a throw is the one thing that can leave the two disagreeing. Left + // true, `beginIfNeeded` is a no-op from then on: writes outside a savepoint + // autocommit one statement at a time, and the outermost `RELEASE` of a + // savepoint created in autocommit mode commits rather than nesting (see + // `createSavepoint`). Setting it false instead means the next `BEGIN` + // throws if SQLite really is still in a transaction, which is the failure + // worth having. db._inTx = false; db._spStack.length = 0; sqlAbortTransaction.step(); diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 45f21f12d2..c081955853 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -122,13 +122,18 @@ describe('Garbage Collection', () => { * vat's own garbage collection, or the attempts run out. * * `bringOutYourDead` can only report an import as dropped once the engine has - * collected the vat's presence and run its finalizer, which forcing a GC pass + * collected the vat's presence and run its finalizer, which `gcAndFinalize` * does not guarantee on the first attempt. Reaping once and then cranking - * buys one attempt rather than several, because `scheduleReap` dedupes — so - * each attempt schedules its own reap, and a message to the vat wakes the run - * loop to consume it. + * repeatedly buys one attempt rather than several, because `nextReapAction` + * shifts the single scheduled entry off and the later cranks find nothing to + * do — so each attempt has to schedule its own reap, with a message to the vat + * to wake the run loop and consume it. * - * @param settled - Whether the state under test has arrived. + * Gives up after five attempts and lets the caller's assertion report the + * failure, which names the refcount that never arrived. + * + * @param settled - Predicate answering whether the state under test has + * arrived. */ async function reapImporterUntil(settled: () => boolean): Promise { const isImporter = (vatId: VatId): boolean => vatId === importerVatId; diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 1cab9b17ef..a4062f06d8 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -66,6 +66,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly +- Keep a crank's store work inside one transaction, so the writes an aborted crank still owes are committed or undone as a unit ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost savepoint ends the enclosing transaction, so terminating the vat and collecting garbage — which follow the rollback, and must survive it — were autocommitting a statement at a time, beyond the reach of any later rollback + - Buffered vat outputs are flushed after that work rather than before it. The flush settles the promise `queueMessage` handed an external caller, so a later failure used to roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A failed rollback or release discards the whole transaction, taking every savepoint with it. Keeping them listed had the next crank number its savepoint `t1` while the database had none, and had `endCrank` throw `No such savepoint: t0` from the run loop's `finally` — over whatever really killed the kernel - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index cd3b7435e3..355218db97 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -135,8 +135,11 @@ export class KernelQueue { // the enclosing transaction (see `rollbackSavepoint`), and the work an // aborted crank still owes — terminating the vat whose delivery failed, // collecting garbage — would then autocommit statement by statement, - // beyond the reach of any later rollback. Only `delivery` is ever rolled - // back; releasing `crank` in `endCrank` is this crank's one commit point. + // beyond the reach of any later rollback. The run loop only ever rolls + // back `delivery`; `crank` is released by `endCrank`, which is this + // crank's one commit point. That release names the *first* savepoint + // created here, by ordinal — see `releaseAllSavepoints` — so `crank` has + // to stay first. this.#kernelStore.createCrankSavepoint('crank'); this.#kernelStore.createCrankSavepoint('delivery'); @@ -342,21 +345,30 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // Vat termination during delivery is triggered by an illegal syscall - // or by syscall.exit(). Its store writes have to survive the rollback above: - // the worker is already gone, so a store that still believed the vat was - // alive would relaunch it after a restart and redeliver what killed it. + // Vat termination during delivery is triggered by an illegal syscall or by + // syscall.exit(). This call is what kills the worker, and its store writes + // have to survive the rollback above: once the worker is gone, a store that + // still believed the vat was alive would relaunch it after a restart and + // redeliver what killed it. Hence the rollback goes only as far as + // `delivery`, leaving these writes inside the crank's transaction. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { - // The crank survived, so hand its buffered outputs on — last, once nothing - // fallible remains. The flush settles the promise `enqueueMessage` gave an - // external caller, reading the result out of the store; were the crank - // rolled back after that, the caller would keep an answer computed from - // state the store discarded, and a restart would deliver the message again. + // The crank survived, so hand its buffered outputs on — after the store + // work above, which can still fail. The flush settles the promise + // `enqueueMessage` gave an external caller, reading the result out of the + // store; were the crank rolled back after that, the caller would keep an + // answer computed from state the store discarded, and a restart would + // deliver the message again. + // + // Not airtight: `#terminateVat` resolves the promises the dying vat was + // deciding via `resolvePromises`, which defaults to `immediate` and so + // invokes their kernel subscriptions before `collectGarbage` runs. Closing + // that would mean deferring those too, which is a change to termination + // semantics rather than to crank ordering. this.#flushCrankBuffer(); } // After the flush, because the audit reads the run queue as ground truth @@ -396,8 +408,9 @@ export class KernelQueue { resolved.push(item.kpid); } } - // Also promises resolved during this crank that don't have kernel-level - // subscribers (e.g., promises from enqueueMessage). + // Plus promises resolved during this crank that produced no notify of their + // own — nothing in the store was subscribed to them — but that the kernel + // itself is waiting on (e.g., promises from `enqueueMessage`). resolved.push(...this.#resolvedWithKernelSubscription); this.#resolvedWithKernelSubscription = []; From fe9980380cdfcf1938838e14ba2f5258fbd7643d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 6 Aug 2026 17:50:32 +0200 Subject: [PATCH 15/34] docs: cut the padding from this branch's comments and changelogs Comment the non-obvious why, in the shortest form that carries it. The two-savepoint rationale was re-argued in full in four places; the tests now point at `#runLoop` and `#processCrankResult` instead of restating them, and the hazard block duplicated across both driver test files is a line. No reasoning removed, only the retelling. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 11 ++- .../kernel-store/src/sqlite/nodejs.test.ts | 5 +- packages/kernel-store/src/sqlite/nodejs.ts | 10 +-- packages/kernel-store/src/sqlite/wasm.test.ts | 17 ++-- packages/kernel-store/src/sqlite/wasm.ts | 30 +++---- .../kernel-test/src/crank-rollback.test.ts | 14 ++- .../src/garbage-collection.test.ts | 16 ++-- packages/ocap-kernel/CHANGELOG.md | 8 +- packages/ocap-kernel/src/KernelQueue.test.ts | 24 ++--- packages/ocap-kernel/src/KernelQueue.ts | 88 +++++++------------ .../src/store/methods/crank.test.ts | 16 ++-- .../ocap-kernel/src/store/methods/crank.ts | 25 +++--- 12 files changed, 98 insertions(+), 166 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index 5e1957b5e8..f49786942c 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,12 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` itself fails, the same way `rollbackSavepoint` already did when `ROLLBACK TO` failed ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - The same hazard by the other door: the savepoint stayed on the stack and the transaction stayed open with nothing left that would ever commit or abort it. The release failure is what gets thrown, even if aborting fails too -- The wasm driver leaves `_inTx` false when aborting a transaction throws, rather than believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - It tracks `_inTx` itself instead of reading it from SQLite, so a throwing abort was the one case that could leave the two disagreeing. Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted a statement at a time; left false, the next `BEGIN` fails loudly if SQLite really is still in a transaction. The nodejs driver reads `db.inTransaction` and was never affected -- An abort that fails while recovering from a failed savepoint operation is logged, in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - The savepoint failure is still the one thrown, but the abandoned transaction it leaves behind was previously silent +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown +- The wasm driver clears `_inTx` when aborting a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted. The nodejs driver reads `db.inTransaction` and was never affected +- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/nodejs.test.ts b/packages/kernel-store/src/sqlite/nodejs.test.ts index e38655d9a9..270f018ae6 100644 --- a/packages/kernel-store/src/sqlite/nodejs.test.ts +++ b/packages/kernel-store/src/sqlite/nodejs.test.ts @@ -360,10 +360,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._spStack).toStrictEqual([]); }); - // The same hazard `rollbackSavepoint` guards against, by the other door: a - // RELEASE that throws leaves the savepoint on the stack and the transaction - // open with nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. + // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb.inTransaction = true; diff --git a/packages/kernel-store/src/sqlite/nodejs.ts b/packages/kernel-store/src/sqlite/nodejs.ts index 5b7e777b0c..fa754bb88f 100644 --- a/packages/kernel-store/src/sqlite/nodejs.ts +++ b/packages/kernel-store/src/sqlite/nodejs.ts @@ -300,9 +300,8 @@ export async function makeSQLKernelDatabase({ rollbackIfNeeded(); } catch (abortError) { // The rollback failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction this driver has stopped - // tracking, which the next crank would silently write into. Nothing here - // can repair that, so at least say so. + // abort leaves SQLite holding a transaction the next crank would + // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -341,9 +340,8 @@ export async function makeSQLKernelDatabase({ rollbackIfNeeded(); } catch (abortError) { // The release failure below is the one worth reporting, but a failed - // abort leaves SQLite holding a transaction this driver has stopped - // tracking, which the next crank would silently write into. Nothing here - // can repair that, so at least say so. + // abort leaves SQLite holding a transaction the next crank would + // silently write into. Nothing here can repair that. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 25403caa47..8e82a00bc9 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -517,10 +517,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // The same hazard `rollbackSavepoint` guards against, by the other door: a - // RELEASE that throws leaves the savepoint on the stack and the transaction - // open with nothing to ever commit or abort it, so every later write on this - // connection joins it, reports success, and vanishes on close. + // The hazard `rollbackSavepoint` guards against, by the other door. it('releaseSavepoint discards the transaction when the release fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -555,9 +552,8 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // `_inTx` is tracked here rather than read from SQLite, so a failed abort is - // the one case that can leave it disagreeing with the database. Left true, - // `beginIfNeeded` becomes a no-op forever after. + // A failed abort is the one case that can leave `_inTx` disagreeing with the + // database. Left true, `beginIfNeeded` is a no-op forever after. it('stops believing it is in a transaction when the abort fails too', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; @@ -576,10 +572,9 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb._inTx).toBe(false); }); - // The consequence of the above, and the reason it is worth asserting: a - // savepoint created outside a transaction autocommits when released - // (Agoric/agoric-sdk#8423), so no later rollback can undo the delivery — an - // aborted crank would silently keep its writes. + // Why that matters: a savepoint created outside a transaction commits when + // released (Agoric/agoric-sdk#8423), so an aborted crank would keep its + // writes. it('begins a transaction for the next savepoint after a failed abort', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index 45313869fa..a43092732b 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -210,16 +210,12 @@ export async function makeSQLKernelDatabase({ */ function rollbackIfNeeded(): void { if (db._inTx) { - // Out of the transaction as far as this driver is concerned before the - // abort is even attempted, because the abort can throw. Unlike the nodejs - // driver, which reads `inTransaction` from SQLite, `_inTx` is tracked here, - // so a throw is the one thing that can leave the two disagreeing. Left - // true, `beginIfNeeded` is a no-op from then on: writes outside a savepoint - // autocommit one statement at a time, and the outermost `RELEASE` of a - // savepoint created in autocommit mode commits rather than nesting (see - // `createSavepoint`). Setting it false instead means the next `BEGIN` - // throws if SQLite really is still in a transaction, which is the failure - // worth having. + // Cleared before the abort is attempted, because the abort can throw and + // `_inTx` is tracked here rather than read from SQLite as the nodejs driver + // does. Left true, `beginIfNeeded` is a no-op forever after and writes + // autocommit one statement at a time (see `createSavepoint`). Cleared, a + // still-open transaction surfaces as a failed `BEGIN` — the louder + // failure. db._inTx = false; db._spStack.length = 0; sqlAbortTransaction.step(); @@ -391,10 +387,9 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The rollback failure below is the one worth reporting. `_inTx` is - // already false by then, so the next `beginIfNeeded` issues its `BEGIN` - // and SQLite says loudly if it really is still in a transaction — but - // that is a crank away, and this is where the evidence is. + // The rollback failure below is the one worth reporting. The next + // `BEGIN` will fail if SQLite really is still in a transaction, but that + // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after rollback', abortError, @@ -432,10 +427,9 @@ export async function makeSQLKernelDatabase({ try { rollbackIfNeeded(); } catch (abortError) { - // The release failure below is the one worth reporting. `_inTx` is - // already false by then, so the next `beginIfNeeded` issues its `BEGIN` - // and SQLite says loudly if it really is still in a transaction — but - // that is a crank away, and this is where the evidence is. + // The release failure below is the one worth reporting. The next + // `BEGIN` will fail if SQLite really is still in a transaction, but that + // is a crank away and this is where the evidence is. logger?.error( 'failed to discard transaction after release', abortError, diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index dce56b3227..edb4fd0e61 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -138,9 +138,8 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('second')).toBe('yes'); }); - // An aborted crank rolls its delivery back and then still has work to do — - // terminating the vat whose delivery failed, collecting garbage — whose writes - // have to survive that rollback, since the vat's worker is already gone. + // An aborted crank still owes work after the rollback — terminating the vat, + // collecting garbage — whose writes have to survive it. it('keeps the writes a crank makes after rolling its delivery back', async () => { const { kernelStore, kdb } = await makeStore(); @@ -157,12 +156,9 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBe('yes'); }); - // And they survive it *as part of the crank's transaction*, which is why the - // delivery gets a savepoint of its own rather than rolling back the crank's: - // rolling back the outermost savepoint discards the transaction, after which - // those writes would autocommit one statement at a time. Nothing in the run - // loop rolls the crank's own savepoint back — it is the only way from here to - // observe that the writes are still undoable at all. + // And survive it *inside the crank's transaction*, not as autocommitted + // statements. Rolling back `crank` is the only way to observe that from here; + // the run loop never does it. it('holds those writes in the transaction rather than autocommitting them', async () => { const { kernelStore, kdb } = await makeStore(); diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index c081955853..2fc1dc1605 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -121,19 +121,15 @@ describe('Garbage Collection', () => { * Reap the importer vat until the kernel's bookkeeping catches up with the * vat's own garbage collection, or the attempts run out. * - * `bringOutYourDead` can only report an import as dropped once the engine has + * `bringOutYourDead` reports an import as dropped only once the engine has * collected the vat's presence and run its finalizer, which `gcAndFinalize` - * does not guarantee on the first attempt. Reaping once and then cranking - * repeatedly buys one attempt rather than several, because `nextReapAction` - * shifts the single scheduled entry off and the later cranks find nothing to - * do — so each attempt has to schedule its own reap, with a message to the vat - * to wake the run loop and consume it. + * does not guarantee on the first attempt. Each attempt needs its own reap — + * `nextReapAction` shifts the one scheduled entry off, so cranking again finds + * nothing to do — plus a message to wake the run loop and consume it. * - * Gives up after five attempts and lets the caller's assertion report the - * failure, which names the refcount that never arrived. + * Gives up after five attempts; the caller's assertion reports the failure. * - * @param settled - Predicate answering whether the state under test has - * arrived. + * @param settled - Whether the state under test has arrived yet. */ async function reapImporterUntil(settled: () => boolean): Promise { const isImporter = (vatId: VatId): boolean => vatId === importerVatId; diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index a4062f06d8..d5ad312806 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -66,11 +66,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction, so the writes an aborted crank still owes are committed or undone as a unit ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost savepoint ends the enclosing transaction, so terminating the vat and collecting garbage — which follow the rollback, and must survive it — were autocommitting a statement at a time, beyond the reach of any later rollback - - Buffered vat outputs are flushed after that work rather than before it. The flush settles the promise `queueMessage` handed an external caller, so a later failure used to roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time + - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - A failed rollback or release discards the whole transaction, taking every savepoint with it. Keeping them listed had the next crank number its savepoint `t1` while the database had none, and had `endCrank` throw `No such savepoint: t0` from the run loop's `finally` — over whatever really killed the kernel + - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index 02336d1b6d..a04fa05feb 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -214,13 +214,8 @@ describe('KernelQueue', () => { expect(kernelStore.endCrank).toHaveBeenCalled(); }); - // `#flushCrankBuffer` settles the promise `enqueueMessage` handed an external - // caller, reading the resolution out of the store on the way. Were the crank - // rolled back after that, the store would un-resolve the promise and restore - // the run queue item, so a restart would deliver the message a second time - // and notify every other subscriber again — while the original caller had - // already been told the first answer. So the flush comes last, after - // everything that could still fail. + // Why the flush comes after the crank's fallible work: see + // `#processCrankResult`. Here the terminate is what fails. it('answers no caller from a crank it then rolls back', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -272,10 +267,8 @@ describe('KernelQueue', () => { ); }); - // The same invariant one level down, inside the flush itself: moving every - // buffered item onto the run queue is store work too, and it can fail - // part-way. Answering the first caller while the second enqueue is still - // ahead would hand out a result the crank's rollback then discards. + // The same invariant inside the flush: `#enqueueRun` is store work and can + // fail part-way, so no caller may be answered until all of it lands. it('answers no caller until every buffered item is enqueued', async () => { const mockItem: RunQueueItem = { type: 'send', @@ -323,13 +316,8 @@ describe('KernelQueue', () => { expect(resolve).not.toHaveBeenCalled(); }); - // Rolling back to the crank's *outermost* savepoint discards the enclosing - // transaction (see `rollbackSavepoint`), which would leave the work an aborted - // crank still owes — terminating the vat whose delivery failed, collecting - // garbage — autocommitting statement by statement, beyond the reach of any - // later rollback. That work has to follow the rollback, since the worker is - // already gone and the store must not go on believing the vat is alive, so it - // is the rollback that has to spare the transaction. + // Why two savepoints: see `#runLoop`. This pins that the rollback spares the + // transaction, so the work an aborted crank still owes stays inside it. it.each([ { label: 'an abort', diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 355218db97..8c598dfcbf 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -56,14 +56,10 @@ export class KernelQueue { /** * Whether this crank's savepoint has already been handed to `rollbackCrank`. - * Attempted, not necessarily succeeded: `rollbackCrank` forgets the savepoint - * whether or not the database call throws, so after either outcome a second - * attempt can only report "no such savepoint" over the real error. - * - * This has to be recorded at the moment of the attempt rather than returned - * from `#processCrankResult`, because that method can throw after rolling back - * (`#terminateVat`, `collectGarbage`), and the catch below must still know not - * to ask twice. + * Attempted, not necessarily succeeded: it is forgotten either way, so a second + * attempt could only report "no such savepoint" over the real error. Recorded + * at the attempt rather than returned, because `#processCrankResult` can throw + * after rolling back. */ #crankRollbackAttempted: boolean = false; @@ -130,24 +126,18 @@ export class KernelQueue { this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; try { - // Two savepoints, because the crank's transaction has to outlive the - // delivery's rollback. Rolling back to the outermost savepoint discards - // the enclosing transaction (see `rollbackSavepoint`), and the work an - // aborted crank still owes — terminating the vat whose delivery failed, - // collecting garbage — would then autocommit statement by statement, - // beyond the reach of any later rollback. The run loop only ever rolls - // back `delivery`; `crank` is released by `endCrank`, which is this - // crank's one commit point. That release names the *first* savepoint - // created here, by ordinal — see `releaseAllSavepoints` — so `crank` has - // to stay first. + // Two savepoints, because rolling back the outermost one discards the + // enclosing transaction (see `rollbackSavepoint`) and an aborted crank + // still has writes to make. Only `delivery` is ever rolled back; + // releasing `crank` in `endCrank` is this crank's one commit point. + // `releaseAllSavepoints` names it by ordinal, so `crank` must stay first. this.#kernelStore.createCrankSavepoint('crank'); this.#kernelStore.createCrankSavepoint('delivery'); // The savepoint exists from here on, so a throw can be undone. Without - // this, `endCrank`'s savepoint release commits the half-finished crank: - // the item this crank dequeued is gone for good, refcount increments - // stick, and promises resolved during it stay resolved while their - // notifies die unflushed. A restart would resume from that. + // this, `endCrank`'s release commits the half-finished crank: the + // dequeued item is gone for good, refcount increments stick, and + // resolved promises keep their unflushed notifies. try { const queueItem = this.#getNextRunQueueItem(); if (queueItem) { @@ -164,15 +154,14 @@ export class KernelQueue { wakeUpPromise = promise; } } catch (error) { - // An aborted crank already asked, and `rollbackCrank` discards the - // savepoint either way; asking again could only throw "no such - // savepoint" over the real error. + // An aborted crank already asked; asking again could only throw "no + // such savepoint" over the real error. if (!this.#crankRollbackAttempted) { try { this.#kernelStore.rollbackCrank('delivery'); } catch (rollbackError) { - // The original failure stays the `cause`, since that is the root - // cause an operator needs; the rollback failure is named here. + // The original failure stays the `cause`; the rollback failure is + // named here. throw new Error( `Run loop died and its crank could not be rolled back: ${String(rollbackError)}`, { cause: error }, @@ -317,11 +306,9 @@ export class KernelQueue { try { this.#kernelStore.rollbackCrank('delivery'); } finally { - // Set even when the rollback threw. `rollbackCrank` forgets the - // savepoint in its own `finally`, so "attempted" and "the savepoint is - // gone" now coincide exactly — and a second attempt from the run loop's - // catch would report a missing savepoint as the reason the kernel died, - // burying the database error that actually killed it. + // Set even when the rollback threw: the savepoint is gone either way, so + // a second attempt would report a missing savepoint as the reason the + // kernel died, burying the error that actually killed it. this.#crankRollbackAttempted = true; } // Discard kernel subscriptions that were queued for invocation @@ -345,30 +332,23 @@ export class KernelQueue { // restart it and terminate the vat only after a certain number of failed // retries. This is probably where we should implement the vat restart logic. } - // Vat termination during delivery is triggered by an illegal syscall or by - // syscall.exit(). This call is what kills the worker, and its store writes - // have to survive the rollback above: once the worker is gone, a store that - // still believed the vat was alive would relaunch it after a restart and - // redeliver what killed it. Hence the rollback goes only as far as - // `delivery`, leaving these writes inside the crank's transaction. + // This call kills the worker, so its writes must outlive the rollback above: + // a store that still believed the vat was alive would relaunch it after a + // restart and redeliver what killed it. if (crankResult?.terminate) { const { vatId, info } = crankResult.terminate; await this.#terminateVat(vatId, info); } this.#kernelStore.collectGarbage(); if (!crankResult?.abort) { - // The crank survived, so hand its buffered outputs on — after the store - // work above, which can still fail. The flush settles the promise - // `enqueueMessage` gave an external caller, reading the result out of the - // store; were the crank rolled back after that, the caller would keep an - // answer computed from state the store discarded, and a restart would - // deliver the message again. + // After the fallible work above, not before it. The flush settles the + // promise `enqueueMessage` gave an external caller, so a later rollback + // would discard the state that answer was computed from. // - // Not airtight: `#terminateVat` resolves the promises the dying vat was - // deciding via `resolvePromises`, which defaults to `immediate` and so - // invokes their kernel subscriptions before `collectGarbage` runs. Closing - // that would mean deferring those too, which is a change to termination - // semantics rather than to crank ordering. + // Not airtight: `#terminateVat` resolves the dying vat's promises through + // `resolvePromises`, which defaults to `immediate` and invokes their + // subscriptions before `collectGarbage`. Deferring those too would change + // termination semantics, not crank ordering. this.#flushCrankBuffer(); } // After the flush, because the audit reads the run queue as ground truth @@ -408,15 +388,13 @@ export class KernelQueue { resolved.push(item.kpid); } } - // Plus promises resolved during this crank that produced no notify of their - // own — nothing in the store was subscribed to them — but that the kernel - // itself is waiting on (e.g., promises from `enqueueMessage`). + // Plus promises with no vat subscriber to notify, which the kernel is + // nonetheless waiting on (e.g. from `enqueueMessage`). resolved.push(...this.#resolvedWithKernelSubscription); this.#resolvedWithKernelSubscription = []; - // Callbacks only once every store write is done. Each hands an external - // caller a result read out of the store, and a write that threw in between - // would have the crank rolled back underneath answers already given. + // Callbacks only once every `#enqueueRun` is done: one that threw partway + // would roll the crank back underneath answers already given. for (const kpid of resolved) { this.#invokeKernelSubscription(kpid); } diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 389afd7a03..08a1e49254 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -154,11 +154,9 @@ describe('crank methods', () => { expect(mockCrankBuffer).toHaveLength(0); }); - // A failed rollback discards the whole transaction, so the enclosing - // savepoints are gone from the database too — not just the one rolled back - // to. Truncating to the ordinal would leave `endCrank` releasing a `t0` the - // database no longer has, and that "No such savepoint" would be thrown from - // the run loop's `finally`, over whatever really killed the kernel. + // A failed rollback discards every savepoint, not just this one. Truncating + // to the ordinal would have `endCrank` release a `t0` the database lacks and + // throw over whatever really killed the kernel. it('forgets every savepoint when the rollback fails', () => { context.inCrank = true; crankMethods.createCrankSavepoint('crank'); @@ -230,11 +228,9 @@ describe('crank methods', () => { expect(await waiter).toBeUndefined(); }); - // What `rollbackCrank` already does when its own rollback fails. Settling - // the crank regardless means callers proceed, so a savepoint left listed - // here has the next crank number its savepoint `t1` while the database still - // has `t0`: from then on `releaseAllSavepoints` releases the wrong one and - // every rollback aims past the crank it meant to undo. + // As `rollbackCrank` does. Left listed, the next crank numbers its savepoint + // `t1` against a database that has none, and every later release and rollback + // aims one crank past its target. it('forgets its savepoints even if releasing them fails', () => { crankMethods.startCrank(); context.savepoints = ['test']; diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index b26219cbb6..887ef2bee1 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -51,17 +51,14 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { if (ctx.savepoints[ordinal] === savepoint) { try { kdb.rollbackSavepoint(`t${ordinal}`); - // Forget the savepoint. Leaving it listed would have `endCrank`'s - // release commit the crank we just abandoned — the half-finished state - // this rollback exists to discard. + // Left listed, `endCrank`'s release would commit the crank we just + // abandoned. ctx.savepoints.length = ordinal; } catch (error) { - // A failed rollback discards the whole transaction (see - // `rollbackSavepoint`), taking every savepoint with it, not just this - // one. Truncating to `ordinal` would leave the enclosing savepoints - // listed against a database that no longer has them, and `endCrank` - // would then throw "No such savepoint: t0" from the run loop's - // `finally` — burying the failure that actually killed the kernel. + // A failed rollback discards the whole transaction, so every savepoint + // is gone, not just this one. Truncating to `ordinal` would have + // `endCrank` release a `t0` the database lacks and throw over whatever + // really killed the kernel. ctx.savepoints.length = 0; throw error; } @@ -86,12 +83,10 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { try { kdb.releaseSavepoint('t0'); } finally { - // Forget the savepoints even if the release failed, as `rollbackCrank` - // does when its own rollback fails. A failed release discards the whole - // transaction (see `releaseSavepoint`), so the database has no savepoints - // left either; leaving them listed here would have the next crank number - // its savepoint `t1`, and from then on every release and rollback would - // aim one crank past the one it meant to end. + // A failed release discards the transaction too, so the database has no + // savepoints left either. Left listed, the next crank would number its + // savepoint `t1` and aim every later release and rollback one crank past + // the one it meant to end. ctx.savepoints.length = 0; } } From f605ec69a46912b995bcadcc72068a780270865f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 10 Aug 2026 12:51:54 +0200 Subject: [PATCH 16/34] fix(ocap-kernel): revert cached values and GC candidates on crank rollback A database rollback cannot reach two pieces of state, so `rollbackCrank` now reverts both itself. Every `provideCachedStoredValue` answers reads from a closure and only writes through to kv. Reverting the database therefore left the closure holding the abandoned crank's value, and the next `set` persisted it. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright rather than retrying it. `reapQueue` was exposed the same way. `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of the decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop. No live bug either way: every `abort` `#deliverGCAction` returns is paired with a `terminate`, which is what made losing the action harmless. The comment there claimed the rollback restored the action, which is the thing a future reader would trust when adding an abort path that isn't paired with a termination; it now states the real causality. The cached values are declared once so that initialization and the refresher cannot disagree about which ones exist. Co-Authored-By: Claude Opus 5 (1M context) --- .../kernel-test/src/crank-rollback.test.ts | 64 +++++++++++++ packages/ocap-kernel/CHANGELOG.md | 4 + packages/ocap-kernel/src/store/index.ts | 89 ++++++++++++------- .../src/store/methods/crank.test.ts | 2 + .../ocap-kernel/src/store/methods/crank.ts | 54 +++++++++-- packages/ocap-kernel/src/store/types.ts | 1 + 6 files changed, 173 insertions(+), 41 deletions(-) diff --git a/packages/kernel-test/src/crank-rollback.test.ts b/packages/kernel-test/src/crank-rollback.test.ts index edb4fd0e61..36131a8a16 100644 --- a/packages/kernel-test/src/crank-rollback.test.ts +++ b/packages/kernel-test/src/crank-rollback.test.ts @@ -174,6 +174,70 @@ describe('crank rollback against a real database', () => { expect(kdb.kernelKVStore.get('terminated')).toBeUndefined(); }); + // Every `provideCachedStoredValue` keeps its value in a closure and writes + // through to kv, so a rollback that only reverts the database leaves the cache + // holding the abandoned crank's value — and the next `set` persists it. The GC + // action set is the case that matters: `processGCActionSet` consumes an action + // before delivering it, so losing the rollback loses the action outright. + it('restores the GC action set consumed by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + kernelStore.addGCActions(['v1 dropExport ko1']); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + // Consume the action the way `processGCActionSet` does. + kernelStore.setGCActions(new Set()); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + expect([...kernelStore.getGCActions()]).toStrictEqual([ + 'v1 dropExport ko1', + ]); + }); + + // Same closure, same failure: a reap scheduled and then consumed by a crank + // that rolls back must still be pending afterwards. + it('restores the reap queue consumed by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + kernelStore.scheduleReap('v1'); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + expect(kernelStore.nextReapAction()).toBeDefined(); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + expect(kernelStore.nextReapAction()).toBeDefined(); + }); + + // `maybeFreeKrefs` is RAM-only, so nothing rolls it back. Left populated, the + // next crank's `collectGarbage` visits krefs whose decrements were undone — + // and `getKernelPromise` throws outright for one the rollback deleted, which + // kills the run loop. + it('discards GC candidates accumulated by a rolled-back crank', async () => { + const { kernelStore } = await makeStore(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + kernelStore.createCrankSavepoint('delivery'); + // Born at 1, so this drops it to 0 and leaves `kpid` in `maybeFreeKrefs` + // while the rollback removes the promise record it names. + const kpid = kernelStore.initKernelPromise()[0]; + kernelStore.decrementRefCount(kpid, 'test'); + + kernelStore.rollbackCrank('delivery'); + kernelStore.endCrank(); + + kernelStore.startCrank(); + kernelStore.createCrankSavepoint('crank'); + expect(() => kernelStore.collectGarbage()).not.toThrow(); + kernelStore.endCrank(); + }); + // `createCrankSavepoint` records the name only once the database has the // savepoint. Asking to roll back one that was never created must therefore say // so, rather than releasing someone else's savepoint. diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index d5ad312806..0276aa11c9 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -71,6 +71,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel +- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - A cached stored value answers reads from a closure and only writes through to the database, so reverting the database left it holding the abandoned crank's value and the next write persisted that. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright instead of retrying it + - `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop + - A failed rollback discards the whole transaction, moving the database back at least as far as a successful rollback would have — so reverting only on success left exactly the state that is least able to tolerate it - Refuse inbound remote deliveries once the run loop is dead, rolling back without acknowledging them, so the peer retries and gives up instead of waiting on a kernel that will never deliver ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Covers `bringOutYourDead` as well as `message` and `notify`: a reap is queue work too, consumed only by the run loop. The remaining GC arms need no guard, since they only touch refcounts - Refuse `launchSubcluster` once the run loop is dead ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) diff --git a/packages/ocap-kernel/src/store/index.ts b/packages/ocap-kernel/src/store/index.ts index 8e082aa8b1..436a21cc86 100644 --- a/packages/ocap-kernel/src/store/index.ts +++ b/packages/ocap-kernel/src/store/index.ts @@ -89,7 +89,7 @@ import { getRevocationMethods } from './methods/revocation.ts'; import { getSubclusterMethods } from './methods/subclusters.ts'; import { getTranslators } from './methods/translators.ts'; import { getVatMethods } from './methods/vat.ts'; -import type { StoreContext } from './types.ts'; +import type { StoreContext, StoredValue } from './types.ts'; /** * Create a new KernelStore object wrapped around a raw kernel database. The @@ -115,6 +115,48 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { const { provideCachedStoredValue, provideStoredQueue } = getBaseMethods(kv); + /** + * Every cached stored value the context holds, as `field: [key, initial]`. + * Declared once so that initialization and `refreshCachedValues` cannot + * disagree about which values exist: adding one here does both. + */ + const CACHED_VALUES = { + /** Counter for allocating kernel object IDs */ + nextObjectId: ['nextObjectId', '1'], + /** Counter for allocating kernel promise IDs */ + nextPromiseId: ['nextPromiseId', '1'], + /** Counter for allocating VatIDs */ + nextVatId: ['nextVatId', '1'], + /** Counter for allocating RemoteIDs */ + nextRemoteId: ['nextRemoteId', '1'], + // Garbage collection + gcActions: ['gcActions', '[]'], + reapQueue: ['reapQueue', '[]'], + terminatedVats: ['vats.terminated', '[]'], + // Subclusters + subclusters: ['subclusters', '[]'], + nextSubclusterId: ['nextSubclusterId', '1'], + vatToSubclusterMap: ['vatToSubclusterMap', '{}'], + } as const satisfies Record; + + /** + * Provide a fresh stored value for each of {@link CACHED_VALUES}, reading its + * current setting out of the database. + * + * @returns The stored values, keyed by the context field that holds each. + */ + function provideCachedValues(): Record< + keyof typeof CACHED_VALUES, + StoredValue + > { + return Object.fromEntries( + Object.entries(CACHED_VALUES).map(([field, [key, init]]) => [ + field, + provideCachedStoredValue(key, init), + ]), + ) as Record; + } + const context: StoreContext = { kv, /** The kernel's run queue. */ @@ -125,14 +167,16 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { refreshRunQueue: () => { context.runQueue = provideStoredQueue('run', true); }, - /** Counter for allocating kernel object IDs */ - nextObjectId: provideCachedStoredValue('nextObjectId', '1'), - /** Counter for allocating kernel promise IDs */ - nextPromiseId: provideCachedStoredValue('nextPromiseId', '1'), - /** Counter for allocating VatIDs */ - nextVatId: provideCachedStoredValue('nextVatId', '1'), - /** Counter for allocating RemoteIDs */ - nextRemoteId: provideCachedStoredValue('nextRemoteId', '1'), + ...provideCachedValues(), + /** + * Re-read every cached stored value from the database. Each one closes over + * the last value written through it (see `provideCachedStoredValue`), so + * reverting the database alone is not enough: the closure would still hold + * the abandoned value and the next `set` would write it straight back. + */ + refreshCachedValues: () => { + Object.assign(context, provideCachedValues()); + }, // As refcounts are decremented, we accumulate a set of krefs for which // action might need to be taken: // * promises which are now resolved and unreferenced can be deleted @@ -144,17 +188,9 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { // the change, else removals might be lost (not performed during the next // replay). maybeFreeKrefs: new Set(), - // Garbage collection - gcActions: provideCachedStoredValue('gcActions', '[]'), - reapQueue: provideCachedStoredValue('reapQueue', '[]'), - terminatedVats: provideCachedStoredValue('vats.terminated', '[]'), inCrank: false, savepoints: [], crankBuffer: [], - // Subclusters - subclusters: provideCachedStoredValue('subclusters', '[]'), - nextSubclusterId: provideCachedStoredValue('nextSubclusterId', '1'), - vatToSubclusterMap: provideCachedStoredValue('vatToSubclusterMap', '{}'), auditRefCounts: false, // Logging logger: logger?.subLogger({ tags: ['kernel-store'] }), @@ -214,23 +250,8 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) { })); kdb.clear(); context.maybeFreeKrefs.clear(); - context.runQueue = provideStoredQueue('run', true); - context.gcActions = provideCachedStoredValue('gcActions', '[]'); - context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); - context.terminatedVats = provideCachedStoredValue('vats.terminated', '[]'); - context.nextObjectId = provideCachedStoredValue('nextObjectId', '1'); - context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1'); - context.nextVatId = provideCachedStoredValue('nextVatId', '1'); - context.nextRemoteId = provideCachedStoredValue('nextRemoteId', '1'); - context.subclusters = provideCachedStoredValue('subclusters', '[]'); - context.nextSubclusterId = provideCachedStoredValue( - 'nextSubclusterId', - '1', - ); - context.vatToSubclusterMap = provideCachedStoredValue( - 'vatToSubclusterMap', - '{}', - ); + context.refreshRunQueue(); + context.refreshCachedValues(); crank.releaseAllSavepoints(); context.crankBuffer.length = 0; preservedState?.forEach(({ key, value }) => { diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 08a1e49254..630444e850 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -17,6 +17,8 @@ describe('crank methods', () => { savepoints: [], crankBuffer: mockCrankBuffer, refreshRunQueue: vi.fn(), + refreshCachedValues: vi.fn(), + maybeFreeKrefs: new Set(), } as unknown as StoreContext; kdb = { diff --git a/packages/ocap-kernel/src/store/methods/crank.ts b/packages/ocap-kernel/src/store/methods/crank.ts index 887ef2bee1..a3bba1956b 100644 --- a/packages/ocap-kernel/src/store/methods/crank.ts +++ b/packages/ocap-kernel/src/store/methods/crank.ts @@ -60,21 +60,61 @@ export function getCrankMethods(ctx: StoreContext, kdb: KernelDatabase) { // `endCrank` release a `t0` the database lacks and throw over whatever // really killed the kernel. ctx.savepoints.length = 0; + // Before the rethrow, and not only on the path below. A failed + // rollback discards the whole transaction, so the database has moved + // back at least as far as a successful rollback would have taken it + // and these caches are at least as stale. Rethrowing ahead of this + // would leave the dying crank holding the GC action it consumed and + // the freed krefs it was about to collect. + revertStateBeneathRollback(error); throw error; } - // The rollback reverted DB state but in-memory caches are stale. - // Recreate the run queue so its cached head/tail are re-read from DB. - ctx.refreshRunQueue(); - // Invalidate the run queue length cache so it's recalculated from - // the database on next access, since the rollback may have restored - // dequeued items. - ctx.runQueueLengthCache = -1; + revertStateBeneathRollback(); return; } } Fail`no such savepoint as "${q(savepoint)}"`; } + /** + * Revert what a database rollback cannot reach: the in-memory caches built + * over the abandoned crank's writes. + * + * @param rollbackError - The error the rollback threw, if it threw. Kept as + * the `cause` should reverting fail too, since it is the root cause an + * operator needs. + */ + function revertStateBeneathRollback(rollbackError?: unknown): void { + try { + // Recreate the run queue so its cached head/tail are re-read from the + // database, and invalidate the length cache, since the rollback may have + // restored dequeued items. + ctx.refreshRunQueue(); + ctx.runQueueLengthCache = -1; + // Same staleness, worse consequence: a cached value reads from its + // closure and only writes through to kv, so one this crank consumed stays + // consumed and the next `set` persists that. `processGCActionSet` takes an + // action out of the set before delivering it, so an action not restored + // here is lost rather than retried. + ctx.refreshCachedValues(); + // Nothing rolls back RAM. These krefs are collection candidates only + // because this crank decremented them, and that is precisely what was just + // undone. Left in place, `collectGarbage` throws on a later crank for any + // promise this one created — killing the run loop over work that no longer + // exists. Correct only while every rollback discards the whole delivery, + // which is all any caller asks for. + ctx.maybeFreeKrefs.clear(); + } catch (revertError) { + if (rollbackError === undefined) { + throw revertError; + } + throw new Error( + `Crank rollback failed and its caches could not be reverted: ${String(revertError)}`, + { cause: rollbackError }, + ); + } + } + /** * Release all savepoints. */ diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index 3bf54862fa..b9886c174f 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -11,6 +11,7 @@ export type StoreContext = { runQueue: StoredQueue; // Holds RunAction[] runQueueLengthCache: number; // Holds number refreshRunQueue: () => void; + refreshCachedValues: () => void; nextObjectId: StoredValue; // Holds string nextPromiseId: StoredValue; // Holds string nextVatId: StoredValue; // Holds string From 5fadbf40ff786bf38e9287b17e127fcfa4e5be79 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:51:25 -0400 Subject: [PATCH 17/34] test(kernel-store): pin the failed COMMIT that wedges `_inTx` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue `rollbackIfNeeded` was corrected in #1012 to clear `_inTx` *before* stepping the abort, because the abort can throw and `_inTx` is tracked in the driver rather than read from SQLite. `commitIfNeeded` has the identical shape and was left alone: function commitIfNeeded(): void { if (db._inTx && db._spStack.length === 0) { sqlCommitTransaction.step(); // can throw sqlCommitTransaction.reset(); db._inTx = false; // ...so this never runs } } A COMMIT that throws leaves `_inTx` true against a database that may hold no transaction. `beginIfNeeded` is then a no-op forever after, so the next `createSavepoint` issues its SAVEPOINT outside a transaction — and a savepoint taken outside a transaction commits when it is released (Agoric/agoric-sdk#8423). That is the hazard the whole `beginIfNeeded` dance exists to prevent, and `commitIfNeeded` is reached from `releaseSavepoint`, which is the crank's commit point. The writes that leak are a whole crank's. The nodejs driver is unaffected, for the same reason it was unaffected by the abort case: it reads `db.inTransaction` live from SQLite. Worth noting that the comment introduced above `stops believing it is in a transaction when the abort fails too` asserts that a failed abort is "the one case that can leave `_inTx` disagreeing with the database". This is the second case, so that comment needs correcting along with the code. ## What we hope to see instead `releaseSavepoint` still throws the COMMIT failure, but `_inTx` is false afterwards, so the next `createSavepoint` opens a transaction of its own instead of creating a bare savepoint. Same two-line reorder as `rollbackIfNeeded`, and the "one case" comment updated. ## Current failure AssertionError: expected true to be false packages/kernel-store/src/sqlite/wasm.test.ts > stops believing it is in a transaction when the commit fails Co-Authored-By: Claude Opus 5 --- packages/kernel-store/src/sqlite/wasm.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index 8e82a00bc9..fa4e2c8a80 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -599,6 +599,44 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); + // FAILING REPRO — see the commit message for this test. + // + // `commitIfNeeded` still steps the COMMIT before clearing `_inTx`, the exact + // ordering `rollbackIfNeeded` was corrected to avoid. A COMMIT that throws + // therefore leaves `_inTx` true against a database that may hold no + // transaction, `beginIfNeeded` is a no-op forever after, and the next + // savepoint is created outside a transaction — which commits when released + // (Agoric/agoric-sdk#8423). This is the crank's commit point, so the writes + // that leak are a whole crank's. + // + // The comment above `stops believing it is in a transaction when the abort + // fails too` calls a failed abort "the one case that can leave `_inTx` + // disagreeing with the database". This is the second case. + it('stops believing it is in a transaction when the commit fails', async () => { + const db = await makeSQLKernelDatabase({}); + mockDb._inTx = true; + mockDb._spStack = ['point1']; + // The RELEASE goes through `exec` and succeeds; COMMIT is the first + // prepared statement this path steps, and it is what fails. + mockStatement.step.mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => db.releaseSavepoint('point1')).toThrowError( + 'disk I/O error', + ); + + expect(mockDb._inTx).toBe(false); + + // And so the next savepoint gets a transaction of its own rather than + // being created bare. + mockDb.exec.mockClear(); + mockStatement.step.mockClear(); + db.createSavepoint('next'); + expect(mockStatement.step).toHaveBeenCalledOnce(); + expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); + }); + it('supports nested savepoints', async () => { const db = await makeSQLKernelDatabase({}); db.createSavepoint('outer'); From 9d78120e527175f976f11dd55d49562f892e568c Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:52:51 -0400 Subject: [PATCH 18/34] test(ocap-kernel): pin the endCrank failure that buries the real error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 fixes one error-masking path at the start of a dying crank and opens another at its end. Before the two-savepoint scheme, `rollbackCrank('start')` emptied `ctx.savepoints`, so `endCrank` -> `releaseAllSavepoints` was a guaranteed no-op on the dying path: nothing to release, nothing that could throw. Now `rollbackCrank('delivery')` truncates to the ordinal and leaves `['crank']` behind (crank.ts:56, deliberately — that is what keeps the transaction open for the work an aborted crank still owes). So `endCrank` issues a real `RELEASE t0`, which commits, which can fail. `#runLoop` calls it from a bare `finally`: } finally { this.#kernelStore.endCrank(); ... } A throw there replaces the pending exception. The disk error that actually killed the kernel is discarded — not demoted to `cause`, discarded — and `run()` rejects with the release failure instead. `#failRunLoop` records that, so `getRunLoopStatus().detail` loses the root cause too, and `onRunLoopFailure` — what the daemon logs as fatal — gets the wrong error. A/B against origin/main with the same repro: main reports `crank exploded`, this branch reports `database is gone` with `cause: undefined`. This is the same class of bug as the `No such savepoint: t0` masking that 82b88ce62 fixes, and the same class the `reports both failures when the rollback also fails` test above already guards on the other path. ## What we hope to see instead Whatever names the release failure, the error that killed the crank stays reachable. The rollback path already has the shape to copy: throw new Error( `Run loop died and its crank could not be rolled back: ${...}`, { cause: error }, ); The assertion is deliberately fix-agnostic — it walks the `cause` chain — so either wrapping `endCrank`'s failure with the original as `cause`, or reporting it and rethrowing the original, will satisfy it. ## Current failure AssertionError: expected [ Error: database is gone ] to include Error: crank exploded packages/ocap-kernel/src/KernelQueue.test.ts > reports both failures when endCrank also fails Co-Authored-By: Claude Opus 5 --- packages/ocap-kernel/src/KernelQueue.test.ts | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index a04fa05feb..c8dc3c5710 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -25,6 +25,24 @@ vi.mock('@endo/promise-kit', () => ({ */ const STOP_RUN_LOOP = 'test: stop run loop'; +/** + * Collect an error and every error reachable through its `cause` chain, so that + * a test can assert a root cause survived without pinning how its reporter + * chose to wrap it. + * + * @param error - The error to walk. + * @returns The chain, outermost first. + */ +const causeChain = (error: unknown): unknown[] => { + const chain: unknown[] = []; + let current = error; + while (current instanceof Error) { + chain.push(current); + current = current.cause; + } + return chain; +}; + describe('KernelQueue', () => { let kernelStore: KernelStore; let kernelQueue: KernelQueue; @@ -540,6 +558,43 @@ describe('KernelQueue', () => { }); }); + // FAILING REPRO — see the commit message for this test. + // + // The companion of the case above, at the other end of the crank. Since the + // delivery rollback now spares `crank`, `endCrank`'s release is a real + // RELEASE + COMMIT on the dying path where it used to be a no-op, and + // `#runLoop` calls it from a bare `finally` — so when it throws it replaces + // the error that killed the kernel instead of being reported alongside it. + it('reports both failures when endCrank also fails', async () => { + ( + kernelStore.runQueueLength as unknown as MockInstance + ).mockReturnValueOnce(1); + (kernelStore.dequeueRun as unknown as MockInstance).mockReturnValueOnce({ + type: 'send', + target: 'ko123', + message: {} as KernelMessage, + }); + (kernelStore.endCrank as unknown as MockInstance).mockImplementation( + () => { + throw new Error('database is gone'); + }, + ); + const crankError = new Error('crank exploded'); + const deliver = vi.fn().mockRejectedValue(crankError); + + const failure = await kernelQueue + .run(deliver) + .catch((error: unknown) => error); + + // However the release failure is named, the error that actually killed the + // kernel has to stay reachable — as the rollback path already manages. + expect(causeChain(failure)).toContain(crankError); + expect(kernelQueue.getRunLoopStatus()).toMatchObject({ + state: 'failed', + detail: expect.stringContaining('crank exploded'), + }); + }); + // `rollbackCrank` discards the savepoint even when its database call throws, // so a second attempt could only report a missing savepoint. Without the // `finally` that records the attempt, the abort path leaves the flag unset, From ad07e8a020d7c2b003a73a560dea05b18952b78b Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:53:48 -0400 Subject: [PATCH 19/34] test(kernel-node-runtime): pin the kernel store's missing logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 replaces four silently-swallowed aborts with `logger?.error(...)` in the SQLite drivers, and its description says: "Four swallowed aborts were silent. Now logged." They are not. No production call site passes a `logger` to `makeSQLKernelDatabase`, so every one of those calls is dead code: packages/kernel-node-runtime/src/kernel/make-kernel.ts:63 packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts:47 packages/kernel-test-local/src/lms-chat.ts:30 packages/kernel-node-runtime/test/helpers/remote-comms.ts:172 `make-kernel.ts` is the clearest case: it builds a `rootLogger` and hands sub-loggers to `NodejsPlatformServices` and to `Kernel.make`, then constructs the store with `{ dbFilename }` alone. The store is the one collaborator that gets no logger. Nor does any test pass one, which is why the gap survived review. This matters more than a missing log line. On the nodejs driver a failed abort leaves `db.inTransaction` true with nothing that will ever commit or abort it, so later writes on that connection join a transaction that vanishes on close. The driver's own comment concedes "Nothing here can repair that" — the log is the entire remedy, and it does not reach anyone. `logger?.error` is the right convention for this package; the injection is what is missing. ## What we hope to see instead `makeKernel` passes a tagged sub-logger to `makeSQLKernelDatabase`, as it already does for its other collaborators — something like `rootLogger.subLogger({ tags: ['store'] })`. The other three call sites want the same treatment, and are worth covering once this one is fixed. ## Current failure AssertionError: expected "vi.fn()" to be called with arguments: [ ObjectContaining{…} ] - "logger": Any, packages/kernel-node-runtime/src/kernel/make-kernel.test.ts > gives the kernel store a logger Co-Authored-By: Claude Opus 5 --- .../src/kernel/make-kernel.test.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts index 57b0293d65..98490dd65f 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts @@ -1,3 +1,5 @@ +import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; +import { Logger } from '@metamask/logger'; import { Kernel } from '@metamask/ocap-kernel'; import { describe, expect, it, vi } from 'vitest'; @@ -8,7 +10,9 @@ vi.mock('@metamask/kernel-store/sqlite/nodejs', async () => { '../../../ocap-kernel/test/storage.ts' ); return { - makeSQLKernelDatabase: makeMapKernelDatabase, + // Wrapped so that a test can see what the database was constructed with, + // while still getting a real store back. + makeSQLKernelDatabase: vi.fn(makeMapKernelDatabase), }; }); @@ -18,4 +22,18 @@ describe('makeKernel', () => { expect(kernel).toBeInstanceOf(Kernel); }); + + // FAILING REPRO — see the commit message for this test. + // + // The kernel store is the only collaborator `makeKernel` builds without + // handing it a logger, so every `logger?.` call inside the SQLite driver is + // dead code in production — including the four abort failures #1012 added + // logging for. + it('gives the kernel store a logger', async () => { + await makeKernel({}); + + expect(makeSQLKernelDatabase).toHaveBeenCalledWith( + expect.objectContaining({ logger: expect.any(Logger) }), + ); + }); }); From 997c10d1534059c08ef6e394e9c302b1ecf10f8e Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:56:04 -0400 Subject: [PATCH 20/34] test(ocap-kernel): pin the release failure lost at the remote savepoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing repro, not a fix. ## The issue #1012 hardens `releaseSavepoint` so that a failed `RELEASE` discards the enclosing transaction, clearing the driver's `_spStack` on the way. Two callers it does not touch depend on the old behaviour, and both are now worse off than before the change. `RemoteHandle.handleRemoteMessage` releases inside the `try` and rolls back in the `catch`: this.#kernelStore.setRemoteHighestReceivedSeq(this.remoteId, seq); this.#kernelStore.releaseSavepoint(savepointName); // fails } catch (error) { this.#kernelStore.rollbackSavepoint(savepointName); // "No such savepoint" throw error; // never reached } Since the release already cleared the stack, the rollback throws `No such savepoint: receive_r0_1`, which escapes the `catch` and replaces the real failure. Not demoted to `cause` — replaced. `RemoteManager` has the same shape at its `peerIncarnation_*` savepoint. A/B verified against origin/main with a real driver: main's rollback succeeds and `database or disk is full` propagates; on this branch the caller gets the missing-savepoint error instead. So the PR description's "the release failure still propagates" holds for the crank path it fixed and not for these two. `crank.ts:57-63` shows the author recognised exactly this hazard — a stale savepoint list producing `No such savepoint` over the real error — and fixed it for the crank only. The remote paths were missed because nothing exercised them. Note the secondary effect these tests don't reach: `ctx.savepoints` still lists the crank's own savepoints after this, so the next `endCrank` throws `No such savepoint: t0` over whatever is left of the failure. ## What we hope to see instead The failure the database reported is what reaches the caller. Any of these does it, and the assertion doesn't care which: - move the release out of the `try`, so a release failure isn't followed by a rollback attempt at all - have the `catch` tolerate a rollback that reports a savepoint already discarded, rethrowing the original either way - make the driver's discard leave the name rollback-able as a no-op The mock models the drivers' bookkeeping rather than the expected outcome, so it is `RemoteHandle`'s error handling under test, not the mock's. ## Current failure AssertionError: expected Error: No such savepoint: receive_r0_1 to be Error: database or disk is full packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts > reports the release failure rather than a missing savepoint Co-Authored-By: Claude Opus 5 --- .../src/remotes/kernel/RemoteHandle.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index 4f53572c0d..ea201ada21 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -220,6 +220,49 @@ describe('RemoteHandle', () => { }); }); + // FAILING REPRO — see the commit message for this test. + // + // `handleRemoteMessage` releases its savepoint inside the `try` and rolls + // back in the `catch`. #1012 made a failed `RELEASE` discard the whole + // savepoint stack, so that rollback now reports a savepoint that no longer + // exists, and it throws out of the `catch` in place of the failure that + // brought it there. + it('reports the release failure rather than a missing savepoint', async () => { + // The drivers' bookkeeping as #1012 leaves it, verified against both: a + // failed RELEASE clears `_spStack`, and rolling back a name that is not on + // it throws `No such savepoint`. + const savepoints: string[] = []; + const releaseFailure = new Error('database or disk is full'); + mockKernelStore = { + ...mockKernelStore, + createSavepoint: (name: string) => { + savepoints.push(name); + }, + releaseSavepoint: () => { + savepoints.length = 0; + throw releaseFailure; + }, + rollbackSavepoint: (name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }, + } as KernelStore; + const remote = makeRemote(); + + const delivery = JSON.stringify({ + seq: 1, + method: 'deliver', + params: ['bringOutYourDead'], + }); + + // The error an operator needs is the one the database gave, not the + // bookkeeping artefact of trying to clean up after it. + await expect(remote.handleRemoteMessage(delivery)).rejects.toBe( + releaseFailure, + ); + }); + // A dead run loop will never deliver the message, and `handleRemoteMessage` // rolls back without advancing the received sequence number, so the peer // retries and gives up rather than being acknowledged by a black hole. From 251e58ff6bc3be38bdf1cb6c32132e1d92627508 Mon Sep 17 00:00:00 2001 From: grypez <143971198+grypez@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:09:39 -0400 Subject: [PATCH 21/34] test: tighten the four repros after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No change to what any of them proves; all four still fail for the reasons their own commits describe. - `RemoteHandle`: assert the rollback is still *attempted*. Without this, deleting the rollback from the `catch` outright would turn the test green, which is not the fix — a `RELEASE` that failed for a reason of its own may well have left the savepoint standing. - `RemoteHandle`: drop an unnecessary `as KernelStore` cast, and say why the store is replaced wholesale rather than having its methods assigned over (`makeKernelStore` hardens what it returns). - `make-kernel`: note that `kernel-worker.ts` omits the logger too, so the wasm driver's pair of `logger?.error` calls stays dead even once this test passes. Use `vi.mocked`, as the sibling `make-kernel-options.test.ts` does. - `causeChain` returns `Error[]`; every element is already narrowed by the loop guard. - Drop "see the commit message for this test" from the four comment blocks: each stands alone, and the reference would not survive a squash-merge. Restate the claim the wasm comment made by citing a neighbouring test's title, which would have broken silently on rename. Co-Authored-By: Claude Opus 5 --- .../src/kernel/make-kernel.test.ts | 7 +++--- packages/kernel-store/src/sqlite/wasm.test.ts | 8 +++---- packages/ocap-kernel/src/KernelQueue.test.ts | 6 ++--- .../src/remotes/kernel/RemoteHandle.test.ts | 23 ++++++++++++------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts index 98490dd65f..672313a02a 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.test.ts @@ -23,16 +23,17 @@ describe('makeKernel', () => { expect(kernel).toBeInstanceOf(Kernel); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // The kernel store is the only collaborator `makeKernel` builds without // handing it a logger, so every `logger?.` call inside the SQLite driver is // dead code in production — including the four abort failures #1012 added - // logging for. + // logging for. `kernel-worker.ts` omits it too, which keeps the wasm driver's + // pair dead even once this passes. it('gives the kernel store a logger', async () => { await makeKernel({}); - expect(makeSQLKernelDatabase).toHaveBeenCalledWith( + expect(vi.mocked(makeSQLKernelDatabase)).toHaveBeenCalledWith( expect.objectContaining({ logger: expect.any(Logger) }), ); }); diff --git a/packages/kernel-store/src/sqlite/wasm.test.ts b/packages/kernel-store/src/sqlite/wasm.test.ts index fa4e2c8a80..1cf496dc95 100644 --- a/packages/kernel-store/src/sqlite/wasm.test.ts +++ b/packages/kernel-store/src/sqlite/wasm.test.ts @@ -599,7 +599,7 @@ describe('makeSQLKernelDatabase', () => { expect(mockDb.exec).toHaveBeenCalledWith('SAVEPOINT next'); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // `commitIfNeeded` still steps the COMMIT before clearing `_inTx`, the exact // ordering `rollbackIfNeeded` was corrected to avoid. A COMMIT that throws @@ -609,9 +609,9 @@ describe('makeSQLKernelDatabase', () => { // (Agoric/agoric-sdk#8423). This is the crank's commit point, so the writes // that leak are a whole crank's. // - // The comment above `stops believing it is in a transaction when the abort - // fails too` calls a failed abort "the one case that can leave `_inTx` - // disagreeing with the database". This is the second case. + // A failed abort is therefore not, as the abort case above claims, the one + // case that can leave `_inTx` disagreeing with the database. This is the + // second. it('stops believing it is in a transaction when the commit fails', async () => { const db = await makeSQLKernelDatabase({}); mockDb._inTx = true; diff --git a/packages/ocap-kernel/src/KernelQueue.test.ts b/packages/ocap-kernel/src/KernelQueue.test.ts index c8dc3c5710..9e4de322eb 100644 --- a/packages/ocap-kernel/src/KernelQueue.test.ts +++ b/packages/ocap-kernel/src/KernelQueue.test.ts @@ -33,8 +33,8 @@ const STOP_RUN_LOOP = 'test: stop run loop'; * @param error - The error to walk. * @returns The chain, outermost first. */ -const causeChain = (error: unknown): unknown[] => { - const chain: unknown[] = []; +const causeChain = (error: unknown): Error[] => { + const chain: Error[] = []; let current = error; while (current instanceof Error) { chain.push(current); @@ -558,7 +558,7 @@ describe('KernelQueue', () => { }); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // The companion of the case above, at the other end of the crank. Since the // delivery rollback now spares `crank`, `endCrank`'s release is a real diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index ea201ada21..cb0142e064 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -220,7 +220,7 @@ describe('RemoteHandle', () => { }); }); - // FAILING REPRO — see the commit message for this test. + // FAILING REPRO. // // `handleRemoteMessage` releases its savepoint inside the `try` and rolls // back in the `catch`. #1012 made a failed `RELEASE` discard the whole @@ -230,9 +230,16 @@ describe('RemoteHandle', () => { it('reports the release failure rather than a missing savepoint', async () => { // The drivers' bookkeeping as #1012 leaves it, verified against both: a // failed RELEASE clears `_spStack`, and rolling back a name that is not on - // it throws `No such savepoint`. + // it throws `No such savepoint`. Replacing the store wholesale rather than + // assigning over its methods because `makeKernelStore` hardens what it + // returns. const savepoints: string[] = []; const releaseFailure = new Error('database or disk is full'); + const rollbackSavepoint = vi.fn((name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }); mockKernelStore = { ...mockKernelStore, createSavepoint: (name: string) => { @@ -242,12 +249,8 @@ describe('RemoteHandle', () => { savepoints.length = 0; throw releaseFailure; }, - rollbackSavepoint: (name: string) => { - if (!savepoints.includes(name)) { - throw new Error(`No such savepoint: ${name}`); - } - }, - } as KernelStore; + rollbackSavepoint, + }; const remote = makeRemote(); const delivery = JSON.stringify({ @@ -261,6 +264,10 @@ describe('RemoteHandle', () => { await expect(remote.handleRemoteMessage(delivery)).rejects.toBe( releaseFailure, ); + // Still attempted, so that abandoning the rollback is not a way to pass + // this test: a release that failed for a reason of its own may well have + // left the savepoint standing. + expect(rollbackSavepoint).toHaveBeenCalledWith('receive_r0_1'); }); // A dead run loop will never deliver the message, and `handleRemoteMessage` From 86d16dc490b66906d8629ba08bc9a1ca6a0d0597 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:17 +0200 Subject: [PATCH 22/34] fix(kernel-store): clear `_inTx` before the COMMIT, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rollbackIfNeeded` was corrected for this ordering already; `commitIfNeeded` still stepped the COMMIT first. `_inTx` is tracked in the driver rather than read from SQLite, so a throwing COMMIT wedged it true: `beginIfNeeded` became a permanent no-op, and the next savepoint was created bare — where its RELEASE commits (Agoric/agoric-sdk#8423) and no later rollback could undo the delivery. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-store/CHANGELOG.md | 4 ++-- packages/kernel-store/src/sqlite/wasm.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index f49786942c..f238b66460 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -14,8 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The rollback failure is still what gets thrown, even if aborting the transaction fails too - `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted. The nodejs driver reads `db.inTransaction` and was never affected +- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected - An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) ## [0.6.0] diff --git a/packages/kernel-store/src/sqlite/wasm.ts b/packages/kernel-store/src/sqlite/wasm.ts index a43092732b..a047d6cbcf 100644 --- a/packages/kernel-store/src/sqlite/wasm.ts +++ b/packages/kernel-store/src/sqlite/wasm.ts @@ -199,9 +199,13 @@ export async function makeSQLKernelDatabase({ */ function commitIfNeeded(): void { if (db._inTx && db._spStack.length === 0) { + // Cleared before the commit is attempted, for the reason `rollbackIfNeeded` + // gives: a throwing COMMIT would otherwise wedge `_inTx` true, and every + // later savepoint would be created bare — where its RELEASE commits + // (Agoric/agoric-sdk#8423) and no rollback can undo the delivery. + db._inTx = false; sqlCommitTransaction.step(); sqlCommitTransaction.reset(); - db._inTx = false; } } From 8adaae59b96530b30687d2f3b3f9915f70f21b04 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:28 +0200 Subject: [PATCH 23/34] fix(ocap-kernel): stop `endCrank` burying the error that killed the run loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path, where it used to be a no-op. `#runLoop` called it from a bare `finally`, so a failing one silently replaced whatever killed the kernel — and only `error.message` crosses the wire, so the real failure reached neither `getStatus` nor the daemon log. Report it with the crank's failure as the `cause`, the shape the rollback path already uses. The in-flight error is boxed rather than left `undefined`, so a crank that threw `undefined` stays distinguishable from one that did not throw. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelQueue.ts | 33 ++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/ocap-kernel/src/KernelQueue.ts b/packages/ocap-kernel/src/KernelQueue.ts index 8c598dfcbf..f4e1eff9f0 100644 --- a/packages/ocap-kernel/src/KernelQueue.ts +++ b/packages/ocap-kernel/src/KernelQueue.ts @@ -122,6 +122,9 @@ export class KernelQueue { ): Promise { for (;;) { let wakeUpPromise: Promise | undefined; + // Boxed rather than left `undefined`, so that a crank which threw + // `undefined` is still distinguishable from one that did not throw. + let crankFailure: { error: unknown } | undefined; this.#kernelStore.startCrank(); this.#crankRollbackAttempted = false; @@ -170,8 +173,11 @@ export class KernelQueue { } throw error; } + } catch (error) { + crankFailure = { error }; + throw error; } finally { - this.#kernelStore.endCrank(); + this.#endCrank(crankFailure); if (wakeUpPromise) { await wakeUpPromise; } @@ -179,6 +185,31 @@ export class KernelQueue { } } + /** + * End the crank without losing the error that is already unwinding. Since the + * delivery rollback now spares `crank`, `endCrank`'s release is a real RELEASE + * and COMMIT on the dying path, where it used to be a no-op — and from a bare + * `finally` a failing one would silently replace whatever killed the kernel. + * + * @param crankFailure - The error already in flight, if the crank threw. + * @param crankFailure.error - That error. + */ + #endCrank(crankFailure?: { error: unknown }): void { + try { + this.#kernelStore.endCrank(); + } catch (endCrankError) { + if (!crankFailure) { + throw endCrankError; + } + // The original failure stays the `cause`, as on the rollback path; the + // release failure is named here. + throw new Error( + `Run loop died and its crank could not be ended: ${String(endCrankError)}`, + { cause: crankFailure.error }, + ); + } + } + /** * Record the death of the run loop and fail the kernel's own message-result * subscriptions, which would otherwise hang forever. Kernel promises in the From fdf9d6ac898e5f5f5b48ced2b8f04f5133e11cd6 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:40 +0200 Subject: [PATCH 24/34] fix(ocap-kernel): report the remote release failure, not a missing savepoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RemoteHandle.handleRemoteMessage` releases its savepoint inside the `try` and rolls back in the `catch`. Now that a failed RELEASE discards the whole savepoint stack, that rollback names a savepoint that is already gone and threw `No such savepoint` out of the `catch` in place of the database failure that brought it there — not even as `cause`. Log the rollback failure instead of throwing it. The rollback is still attempted, because a release that failed for a reason of its own may well have left the savepoint standing. `RemoteManager`'s `peerIncarnation_*` savepoint has the identical shape and had no coverage of it at all, so a fix applied here and forgotten there would have left its suite green. Fixed alike, and the savepoint-stack model both tests drive the drivers with is now shared. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/remotes/kernel/RemoteHandle.test.ts | 35 +++--------- .../src/remotes/kernel/RemoteHandle.ts | 14 ++++- .../src/remotes/kernel/RemoteManager.test.ts | 40 ++++++++++++++ .../src/remotes/kernel/RemoteManager.ts | 11 +++- packages/ocap-kernel/test/savepoint-stack.ts | 53 +++++++++++++++++++ 5 files changed, 123 insertions(+), 30 deletions(-) create mode 100644 packages/ocap-kernel/test/savepoint-stack.ts diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts index cb0142e064..90f4ef3bc5 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { RemoteHandle } from './RemoteHandle.ts'; import { createMockRemotesFactory } from '../../../test/remotes-mocks.ts'; +import { withFailingSavepointRelease } from '../../../test/savepoint-stack.ts'; import type { KernelQueue } from '../../KernelQueue.ts'; import type { KernelStore } from '../../store/index.ts'; import { parseRef } from '../../store/utils/parse-ref.ts'; @@ -220,37 +221,15 @@ describe('RemoteHandle', () => { }); }); - // FAILING REPRO. - // // `handleRemoteMessage` releases its savepoint inside the `try` and rolls // back in the `catch`. #1012 made a failed `RELEASE` discard the whole - // savepoint stack, so that rollback now reports a savepoint that no longer - // exists, and it throws out of the `catch` in place of the failure that - // brought it there. + // savepoint stack, so that rollback reports a savepoint that no longer + // exists, and left unguarded it throws out of the `catch` in place of the + // failure that brought it there. it('reports the release failure rather than a missing savepoint', async () => { - // The drivers' bookkeeping as #1012 leaves it, verified against both: a - // failed RELEASE clears `_spStack`, and rolling back a name that is not on - // it throws `No such savepoint`. Replacing the store wholesale rather than - // assigning over its methods because `makeKernelStore` hardens what it - // returns. - const savepoints: string[] = []; - const releaseFailure = new Error('database or disk is full'); - const rollbackSavepoint = vi.fn((name: string) => { - if (!savepoints.includes(name)) { - throw new Error(`No such savepoint: ${name}`); - } - }); - mockKernelStore = { - ...mockKernelStore, - createSavepoint: (name: string) => { - savepoints.push(name); - }, - releaseSavepoint: () => { - savepoints.length = 0; - throw releaseFailure; - }, - rollbackSavepoint, - }; + const failing = withFailingSavepointRelease(mockKernelStore); + const { releaseFailure, rollbackSavepoint } = failing; + mockKernelStore = failing.kernelStore; const remote = makeRemote(); const delivery = JSON.stringify({ diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts index d92359b5c0..34181df661 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteHandle.ts @@ -1032,7 +1032,19 @@ export class RemoteHandle implements EndpointHandle { this.#kernelStore.releaseSavepoint(savepointName); } catch (error) { // Rollback on any error - in-memory state unchanged since we didn't update it yet - this.#kernelStore.rollbackSavepoint(savepointName); + try { + this.#kernelStore.rollbackSavepoint(savepointName); + } catch (rollbackError) { + // The release above is inside the `try`, and a failed RELEASE discards + // the whole savepoint stack — so this rollback reports a savepoint that + // is already gone, over the database failure an operator actually needs. + // Still attempted, because a release that failed for a reason of its own + // may well have left the savepoint standing. + this.#logger.error( + `${this.#peerId.slice(0, 8)}:: rollback of ${savepointName} failed`, + rollbackError, + ); + } throw error; } diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts index 29ca0b2da7..c3257d85df 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.test.ts @@ -4,6 +4,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import * as remoteComms from './remote-comms.ts'; import { RemoteManager } from './RemoteManager.ts'; import { createMockRemotesFactory } from '../../../test/remotes-mocks.ts'; +import { withFailingSavepointRelease } from '../../../test/savepoint-stack.ts'; import { makeMapKernelDatabase } from '../../../test/storage.ts'; import type { KernelQueue } from '../../KernelQueue.ts'; import { makeKernelStore } from '../../store/index.ts'; @@ -948,5 +949,44 @@ describe('RemoteManager', () => { // mutations would otherwise drift from the rolled-back kv view. expect(finalizeSpy).not.toHaveBeenCalled(); }); + + // The same shape `RemoteHandle.handleRemoteMessage` has: the release sits + // inside the `try` and the rollback in the `catch`, so a failed RELEASE — + // which discards the whole savepoint stack — leaves the rollback naming a + // savepoint that is gone. + it('reports the release failure rather than a missing savepoint', async () => { + const peerId = 'peer-whose-release-fails'; + const { + kernelStore: failingStore, + releaseFailure, + rollbackSavepoint, + } = withFailingSavepointRelease(kernelStore); + remoteManager = new RemoteManager({ + platformServices: mockPlatformServices, + kernelStore: failingStore, + kernelQueue: mockKernelQueue, + logger, + }); + remoteManager.setMessageHandler(vi.fn()); + await remoteManager.initRemoteComms(); + const onIncarnationChange = vi + .mocked(remoteComms.initRemoteComms) + .mock.calls.at(-1)?.[8] as ( + peerId: string, + observedIncarnation: string, + ) => Promise; + + // The error an operator needs is the one the database gave, not the + // bookkeeping artefact of trying to clean up after it. + await expect(onIncarnationChange(peerId, 'incarnation-A')).rejects.toBe( + releaseFailure, + ); + // Still attempted, so that abandoning the rollback is not a way to pass: + // a release that failed for a reason of its own may well have left the + // savepoint standing. + expect(rollbackSavepoint).toHaveBeenCalledWith( + `peerIncarnation_${peerId}`, + ); + }); }); }); diff --git a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts index d9b7c3c946..9a477cf398 100644 --- a/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts +++ b/packages/ocap-kernel/src/remotes/kernel/RemoteManager.ts @@ -261,7 +261,16 @@ export class RemoteManager { this.#kernelStore.setPeerIncarnation(peerId, observedIncarnation); this.#kernelStore.releaseSavepoint(savepoint); } catch (error) { - this.#kernelStore.rollbackSavepoint(savepoint); + try { + this.#kernelStore.rollbackSavepoint(savepoint); + } catch (rollbackError) { + // The release above is inside the `try`, and a failed RELEASE discards + // the whole savepoint stack — so this rollback reports a savepoint that + // is already gone, over the database failure an operator actually needs. + // Still attempted, because a release that failed for a reason of its own + // may well have left the savepoint standing. + this.#logger?.error(`Rollback of ${savepoint} failed`, rollbackError); + } throw error; } diff --git a/packages/ocap-kernel/test/savepoint-stack.ts b/packages/ocap-kernel/test/savepoint-stack.ts new file mode 100644 index 0000000000..c4b7e587fc --- /dev/null +++ b/packages/ocap-kernel/test/savepoint-stack.ts @@ -0,0 +1,53 @@ +import type { MockedFunction } from 'vitest'; +import { vi } from 'vitest'; + +import type { KernelStore } from '../src/store/index.ts'; + +export type FailingReleaseStore = { + /** The store to hand the subject under test. */ + kernelStore: KernelStore; + /** The error every `releaseSavepoint` throws. */ + releaseFailure: Error; + /** Exposed so a test can assert the rollback was still attempted. */ + rollbackSavepoint: MockedFunction<(name: string) => void>; +}; + +/** + * Wrap a kernel store so that `releaseSavepoint` fails the way a full disk does, + * modelling the drivers' bookkeeping as of #1012 and verified against both: a + * failed RELEASE clears the savepoint stack, and rolling back a name that is no + * longer on it throws `No such savepoint`. What is under test is therefore the + * caller's error handling, not an expected outcome baked into the mock. + * + * Replaces the store wholesale rather than assigning over its methods, because + * `makeKernelStore` hardens what it returns. + * + * @param kernelStore - The store to wrap. + * @returns The wrapped store and the handles a test needs to assert against. + */ +export function withFailingSavepointRelease( + kernelStore: KernelStore, +): FailingReleaseStore { + const savepoints: string[] = []; + const releaseFailure = new Error('database or disk is full'); + const rollbackSavepoint = vi.fn((name: string) => { + if (!savepoints.includes(name)) { + throw new Error(`No such savepoint: ${name}`); + } + }); + return { + kernelStore: { + ...kernelStore, + createSavepoint: (name: string) => { + savepoints.push(name); + }, + releaseSavepoint: () => { + savepoints.length = 0; + throw releaseFailure; + }, + rollbackSavepoint, + }, + releaseFailure, + rollbackSavepoint, + }; +} From 80cb871bbc6f272c20b2eab9a44b5f67c43b111f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:01:54 +0200 Subject: [PATCH 25/34] test(ocap-kernel): pin the in-memory revert against a failed crank rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rollbackCrank` gained two pieces of work that compose the wrong way round if the failure path simply rethrows: forgetting every savepoint, and reverting the caches a database rollback cannot reach. A failed rollback discards the whole transaction, so the database has moved back at least as far as a successful rollback would have taken it and those caches are at least as stale — the one case where skipping the revert leaves the consumed GC action lost and krefs queued for a collection that then kills the run loop. The second test pins the other direction: reverting must not become a way to lose the database error either. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 4 ++ .../src/store/methods/crank.test.ts | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 0276aa11c9..6cb41e0a0b 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -69,6 +69,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately +- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path where it used to be a no-op — and the run loop called it from a bare `finally` +- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) + - Both release inside the `try` and roll back in the `catch`, so once a failed RELEASE discarded the whole savepoint stack, the rollback reported a savepoint that no longer existed in place of the real error. The rollback is still attempted: a release that failed for a reason of its own may well have left the savepoint standing - Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel - `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) diff --git a/packages/ocap-kernel/src/store/methods/crank.test.ts b/packages/ocap-kernel/src/store/methods/crank.test.ts index 630444e850..426890d209 100644 --- a/packages/ocap-kernel/src/store/methods/crank.test.ts +++ b/packages/ocap-kernel/src/store/methods/crank.test.ts @@ -176,6 +176,48 @@ describe('crank methods', () => { crankMethods.endCrank(); expect(kdb.releaseSavepoint).not.toHaveBeenCalled(); }); + + // The two halves of this function compose the wrong way round if the + // failure path simply rethrows: a failed rollback discards the whole + // transaction, so the database has moved back at least as far as a + // successful rollback would have taken it and the caches it left behind are + // at least as stale. + it('reverts the caches the database cannot reach even when the rollback fails', () => { + context.inCrank = true; + context.maybeFreeKrefs.add('kp1'); + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw new Error('disk I/O error'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + 'disk I/O error', + ); + + expect(context.refreshCachedValues).toHaveBeenCalled(); + expect(context.refreshRunQueue).toHaveBeenCalled(); + expect(context.runQueueLengthCache).toBe(-1); + expect([...context.maybeFreeKrefs]).toStrictEqual([]); + }); + + // Reverting must not become a way to lose the database error either. + it('keeps the rollback failure as the cause when reverting also fails', () => { + context.inCrank = true; + const rollbackFailure = new Error('disk I/O error'); + crankMethods.createCrankSavepoint('crank'); + crankMethods.createCrankSavepoint('delivery'); + vi.mocked(kdb.rollbackSavepoint).mockImplementationOnce(() => { + throw rollbackFailure; + }); + vi.mocked(context.refreshCachedValues).mockImplementationOnce(() => { + throw new Error('database is gone'); + }); + + expect(() => crankMethods.rollbackCrank('delivery')).toThrow( + expect.objectContaining({ cause: rollbackFailure }), + ); + }); }); describe('endCrank', () => { From 90b42e9cbf2f48b81d3dd9e23dabad8d0212cee3 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:02:03 +0200 Subject: [PATCH 26/34] fix(runtimes): give the kernel store a logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No production call site passed one, so every `logger?.` call in the SQLite drivers was dead code — including the aborts they report while discarding a transaction, which fire on exactly the path where the kernel is already dying and a diagnostic is worth most. The browser worker has a module-level `Logger` already, so both drivers are covered rather than only the nodejs one. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 1 + .../src/kernel-worker/kernel-worker.ts | 5 ++++- packages/kernel-node-runtime/CHANGELOG.md | 1 + packages/kernel-node-runtime/src/kernel/make-kernel.ts | 5 ++++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index e101fdcb30..c55d0b2a8f 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts b/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts index 832a57a906..afffa24e32 100644 --- a/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts +++ b/packages/kernel-browser-runtime/src/kernel-worker/kernel-worker.ts @@ -44,7 +44,10 @@ async function main(): Promise { isJsonRpcMessage, ), PlatformServicesClient.make(globalThis as PostMessageTarget), - makeSQLKernelDatabase({ dbFilename: DB_FILENAME }), + makeSQLKernelDatabase({ + dbFilename: DB_FILENAME, + logger: logger.subLogger({ tags: ['kernel-store'] }), + }), ]); setupConsoleForwarding({ diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index 5a75a45781..d935c4e961 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-node-runtime/src/kernel/make-kernel.ts b/packages/kernel-node-runtime/src/kernel/make-kernel.ts index 901e79982a..529eb7af2d 100644 --- a/packages/kernel-node-runtime/src/kernel/make-kernel.ts +++ b/packages/kernel-node-runtime/src/kernel/make-kernel.ts @@ -60,7 +60,10 @@ export async function makeKernel({ }); // Initialize kernel store. - const kernelDatabase = await makeSQLKernelDatabase({ dbFilename }); + const kernelDatabase = await makeSQLKernelDatabase({ + dbFilename, + logger: rootLogger.subLogger({ tags: ['kernel-store'] }), + }); // Create and start kernel. const kernel = await Kernel.make(platformServicesClient, kernelDatabase, { From 0892784d24707aae5d9e9f2fdb006fe027095250 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:28:46 +0200 Subject: [PATCH 27/34] chore: cite this PR in the changelogs The crank-transaction and rollback work landed here rather than in #1012, which is closed and replaced. #1021 is a placeholder until the PR exists. Co-Authored-By: Claude Opus 5 (1M context) --- packages/kernel-browser-runtime/CHANGELOG.md | 2 +- packages/kernel-node-runtime/CHANGELOG.md | 2 +- packages/kernel-store/CHANGELOG.md | 6 +++--- packages/ocap-kernel/CHANGELOG.md | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/kernel-browser-runtime/CHANGELOG.md b/packages/kernel-browser-runtime/CHANGELOG.md index c55d0b2a8f..31e6f07c54 100644 --- a/packages/kernel-browser-runtime/CHANGELOG.md +++ b/packages/kernel-browser-runtime/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- The kernel worker gives the kernel store a logger, so the wasm SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Process platform-services RPC request handlers in the background so a request handler that fires a reentrant outbound RPC (e.g. transport handshake calling back into the kernel) cannot deadlock waiting for its response ([#948](https://github.com/MetaMask/ocap-kernel/pull/948)) ## [0.6.0] diff --git a/packages/kernel-node-runtime/CHANGELOG.md b/packages/kernel-node-runtime/CHANGELOG.md index d935c4e961..54296e31c0 100644 --- a/packages/kernel-node-runtime/CHANGELOG.md +++ b/packages/kernel-node-runtime/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `makeKernel` gives the kernel store a logger, so the SQLite driver's diagnostics — including the transaction aborts it reports on the kernel's dying path — reach the log rather than nowhere ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - The RPC socket server refuses to bind a Unix socket that has a live listener, rather than unlinking it and orphaning the previous owner; stale socket files with no listener are still cleaned up automatically ([#952](https://github.com/MetaMask/ocap-kernel/pull/952)) ## [0.1.0] diff --git a/packages/kernel-store/CHANGELOG.md b/packages/kernel-store/CHANGELOG.md index f238b66460..df9b23b671 100644 --- a/packages/kernel-store/CHANGELOG.md +++ b/packages/kernel-store/CHANGELOG.md @@ -12,11 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `rollbackSavepoint` discards the enclosing transaction when `ROLLBACK TO` itself fails, instead of leaving the savepoint on its stack and the transaction open ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Nothing would ever commit or abort that transaction, so every later write on the connection silently joined it, reported success, and vanished on close. Discarding it is no wider than the caller asked for: the transaction begins with the outermost savepoint, so it holds only the work the rollback was abandoning - The rollback failure is still what gets thrown, even if aborting the transaction fails too -- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `releaseSavepoint` discards the enclosing transaction when `RELEASE` fails, as `rollbackSavepoint` already did for `ROLLBACK TO` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Otherwise the savepoint stayed on the stack and the transaction open with nothing left to commit or abort it. The release failure is still what gets thrown -- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- The wasm driver clears `_inTx` when aborting or committing a transaction throws, instead of believing it is still in one ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Left true, `beginIfNeeded` became a permanent no-op and later writes autocommitted — including the next savepoint, which was then created bare, where its `RELEASE` commits and no rollback could undo the delivery. The nodejs driver reads `db.inTransaction` and was never affected -- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- An abort that fails while recovering from a failed savepoint operation is now logged in both drivers ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) ## [0.6.0] diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 6cb41e0a0b..4c363fd234 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -66,16 +66,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roll back the crank the run loop died in instead of committing it, so a restart resumes from a consistent boundary ([#1005](https://github.com/MetaMask/ocap-kernel/pull/1005)) - Because the killing item is no longer consumed, a restart re-dequeues it; an item that reliably kills a crank needs `clearState`/`reset` rather than a restart - Store state only — a crank that had already flushed its buffer settled JS-side subscriptions irreversibly -- Keep a crank's store work inside one transaction ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- Keep a crank's store work inside one transaction ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A crank now takes two savepoints, `crank` and `delivery`, and rolls back only `delivery`. Rolling back the outermost one ends the transaction, so terminating the vat and collecting garbage — which follow the rollback and must survive it — were autocommitting a statement at a time - Buffered vat outputs are flushed after that work rather than before it, so a later failure can no longer roll the crank back underneath an answer already given. Termination still settles the dying vat's own promises immediately -- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- A failing `endCrank` no longer replaces the error that killed the run loop; it is reported with that error as its `cause`, as the rollback path already did ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Now that the delivery rollback spares the `crank` savepoint, `endCrank`'s release is a real RELEASE and COMMIT on the dying path where it used to be a no-op — and the run loop called it from a bare `finally` -- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- A failing savepoint rollback in `RemoteHandle.handleRemoteMessage` and `RemoteManager`'s incarnation change is logged rather than thrown, so the failure it was cleaning up after is what reaches the caller ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - Both release inside the `try` and roll back in the `catch`, so once a failed RELEASE discarded the whole savepoint stack, the rollback reported a savepoint that no longer existed in place of the real error. The rollback is still attempted: a release that failed for a reason of its own may well have left the savepoint standing -- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- Forget a crank's savepoints when the store call that ends them fails, in both `rollbackCrank` and `endCrank` ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A failed rollback or release discards the whole transaction, savepoints included. Keeping them listed had `endCrank` throw `No such savepoint: t0` from the run loop's `finally`, over whatever really killed the kernel -- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1012](https://github.com/MetaMask/ocap-kernel/pull/1012)) +- `rollbackCrank` now also reverts the two pieces of state a database rollback cannot reach, whether or not the rollback itself succeeded: every cached stored value is re-read, and the crank's accumulated garbage-collection candidates are discarded ([#1021](https://github.com/MetaMask/ocap-kernel/pull/1021)) - A cached stored value answers reads from a closure and only writes through to the database, so reverting the database left it holding the abandoned crank's value and the next write persisted that. `processGCActionSet` takes an action out of the set before delivering it, so an aborted delivery lost the action outright instead of retrying it - `maybeFreeKrefs` lives in RAM, so nothing reverted it either. Its entries are collection candidates only because of decrements the rollback undid, and a later `collectGarbage` threw outright on a promise the rollback had deleted, killing the run loop - A failed rollback discards the whole transaction, moving the database back at least as far as a successful rollback would have — so reverting only on success left exactly the state that is least able to tolerate it From da4029849e26f8b2dc44f560924d879a93e5e59b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Aug 2026 18:39:44 +0200 Subject: [PATCH 28/34] fix(ocap-kernel): free retired exports and harden GC delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the c-list accounting fix, addressing defects found in review. An owner that stops naming its own export left the object behind. Both the delivered `retireExport` and the `retireExports`/`abandonExports` syscalls tore down the owner's c-list entry but left `owner` and `refCount` in place, with no path that could ever reclaim them: `cleanupTerminatedVat` finds krefs by walking the owner's c-list, and the collector only revisits krefs in `maybeFreeKrefs`. The records leaked, and the next collection to visit such a kref read the owner's deleted entry through `getRequired` and took the run loop down with it. New `orphanKernelObject` drops the owner mapping and hands the object to the collector, which already knows how to retire an orphan. `collectGarbage` also treats an owner with no c-list entry as orphaned rather than trusting the mapping. Reporting a dead run loop belongs to #1005, which landed on main first. It is what makes the audit usable at all: `assertRefCountsIfAuditing` throws from inside a crank, so with the failure logged and swallowed a violation's sole symptom was a test hanging to its timeout with no mention of reference counts. The `kernel-test` case here asserts that shape — the caller is told the run loop died, and the audit error rides along as the `cause`. Also: GC action delivery survives a vanished endpoint or a failed delivery instead of stopping the loop; `launchVat` tears down a worker whose kernel-side registration failed rather than stranding it; `RefCountViolation` discriminates on `kind` instead of sentinel-matching `stored`; and the store context's auditing flag no longer shares a name with `auditRefCounts()`. Tests cover the crash path, the orphan-and-collect sequence, retiring stragglers, GC-action robustness, and that a violation reaches a caller. The `item.target` charge and both `deliver|notify` early returns now have assertions that fail if the fix is reverted. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/garbage-collection.test.ts | 17 +++- .../kernel-test/src/refcount-audit.test.ts | 63 ++++++++++++ packages/ocap-kernel/src/KernelRouter.test.ts | 97 +++++++++++++++++++ packages/ocap-kernel/src/KernelRouter.ts | 56 ++++++++--- .../src/garbage-collection/gc-handlers.ts | 4 + packages/ocap-kernel/src/store/index.test.ts | 1 + .../store/methods/clist-accounting.test.ts | 70 ++++++++++++- .../ocap-kernel/src/store/methods/clist.ts | 10 +- packages/ocap-kernel/src/store/methods/gc.ts | 30 +++++- .../src/store/methods/reachable.test.ts | 56 +++++++---- .../src/store/methods/refcount-audit.test.ts | 42 +++++++- .../src/store/methods/refcount-audit.ts | 65 ++++++++----- packages/ocap-kernel/src/store/methods/vat.ts | 7 +- packages/ocap-kernel/src/store/types.ts | 2 +- packages/ocap-kernel/src/vats/VatManager.ts | 38 +++++--- .../ocap-kernel/src/vats/VatSyscall.test.ts | 1 + 16 files changed, 475 insertions(+), 84 deletions(-) create mode 100644 packages/kernel-test/src/refcount-audit.test.ts diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 2fc1dc1605..44345bfcdd 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -254,17 +254,30 @@ describe('Garbage Collection', () => { }); /** - * Give an importer a chance to notice a dropped object and tell the kernel. + * Give an importer a chance to notice a dropped object and tell the kernel, + * then keep cranking until the resulting GC actions have all been consumed. * * @param vatId - The vat to reap. * @param rootKRef - That vat's root, to poke with cranks afterwards. */ async function reapAndSettle(vatId: VatId, rootKRef: KRef): Promise { kernel.reapVats((id) => id === vatId); - for (let i = 0; i < 3; i++) { + // BOYD has to reach the vat, the vat has to answer, and the kernel has to + // act on the answer — but a round can queue more work, so loop until the + // queue is actually empty rather than guessing at a crank count. + const maxRounds = 10; + for (let round = 0; round < maxRounds; round++) { await kernel.queueMessage(rootKRef, 'noop', []); await waitUntilQuiescent(500); + if ([...kernelStore.getGCActions()].length === 0) { + return; + } } + throw Error( + `GC actions still pending after ${maxRounds} rounds: ${[ + ...kernelStore.getGCActions(), + ].join(', ')}`, + ); } it('survives until both importers let go', async () => { diff --git a/packages/kernel-test/src/refcount-audit.test.ts b/packages/kernel-test/src/refcount-audit.test.ts new file mode 100644 index 0000000000..3ab09fdba6 --- /dev/null +++ b/packages/kernel-test/src/refcount-audit.test.ts @@ -0,0 +1,63 @@ +import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; +import { makeKernelStore } from '@metamask/ocap-kernel'; +import type { KRef, VatId } from '@metamask/ocap-kernel'; +import { expect, describe, it } from 'vitest'; + +import { + getBundleSpec, + makeKernel, + makeMockLogger, + runTestVats, +} from './utils.ts'; + +/** + * The per-crank audit throws from inside the run loop, which nothing restarts. + * Unless that failure is reported to whoever is waiting on the kernel, the only + * symptom is a test that hangs until its timeout, with no mention of reference + * counts anywhere — which would make the audit worthless as a build gate. + */ +describe('reference count audit', () => { + it('reports a violation to kernel callers rather than hanging', async () => { + const kernelDatabase = await makeSQLKernelDatabase({ + dbFilename: ':memory:', + }); + const kernelStore = makeKernelStore(kernelDatabase); + const kernel = await makeKernel(kernelDatabase, true, makeMockLogger()); + await runTestVats(kernel, { + bootstrap: 'exporter', + forceReset: true, + vats: { + exporter: { + bundleSpec: getBundleSpec('exporter-vat'), + parameters: { name: 'Exporter' }, + }, + }, + }); + + const exporterVatId = kernel.getVats()[0]?.id as VatId; + const exporterKRef = kernelStore.getRootObject(exporterVatId) as KRef; + + kernelStore.setObjectRefCount(exporterKRef, { + reachable: 7, + recognizable: 9, + }); + + // The crank carrying this message settles its result before the + // end-of-crank audit runs, so this one may still succeed. + await kernel + .queueMessage(exporterKRef, 'createObject', ['x']) + .catch(() => undefined); + + // What a caller is told directly is that the run loop is gone; the audit + // failure that killed it rides along as the `cause`. That chain is the part + // that has to survive, since "run loop died" on its own names nothing. + const failure = (await kernel + .queueMessage(exporterKRef, 'createObject', ['y']) + .catch((error) => error)) as Error; + + expect(failure.message).toMatch(/Kernel run loop died/u); + expect(String(failure.cause)).toMatch( + /reference count invariant violated/u, + ); + }, 30000); +}); diff --git a/packages/ocap-kernel/src/KernelRouter.test.ts b/packages/ocap-kernel/src/KernelRouter.test.ts index 11aa8922c1..3ef971acf9 100644 --- a/packages/ocap-kernel/src/KernelRouter.test.ts +++ b/packages/ocap-kernel/src/KernelRouter.test.ts @@ -65,6 +65,7 @@ describe('KernelRouter', () => { clearReachableFlag: vi.fn(), deleteCListEntry: vi.fn(), forgetKref: vi.fn(), + orphanKernelObject: vi.fn(), createCrankSavepoint: vi.fn(), } as unknown as KernelStore; @@ -317,6 +318,38 @@ describe('KernelRouter', () => { ]); }); + it('charges the promise, not the object it resolved to', async () => { + const promiseId = 'kp123'; + const resolvedObject = 'ko456'; + ( + kernelStore.getKernelPromise as unknown as MockInstance + ).mockReturnValueOnce({ + state: 'fulfilled', + value: { body: '#"$0"', slots: [resolvedObject] }, + }); + (kernelStore.getOwner as unknown as MockInstance).mockReturnValue('v1'); + + await kernelRouter.deliver({ + type: 'send', + target: promiseId, + message: { + methargs: { body: 'method args', slots: [] }, + result: null, + }, + }); + + // The run queue item was charged against the promise it named, so that + // is what has to be released — not whatever routing resolved it to. + expect(kernelStore.decrementRefCount).toHaveBeenCalledWith( + promiseId, + 'deliver|send|target', + ); + expect(kernelStore.decrementRefCount).not.toHaveBeenCalledWith( + resolvedObject, + 'deliver|send|target', + ); + }); + it('splats message when promise resolves to a non-object', async () => { // Setup a fulfilled promise that doesn't resolve to an object const promiseId = 'kp123'; @@ -580,6 +613,12 @@ describe('KernelRouter', () => { // Verify no notification was delivered to the vat expect(endpointHandle.deliverNotify).not.toHaveBeenCalled(); expect(result).toStrictEqual({ didDelivery: endpointId }); + // Nothing was delivered, but the queued notification is gone either + // way, so its reference has to be released on this path too. + expect(kernelStore.decrementRefCount).toHaveBeenCalledWith( + kpid, + 'deliver|notify', + ); }); it('returns didDelivery when no kpids to retire', async () => { @@ -618,6 +657,10 @@ describe('KernelRouter', () => { // Verify no notification was delivered to the vat expect(endpointHandle.deliverNotify).not.toHaveBeenCalled(); expect(result).toStrictEqual({ didDelivery: endpointId }); + expect(kernelStore.decrementRefCount).toHaveBeenCalledWith( + kpid, + 'deliver|notify', + ); }); it('throws if notification is for an unresolved promise', async () => { @@ -715,6 +758,60 @@ describe('KernelRouter', () => { ]); }, ); + + it('orphans the object when delivering retireExports', async () => { + await kernelRouter.deliver({ + type: 'retireExports', + endpointId: 'v1', + krefs: ['ko1', 'ko2'], + }); + + // The owner has given up the last name for the object, so the kernel's + // record of who owns it must go too or it outlives every reference. + expect( + (kernelStore.orphanKernelObject as unknown as MockInstance).mock + .calls, + ).toStrictEqual([['ko1'], ['ko2']]); + }); + + it('leaves ownership alone when delivering retireImports', async () => { + await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'v1', + krefs: ['ko1'], + }); + + expect(kernelStore.orphanKernelObject).not.toHaveBeenCalled(); + }); + + it('skips the action when the endpoint has vanished', async () => { + getEndpoint.mockImplementationOnce(() => { + throw Error('vat v1 not found'); + }); + + const result = await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'v1', + krefs: ['ko1'], + }); + + expect(result).toStrictEqual({ didDelivery: 'v1' }); + expect(kernelStore.deleteCListEntry).not.toHaveBeenCalled(); + }); + + it('survives a failed delivery', async () => { + ( + endpointHandle.deliverRetireImports as unknown as MockInstance + ).mockRejectedValueOnce(Error('endpoint went away mid-delivery')); + + const result = await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'v1', + krefs: ['ko1'], + }); + + expect(result).toStrictEqual({ didDelivery: 'v1' }); + }); }); describe('bringOutYourDead', () => { diff --git a/packages/ocap-kernel/src/KernelRouter.ts b/packages/ocap-kernel/src/KernelRouter.ts index 6bd080e7c3..9c6266d737 100644 --- a/packages/ocap-kernel/src/KernelRouter.ts +++ b/packages/ocap-kernel/src/KernelRouter.ts @@ -405,10 +405,12 @@ export class KernelRouter { this.#kernelStore.translateCapDataKtoE(endpointId, tPromise.value), ]); } - // TODO(#1006 follow-up): SwingSet also tears down the c-list entry for each - // promise in the batch here, since the endpoint can never refer to a - // settled promise by that eref again. Left alone for now because the - // debug UI discovers exported ocap URLs by scanning these entries. + // TODO: SwingSet also tears down the c-list entry for each promise in the + // batch here, since the endpoint can never refer to a settled promise by + // that eref again. Left alone for now because the debug UI discovers + // exported ocap URLs by scanning these entries. The cost of keeping them is + // that a settled promise reached this way holds a count forever, so it is + // never collected and its resolution slots are never released. const endpoint = this.#getEndpoint(endpointId); return await endpoint.deliverNotify(resolutions); } @@ -424,7 +426,19 @@ export class KernelRouter { this.#logger?.log( `@@@@ deliver ${endpointId} ${type} ${JSON.stringify(krefs)}`, ); - const endpoint = this.#getEndpoint(endpointId); + let endpoint: EndpointHandle; + try { + endpoint = this.#getEndpoint(endpointId); + } catch (error) { + // The endpoint was selected for this action while its c-list still + // existed, but it has since gone away (terminated, and cleaned up in the + // same crank). Nothing left to tell; its c-list goes with it. + this.#logger?.error( + `Skipping ${type} for vanished endpoint ${endpointId}:`, + error, + ); + return { didDelivery: endpointId }; + } const erefs = this.#kernelStore.krefsToErefs(endpointId, krefs); // Telling an endpoint to let go is also the kernel letting go. Otherwise a // dropped export stays flagged reachable, so the same action gets derived @@ -432,12 +446,19 @@ export class KernelRouter { krefs.forEach((kref, index) => { if (type === 'dropExports') { this.#kernelStore.clearReachableFlag(endpointId, kref); - } else { - this.#kernelStore.deleteCListEntry( - endpointId, - kref, - erefs[index] as ERef, - ); + return; + } + // `erefs` is parallel to `krefs`: krefsToErefs throws rather than + // returning a short array, so every index is populated. + this.#kernelStore.deleteCListEntry( + endpointId, + kref, + erefs[index] as ERef, + ); + if (type === 'retireExports') { + // Retiring an export is the owner giving up the last name for the + // object, so the kernel's record of who owns it goes too. + this.#kernelStore.orphanKernelObject(kref); } }); const method = @@ -445,8 +466,17 @@ export class KernelRouter { | 'deliverDropExports' | 'deliverRetireExports' | 'deliverRetireImports'; - const crankResult = await endpoint[method](erefs); - return crankResult; + try { + return await endpoint[method](erefs); + } catch (error) { + // The kernel has already let go above, which is the part that matters for + // accounting. Don't let a failed notification take down the run loop. + this.#logger?.error( + `Delivery of ${type} to ${endpointId} failed:`, + error, + ); + return { didDelivery: endpointId }; + } } /** diff --git a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts index c96b6e26c8..cbc2e57854 100644 --- a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts +++ b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts @@ -84,5 +84,9 @@ export function performExportCleanup( } } kernelStore.forgetKref(endpointId, kref); + // The owner no longer names the object, so nothing can reach it through + // this endpoint again. Drop the owner mapping too, or the kernel's record + // of the object outlives the only c-list entry it was reachable through. + kernelStore.orphanKernelObject(kref); } } diff --git a/packages/ocap-kernel/src/store/index.test.ts b/packages/ocap-kernel/src/store/index.test.ts index 0344da12d4..d2fb46efe3 100644 --- a/packages/ocap-kernel/src/store/index.test.ts +++ b/packages/ocap-kernel/src/store/index.test.ts @@ -150,6 +150,7 @@ describe('kernel store', () => { 'markVatAsTerminated', 'nextReapAction', 'nextTerminatedVatCleanup', + 'orphanKernelObject', 'pinObject', 'provideIncarnationId', 'recomputeRefCounts', diff --git a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts index c7d1cf22d1..6e66baed0a 100644 --- a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts +++ b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts @@ -5,10 +5,9 @@ import type { VatConfig, VatId } from '../../types.ts'; import { makeKernelStore } from '../index.ts'; /** - * Regressions for the asymmetry described in - * https://github.com/MetaMask/ocap-kernel/issues/1006: creating an import - * c-list entry changed no refcount while tearing one down decremented both, - * and `initKernelObject` compensated by minting every object at (1, 1). That + * Regressions for an asymmetry in c-list accounting: creating an import c-list + * entry changed no refcount while tearing one down decremented both, and + * `initKernelObject` compensated by minting every object at (1, 1). That * constant came out right for exactly one importer, which is why nothing * noticed. */ @@ -159,6 +158,69 @@ describe('c-list reference accounting', () => { expect(kernelStore.getImporters(kref)).toStrictEqual([]); }); + describe('an owner that gives up its own export', () => { + it('frees the object once the last importer lets go', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + kernelStore.collectGarbage(); + + // The owner is told to drop, which clears its flag, and it then retires + // the export itself — leaving nothing naming the object from its side. + kernelStore.clearReachableFlag('v1', kref); + kernelStore.forgetKref('v1', kref); + kernelStore.orphanKernelObject(kref); + + kernelStore.forgetKref('v2', kref); + kernelStore.collectGarbage(); + + expect(kernelStore.kernelRefExists(kref)).toBe(false); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('collects an orphan that no importer ever recognized', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + + kernelStore.forgetKref('v1', kref); + kernelStore.orphanKernelObject(kref); + kernelStore.collectGarbage(); + + expect(kernelStore.getOwner(kref)).toBeUndefined(); + expect(kernelStore.kernelRefExists(kref)).toBe(false); + }); + + it('retires stragglers that still recognize an orphaned object', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + + kernelStore.clearReachableFlag('v1', kref); + kernelStore.forgetKref('v1', kref); + kernelStore.orphanKernelObject(kref); + kernelStore.collectGarbage(); + + // v2 can still recognize it, so it has to be told the name is dead + expect([...kernelStore.getGCActions()]).toStrictEqual([ + `v2 retireImport ${kref}`, + ]); + }); + + it('survives an owner mapping left behind without a c-list entry', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + kernelStore.clearReachableFlag('v1', kref); + // Tear the owner's side down but leave the ownership record, the shape + // that used to make the next collection read a key that wasn't there. + kernelStore.forgetKref('v1', kref); + kernelStore.forgetKref('v2', kref); + + expect(() => kernelStore.collectGarbage()).not.toThrow(); + expect(kernelStore.getOwner(kref)).toBeUndefined(); + expect(kernelStore.kernelRefExists(kref)).toBe(false); + }); + }); + describe('cleanupTerminatedVat', () => { it('does nothing for a vat that is not terminated', () => { expect(kernelStore.cleanupTerminatedVat('v1')).toStrictEqual({ diff --git a/packages/ocap-kernel/src/store/methods/clist.ts b/packages/ocap-kernel/src/store/methods/clist.ts index d079100ae4..d7e3a52af0 100644 --- a/packages/ocap-kernel/src/store/methods/clist.ts +++ b/packages/ocap-kernel/src/store/methods/clist.ts @@ -33,7 +33,8 @@ export function getCListMethods(ctx: StoreContext) { * {@link deleteCListEntry}. An import is born recognizing but not reaching: * reachability is `setReachableFlag`'s job, when the reference is handed * over. An export takes no count for an object — the owner is not one of its - * own referrers — and is born flagged. + * own referrers — and is born flagged. For a promise both directions count; + * only objects exempt the owner. * * @param endpointId - The endpoint whose c-list is to be added to. * @param kref - The KRef. @@ -146,10 +147,9 @@ export function getCListMethods(ctx: StoreContext) { * Look up the ERefs that an endpoint's c-list maps a list of KRefs to, * without allocating entries or disturbing reachability. * - * Every kref must already be mapped. Garbage collection is the only caller - * and has already established that each kref has an entry, so a missing one - * means the two disagree — worth hearing about rather than silently dropping - * the notification. + * Every kref must already be mapped: a missing entry means the caller's list + * of krefs and the c-list disagree, which is worth hearing about rather than + * silently dropping the one that got away. * * @param endpointId - The endpoint in question. * @param krefs - The KRefs to look up. diff --git a/packages/ocap-kernel/src/store/methods/gc.ts b/packages/ocap-kernel/src/store/methods/gc.ts index 31b21294c8..2e397b6aeb 100644 --- a/packages/ocap-kernel/src/store/methods/gc.ts +++ b/packages/ocap-kernel/src/store/methods/gc.ts @@ -1,6 +1,7 @@ import { Fail } from '@endo/errors'; import { getBaseMethods } from './base.ts'; +import { getCListMethods } from './clist.ts'; import { getObjectMethods } from './object.ts'; import { getPromiseMethods } from './promise.ts'; import { getReachableMethods } from './reachable.ts'; @@ -33,6 +34,25 @@ export function getGCMethods(ctx: StoreContext) { const { getImporters, isVatTerminated } = getVatMethods(ctx); const { getReachableFlag, getReachableAndVatSlot } = getReachableMethods(ctx); const { clearEmptySubclusters } = getSubclusterMethods(ctx); + const { hasCListEntry } = getCListMethods(ctx); + + /** + * Give up the kernel's record of who owns an object. The object survives only + * as long as something still names it; the collector disposes of it from + * there, retiring any stragglers that still recognize it. + * + * Called when an owner stops naming its own export — it retired or abandoned + * it, or a GC `retireExport` was delivered. Without this the owner mapping + * outlives the c-list entry it was reachable through, which both leaks the + * object record and leaves `collectGarbage` reading a c-list entry that is no + * longer there. + * + * @param kref - The object whose owner mapping is to be dropped. + */ + function orphanKernelObject(kref: KRef): void { + ctx.kv.delete(getOwnerKey(kref)); + ctx.maybeFreeKrefs.add(kref); + } /** * Get the set of GC actions to perform. @@ -158,7 +178,14 @@ export function getGCMethods(ctx: StoreContext) { // might still alive, or might be terminated and in the // process of being deleted. These two clauses are // mutually exclusive. - if (ownerVatID && !terminated) { + if (ownerVatID && !terminated && !hasCListEntry(ownerVatID, kref)) { + // The owner still claims this object but no longer names it, having + // retired or abandoned the export itself. There is nobody to notify, + // and reading its reachable flag would throw, so treat it as + // orphaned and let the clause below dispose of it. + orphanKernelObject(kref); + ownerVatID = undefined; + } else if (ownerVatID && !terminated) { const vatConsidersReachable = getReachableFlag(ownerVatID, kref); if (vatConsidersReachable) { // the reachable count is zero, but the vat doesn't realize it @@ -221,6 +248,7 @@ export function getGCMethods(ctx: StoreContext) { scheduleReap, nextReapAction, retireKernelObjects, + orphanKernelObject, collectGarbage, }; } diff --git a/packages/ocap-kernel/src/store/methods/reachable.test.ts b/packages/ocap-kernel/src/store/methods/reachable.test.ts index 1ec32bdb9f..2464339b68 100644 --- a/packages/ocap-kernel/src/store/methods/reachable.test.ts +++ b/packages/ocap-kernel/src/store/methods/reachable.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import type { KRef } from '../../types.ts'; import { makeKernelStore } from '../index.ts'; describe('GC methods', () => { @@ -37,25 +38,42 @@ describe('GC methods', () => { }); }); - it.each(['setReachableFlag', 'clearReachableFlag'] as const)( - 'is idempotent: %s', - (method) => { - const ko1 = kernelStore.initKernelObject('v1'); - kernelStore.addCListEntry('v1', ko1, 'o-1'); - kernelStore.setReachableFlag('v1', ko1); - - const before = kernelStore.getObjectRefCount(ko1); - kernelStore[method]('v1', ko1); - kernelStore[method]('v1', ko1); - const after = kernelStore.getObjectRefCount(ko1); - - expect(after).toStrictEqual( - method === 'setReachableFlag' - ? before - : { reachable: 0, recognizable: 1 }, - ); - }, - ); + /** + * Give v1 an import entry it reaches, the state both idempotence tests + * start from. + * + * @returns The kref of the reached import. + */ + function givenReachedImport(): KRef { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addCListEntry('v1', ko1, 'o-1'); + kernelStore.setReachableFlag('v1', ko1); + return ko1; + } + + it('setReachableFlag is idempotent', () => { + const ko1 = givenReachedImport(); + + kernelStore.setReachableFlag('v1', ko1); + kernelStore.setReachableFlag('v1', ko1); + + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 1, + recognizable: 1, + }); + }); + + it('clearReachableFlag is idempotent', () => { + const ko1 = givenReachedImport(); + + kernelStore.clearReachableFlag('v1', ko1); + kernelStore.clearReachableFlag('v1', ko1); + + expect(kernelStore.getObjectRefCount(ko1)).toStrictEqual({ + reachable: 0, + recognizable: 1, + }); + }); it('leaves an export entry alone: it carries no reachable count', () => { const ko1 = kernelStore.initKernelObject('v1'); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts index 70af7d4075..bccab4073e 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts @@ -93,6 +93,14 @@ describe('reference count audit', () => { expect(kernelStore.auditRefCounts()).toStrictEqual([]); }); + it('holds for a queued notification', () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.enqueueRun({ type: 'notify', endpointId: 'v2', kpid }); + kernelStore.incrementRefCount(kpid, 'notify'); + + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + it('holds for an unsettled promise with importers', () => { const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); kernelStore.translateRefKtoE('v2', kpid, true); @@ -120,6 +128,7 @@ describe('reference count audit', () => { expect(kernelStore.auditRefCounts()).toStrictEqual([ { + kind: 'mismatch', kref, stored: '0,0', expected: '1,1', @@ -133,7 +142,7 @@ describe('reference count audit', () => { kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); expect(kernelStore.auditRefCounts()).toStrictEqual([ - { kref, stored: '1,1', expected: '0,0', holders: [] }, + { kind: 'mismatch', kref, stored: '1,1', expected: '0,0', holders: [] }, ]); }); @@ -144,8 +153,8 @@ describe('reference count audit', () => { expect(kernelStore.auditRefCounts()).toStrictEqual([ { + kind: 'dangling', kref, - stored: '(deleted)', expected: '1,1', holders: ['v2 c-list import o-1'], }, @@ -204,6 +213,7 @@ describe('reference count audit', () => { expect(corrected).toStrictEqual([ { + kind: 'mismatch', kref, stored: '1,1', expected: '2,2', @@ -230,6 +240,34 @@ describe('reference count audit', () => { expect(unfixable[0]?.kref).toBe(kref); }); + it('rebuilds a promise count', () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.translateRefKtoE('v2', kpid, true); + // A promise has one undifferentiated count, so its repair goes down a + // different path from an object's pair. + kernelStore.incrementRefCount(kpid, 'phantom'); + kernelStore.incrementRefCount(kpid, 'phantom'); + + const { corrected, unfixable } = kernelStore.recomputeRefCounts(); + + expect(corrected).toStrictEqual([ + { + kind: 'mismatch', + kref: kpid, + stored: '5', + expected: '3', + holders: [ + 'unsettled promise', + 'v1 c-list export p+1', + 'v2 c-list import p-1', + ], + }, + ]); + expect(unfixable).toStrictEqual([]); + expect(kernelStore.getRefCount(kpid)).toBe(3); + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + it('queues krefs it zeroes for collection', () => { const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); kernelStore.setObjectRefCount(kref, { reachable: 1, recognizable: 1 }); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index b4178e5c6f..5ccd8d118c 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -13,19 +13,34 @@ import { parseReachableAndVatSlot } from '../utils/reachable.ts'; * A kref whose stored reference counts disagree with the counts implied by the * references the kernel can actually be seen to hold. */ -export type RefCountViolation = { - kref: KRef; - /** - * The counts as stored, in the store's own encoding: `"reachable,recognizable"` - * for objects, a single number for promises, or `"(deleted)"` if the kref has - * no refcount entry at all. - */ - stored: string; - /** The counts implied by `holders`, in the same encoding as `stored`. */ - expected: string; - /** One entry per reference found, so a mismatch can be traced to its source. */ - holders: string[]; -}; +export type RefCountViolation = + | { + /** The kref is still counted, just by the wrong amount. */ + kind: 'mismatch'; + kref: KRef; + /** + * The counts as stored, in the store's own encoding: + * `"reachable,recognizable"` for objects, a single number for promises. + */ + stored: string; + /** The counts implied by `holders`, in the same encoding as `stored`. */ + expected: string; + /** One entry per reference found, so a mismatch can be traced to its source. */ + holders: string[]; + } + | { + /** + * The kref has no refcount entry, so each entry in `holders` names + * something the kernel has already deleted. Rewriting a count cannot + * repair this. + */ + kind: 'dangling'; + kref: KRef; + /** The counts `holders` imply, which there is nothing left to credit. */ + expected: string; + /** One entry per dangling reference found. */ + holders: string[]; + }; /** * The running total of references found for one kref. For a promise, which has @@ -267,8 +282,8 @@ export function getRefCountAuditMethods(ctx: StoreContext) { // pointing at it is a dangling reference. if (tally.holders.length > 0) { violations.push({ + kind: 'dangling', kref, - stored: '(deleted)', expected: expectedText, holders: tally.holders, }); @@ -280,6 +295,7 @@ export function getRefCountAuditMethods(ctx: StoreContext) { : renderCounts(kref, getObjectRefCount(kref)); if (storedText !== expectedText) { violations.push({ + kind: 'mismatch', kref, stored: storedText, expected: expectedText, @@ -313,7 +329,7 @@ export function getRefCountAuditMethods(ctx: StoreContext) { const corrected: RefCountViolation[] = []; const unfixable: RefCountViolation[] = []; for (const violation of auditRefCounts()) { - if (violation.stored === '(deleted)') { + if (violation.kind === 'dangling') { unfixable.push(violation); continue; } @@ -330,11 +346,14 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * Render violations as a human-readable report. * * @param violations - The violations to describe. - * @returns A multi-line description, one paragraph per violation. + * @returns A newline-separated report, one line per violation. */ function formatRefCountViolations(violations: RefCountViolation[]): string { return violations - .map(({ kref, stored, expected, holders }) => { + .map((violation) => { + const { kref, expected, holders } = violation; + const stored = + violation.kind === 'dangling' ? '(deleted)' : violation.stored; const held = holders.length > 0 ? holders.join(', ') : 'nothing'; return `${kref}: stored ${stored}, expected ${expected} (held by: ${held})`; }) @@ -346,14 +365,16 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * via the `auditRefCounts` option, and run at the end of every crank. */ function assertRefCountsIfAuditing(): void { - if (!ctx.auditRefCounts) { + if (!ctx.refCountAuditingEnabled) { return; } const violations = auditRefCounts(); if (violations.length > 0) { - throw Error( - `reference count invariant violated:\n${formatRefCountViolations(violations)}`, - ); + const report = formatRefCountViolations(violations); + // Logged as well as thrown: this fires from inside a crank, and whoever + // catches that has no way to render the report itself. + ctx.logger?.error(`reference count invariant violated:\n${report}`); + throw Error(`reference count invariant violated:\n${report}`); } } @@ -363,7 +384,7 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * @param enabled - Whether to audit after every crank. */ function setRefCountAuditing(enabled: boolean): void { - ctx.auditRefCounts = enabled; + ctx.refCountAuditingEnabled = enabled; } return { diff --git a/packages/ocap-kernel/src/store/methods/vat.ts b/packages/ocap-kernel/src/store/methods/vat.ts index 29e2a85dff..c1875d8581 100644 --- a/packages/ocap-kernel/src/store/methods/vat.ts +++ b/packages/ocap-kernel/src/store/methods/vat.ts @@ -271,9 +271,10 @@ export function getVatMethods(ctx: StoreContext) { work.imports += 1; } - // The caller rejected the orphan promises via getPromisesByDecider() before - // calling us, which is what released each promise's unsettled reference, - // but their kpids are still in the dead vat's c-list. Clean those up now. + // The caller looked the orphan promises up with getPromisesByDecider() and + // rejected them before calling us; that rejection is what released each + // promise's unsettled reference. Their kpids are still in the dead vat's + // c-list, so clean those up now. for (const key of getPrefixedKeys(promisePrefix)) { const krefStr = ctx.kv.get(key) ?? Fail`getNextKey ensures get`; assert(key.startsWith(clistPrefix), key); diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index b9886c174f..909c070d8f 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -28,7 +28,7 @@ export type StoreContext = { subclusters: StoredValue; // Holds Subcluster[] nextSubclusterId: StoredValue; // Holds string vatToSubclusterMap: StoredValue; // Holds Record - auditRefCounts: boolean; // If set, verify refcounts against ground truth every crank + refCountAuditingEnabled: boolean; // If true, verify refcounts against ground truth every crank logger?: Logger | undefined; }; diff --git a/packages/ocap-kernel/src/vats/VatManager.ts b/packages/ocap-kernel/src/vats/VatManager.ts index 5da2bebe2d..47a0a9f771 100644 --- a/packages/ocap-kernel/src/vats/VatManager.ts +++ b/packages/ocap-kernel/src/vats/VatManager.ts @@ -123,18 +123,32 @@ export class VatManager { cause: error, }); } - this.#kernelStore.initEndpoint(vatId); - const rootRef = this.#kernelStore.exportFromEndpoint( - vatId, - ROOT_OBJECT_VREF, - ); - // A root is addressable for as long as its vat lives, whether or not - // anyone currently imports it: the kernel's own API hands out root krefs - // and `getRootObject` resolves them through this c-list entry. Without a - // pin, GC would retire the entry the moment the last importer let go. - this.#kernelStore.pinObject(rootRef); - this.#kernelStore.setVatConfig(vatId, vatConfig); - return rootRef; + try { + this.#kernelStore.initEndpoint(vatId); + const rootRef = this.#kernelStore.exportFromEndpoint( + vatId, + ROOT_OBJECT_VREF, + ); + // A root is addressable for as long as its vat lives, whether or not + // anyone currently imports it: the kernel's own API hands out root krefs + // and `getRootObject` resolves them through this c-list entry. Without a + // pin, GC would retire the entry the moment the last importer let go. + this.#kernelStore.pinObject(rootRef); + this.#kernelStore.setVatConfig(vatId, vatConfig); + return rootRef; + } catch (error) { + // The worker is already running, so leaving it would strand a vat the + // kernel has no record of. Tear it down before reporting the failure. + await this.stopVat(vatId, true).catch((stopError) => { + this.#logger.error( + `Failed to stop vat ${vatId} after incomplete launch:`, + stopError, + ); + }); + throw new Error(`Failed to launch vat ${vatId} (${vatName})`, { + cause: error, + }); + } } /** diff --git a/packages/ocap-kernel/src/vats/VatSyscall.test.ts b/packages/ocap-kernel/src/vats/VatSyscall.test.ts index 04a0e8bf42..6246e3bf75 100644 --- a/packages/ocap-kernel/src/vats/VatSyscall.test.ts +++ b/packages/ocap-kernel/src/vats/VatSyscall.test.ts @@ -30,6 +30,7 @@ describe('VatSyscall', () => { clearReachableFlag: vi.fn(), getReachableFlag: vi.fn(), forgetKref: vi.fn(), + orphanKernelObject: vi.fn(), getVatConfig: vi.fn(() => ({})), isVatActive: vi.fn(() => true), isInCrank: vi.fn(() => true), From f6791a50b075a0e01a2882ccd2d05bdc8cb6e510 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Aug 2026 19:00:16 +0200 Subject: [PATCH 29/34] fix(ocap-kernel): guard disowning, and stop hiding GC delivery failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the previous commit found that four of the five error handlers it added turned a crash into a state the kernel can no longer detect. Corrects that, and closes a hole the orphaning opened. `orphanKernelObject` took an object's owner mapping on trust. Nothing upstream of `performExportCleanup` checks that the vref it was handed is even an export — `translateSyscallVtoK` maps both directions alike — so a vat could pass an import to `abandonExports`, which needs no precondition at all, and erase a different live vat's claim to an object it was still exporting. Sends to that object then went splat with OBJECT_DELETED, terminating the victim tripped `cleanupTerminatedVat`'s ownership assertion and took the run loop with it, and the audit could not see any of it, because an export entry carries no count. Disowning is now the owner's own doing: the expected owner is a required argument and must match, and the syscall path rejects a mismatch outright. The vanished-endpoint catch returned before the teardown, but `processGCActionSet` had already consumed the action, so neither the kernel nor the durable set remembered the object — a permanent leak, also invisible to the audit. The kernel's side is now released whether or not anyone is left to tell, and krefs whose entries a cleanup already removed are skipped rather than assumed present. The delivery-failure catch committed the teardown after the endpoint had failed to hear about it, so the endpoint would go on to mint a fresh kref for an object the kernel believed it had let go of — the same object with two identities. It now aborts, which restores both the entries and the action, and terminates the vat that could not accept the delivery. `launchVat`'s cleanup path stopped the worker without marking the vat terminated, so nothing ever reclaimed the records a partial launch had written. The audit counted an importer's c-list entry as a holder during the window between `retireKernelObjects` deleting an object and delivering the matching `retireImport`, so the collector's own output failed the end-of-crank check. The missing assertion in the test covering that sequence is now present. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelRouter.test.ts | 69 ++++++++++++++++-- packages/ocap-kernel/src/KernelRouter.ts | 71 ++++++++++++++----- .../src/garbage-collection/gc-handlers.ts | 12 +++- .../store/methods/clist-accounting.test.ts | 20 +++++- packages/ocap-kernel/src/store/methods/gc.ts | 24 +++++-- .../src/store/methods/refcount-audit.ts | 4 +- packages/ocap-kernel/src/vats/VatManager.ts | 24 +++++-- .../ocap-kernel/src/vats/VatSyscall.test.ts | 2 + 8 files changed, 185 insertions(+), 41 deletions(-) diff --git a/packages/ocap-kernel/src/KernelRouter.test.ts b/packages/ocap-kernel/src/KernelRouter.test.ts index 3ef971acf9..650933297f 100644 --- a/packages/ocap-kernel/src/KernelRouter.test.ts +++ b/packages/ocap-kernel/src/KernelRouter.test.ts @@ -66,6 +66,7 @@ describe('KernelRouter', () => { deleteCListEntry: vi.fn(), forgetKref: vi.fn(), orphanKernelObject: vi.fn(), + hasCListEntry: vi.fn().mockReturnValue(true), createCrankSavepoint: vi.fn(), } as unknown as KernelStore; @@ -771,7 +772,10 @@ describe('KernelRouter', () => { expect( (kernelStore.orphanKernelObject as unknown as MockInstance).mock .calls, - ).toStrictEqual([['ko1'], ['ko2']]); + ).toStrictEqual([ + ['ko1', 'v1'], + ['ko2', 'v1'], + ]); }); it('leaves ownership alone when delivering retireImports', async () => { @@ -784,7 +788,7 @@ describe('KernelRouter', () => { expect(kernelStore.orphanKernelObject).not.toHaveBeenCalled(); }); - it('skips the action when the endpoint has vanished', async () => { + it('still releases the kernel side when the endpoint has vanished', async () => { getEndpoint.mockImplementationOnce(() => { throw Error('vat v1 not found'); }); @@ -795,11 +799,51 @@ describe('KernelRouter', () => { krefs: ['ko1'], }); + expect(result).toStrictEqual({ didDelivery: 'v1' }); + // The action has already been consumed, so skipping the teardown would + // lose it and leave the entry behind for good + expect(kernelStore.deleteCListEntry).toHaveBeenCalledWith( + 'v1', + 'ko1', + 'translated-ko1', + ); + }); + + it('skips krefs already cleaned up before delivery', async () => { + ( + kernelStore.hasCListEntry as unknown as MockInstance + ).mockImplementation( + (_endpointId: string, kref: string) => kref === 'ko1', + ); + + await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'v1', + krefs: ['ko1', 'ko2'], + }); + + expect( + (kernelStore.deleteCListEntry as unknown as MockInstance).mock.calls, + ).toStrictEqual([['v1', 'ko1', 'translated-ko1']]); + }); + + it('does nothing when every kref is already gone', async () => { + (kernelStore.hasCListEntry as unknown as MockInstance).mockReturnValue( + false, + ); + + const result = await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'v1', + krefs: ['ko1'], + }); + expect(result).toStrictEqual({ didDelivery: 'v1' }); expect(kernelStore.deleteCListEntry).not.toHaveBeenCalled(); + expect(endpointHandle.deliverRetireImports).not.toHaveBeenCalled(); }); - it('survives a failed delivery', async () => { + it('rolls back and terminates the vat when delivery fails', async () => { ( endpointHandle.deliverRetireImports as unknown as MockInstance ).mockRejectedValueOnce(Error('endpoint went away mid-delivery')); @@ -810,7 +854,24 @@ describe('KernelRouter', () => { krefs: ['ko1'], }); - expect(result).toStrictEqual({ didDelivery: 'v1' }); + // Committing the release while v1 still holds the eref would leave the + // two disagreeing, and v1 would mint a fresh kref for the same object + expect(result?.abort).toBe(true); + expect(result?.terminate?.vatId).toBe('v1'); + }); + + it('rolls back without terminating when a remote fails', async () => { + ( + endpointHandle.deliverRetireImports as unknown as MockInstance + ).mockRejectedValueOnce(Error('remote queue full')); + + const result = await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'r1', + krefs: ['ko1'], + }); + + expect(result).toStrictEqual({ abort: true }); }); }); diff --git a/packages/ocap-kernel/src/KernelRouter.ts b/packages/ocap-kernel/src/KernelRouter.ts index 9c6266d737..0deb9911da 100644 --- a/packages/ocap-kernel/src/KernelRouter.ts +++ b/packages/ocap-kernel/src/KernelRouter.ts @@ -3,7 +3,10 @@ import type { CapData } from '@endo/marshal'; import { Logger } from '@metamask/logger'; import { KernelQueue } from './KernelQueue.ts'; -import { makeKernelError } from './liveslots/kernel-marshal.ts'; +import { + makeFatalKernelError, + makeKernelError, +} from './liveslots/kernel-marshal.ts'; import type { KernelStore } from './store/index.ts'; import { extractSingleRef } from './store/utils/extract-ref.ts'; import { parseRef } from './store/utils/parse-ref.ts'; @@ -21,6 +24,7 @@ import type { RunQueueItemGCAction, CrankResult, } from './types.ts'; +import { isVatId } from './types.ts'; import { assert, Fail } from './utils/assert.ts'; type MessageRoute = { @@ -426,29 +430,33 @@ export class KernelRouter { this.#logger?.log( `@@@@ deliver ${endpointId} ${type} ${JSON.stringify(krefs)}`, ); - let endpoint: EndpointHandle; - try { - endpoint = this.#getEndpoint(endpointId); - } catch (error) { - // The endpoint was selected for this action while its c-list still - // existed, but it has since gone away (terminated, and cleaned up in the - // same crank). Nothing left to tell; its c-list goes with it. + // This action was selected while the endpoint's c-list held every one of + // these krefs, but `nextTerminatedVatCleanup` runs between selection and + // here and can take the entries — and the endpoint — with it. Whatever + // survives still has to be released on the kernel's side: the action has + // already been consumed from the durable set, so skipping the teardown + // would lose it and leave the entry behind for good. + const live = krefs.filter((kref) => + this.#kernelStore.hasCListEntry(endpointId, kref), + ); + if (live.length < krefs.length) { this.#logger?.error( - `Skipping ${type} for vanished endpoint ${endpointId}:`, - error, + `${type} for ${endpointId}: ${krefs.length - live.length} of ${krefs.length} kref(s) were cleaned up before delivery`, ); + } + if (live.length === 0) { return { didDelivery: endpointId }; } - const erefs = this.#kernelStore.krefsToErefs(endpointId, krefs); + const erefs = this.#kernelStore.krefsToErefs(endpointId, live); // Telling an endpoint to let go is also the kernel letting go. Otherwise a // dropped export stays flagged reachable, so the same action gets derived // again, and retired entries outlive the objects they name. - krefs.forEach((kref, index) => { + live.forEach((kref, index) => { if (type === 'dropExports') { this.#kernelStore.clearReachableFlag(endpointId, kref); return; } - // `erefs` is parallel to `krefs`: krefsToErefs throws rather than + // `erefs` is parallel to `live`: krefsToErefs throws rather than // returning a short array, so every index is populated. this.#kernelStore.deleteCListEntry( endpointId, @@ -458,9 +466,19 @@ export class KernelRouter { if (type === 'retireExports') { // Retiring an export is the owner giving up the last name for the // object, so the kernel's record of who owns it goes too. - this.#kernelStore.orphanKernelObject(kref); + this.#kernelStore.orphanKernelObject(kref, endpointId); } }); + let endpoint: EndpointHandle; + try { + endpoint = this.#getEndpoint(endpointId); + } catch (error) { + this.#logger?.error( + `Endpoint ${endpointId} vanished before ${type} of ${JSON.stringify(live)}; released the kernel's side anyway:`, + error, + ); + return { didDelivery: endpointId }; + } const method = `deliver${(type[0] as string).toUpperCase()}${type.slice(1)}` as | 'deliverDropExports' @@ -469,13 +487,30 @@ export class KernelRouter { try { return await endpoint[method](erefs); } catch (error) { - // The kernel has already let go above, which is the part that matters for - // accounting. Don't let a failed notification take down the run loop. + // The teardown above has to be undone with it: committing it while the + // endpoint still holds the erefs would leave the two disagreeing, and the + // endpoint would go on to mint fresh krefs for objects the kernel thinks + // it has let go of. Aborting restores both the entries and the action. this.#logger?.error( - `Delivery of ${type} to ${endpointId} failed:`, + `Delivery of ${type} to ${endpointId} failed; rolling back the kernel's release of ${JSON.stringify(live)}:`, error, ); - return { didDelivery: endpointId }; + if (!isVatId(endpointId)) { + // A remote gets reconciled by the incarnation-change path when it comes + // back; there is no worker to terminate. + return { abort: true }; + } + return { + abort: true, + terminate: { + vatId: endpointId, + reject: true, + info: makeFatalKernelError( + 'INTERNAL_ERROR', + `failed to accept ${type}: ${error instanceof Error ? error.message : String(error)}`, + ), + }, + }; } } diff --git a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts index cbc2e57854..79ea340d86 100644 --- a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts +++ b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts @@ -78,6 +78,16 @@ export function performExportCleanup( `endpoint ${endpointId} issued invalid ${action}Exports for ${kref}`, ); } + // Only an owner may give up an object. Nothing upstream of here checks that + // the vref is even an export — `translateSyscallVtoK` maps import and + // export directions alike — so without this a vat could disown an object + // belonging to a different, live vat. + const owner = kernelStore.getOwner(kref); + if (owner !== endpointId) { + throw Error( + `endpoint ${endpointId} issued ${action}Exports for ${kref}, which is owned by ${owner ?? 'nobody'}`, + ); + } if (checkReachable) { if (kernelStore.getReachableFlag(endpointId, kref)) { throw Error(`${action}Exports but ${kref} is still reachable`); @@ -87,6 +97,6 @@ export function performExportCleanup( // The owner no longer names the object, so nothing can reach it through // this endpoint again. Drop the owner mapping too, or the kernel's record // of the object outlives the only c-list entry it was reachable through. - kernelStore.orphanKernelObject(kref); + kernelStore.orphanKernelObject(kref, endpointId); } } diff --git a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts index 6e66baed0a..5ce0dc12b1 100644 --- a/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts +++ b/packages/ocap-kernel/src/store/methods/clist-accounting.test.ts @@ -169,7 +169,7 @@ describe('c-list reference accounting', () => { // the export itself — leaving nothing naming the object from its side. kernelStore.clearReachableFlag('v1', kref); kernelStore.forgetKref('v1', kref); - kernelStore.orphanKernelObject(kref); + kernelStore.orphanKernelObject(kref, 'v1'); kernelStore.forgetKref('v2', kref); kernelStore.collectGarbage(); @@ -182,7 +182,7 @@ describe('c-list reference accounting', () => { const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); kernelStore.forgetKref('v1', kref); - kernelStore.orphanKernelObject(kref); + kernelStore.orphanKernelObject(kref, 'v1'); kernelStore.collectGarbage(); expect(kernelStore.getOwner(kref)).toBeUndefined(); @@ -196,13 +196,27 @@ describe('c-list reference accounting', () => { kernelStore.clearReachableFlag('v1', kref); kernelStore.forgetKref('v1', kref); - kernelStore.orphanKernelObject(kref); + kernelStore.orphanKernelObject(kref, 'v1'); kernelStore.collectGarbage(); // v2 can still recognize it, so it has to be told the name is dead expect([...kernelStore.getGCActions()]).toStrictEqual([ `v2 retireImport ${kref}`, ]); + // v2's entry outlives the object it names until that action is delivered. + // The audit has to tolerate that window, or the end-of-crank check throws + // on a state the collector itself just created. + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + }); + + it('rejects an endpoint disowning an object it does not own', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + + expect(() => kernelStore.orphanKernelObject(kref, 'v2')).toThrow( + 'owned by "v1"', + ); + expect(kernelStore.getOwner(kref)).toBe('v1'); }); it('survives an owner mapping left behind without a c-list entry', () => { diff --git a/packages/ocap-kernel/src/store/methods/gc.ts b/packages/ocap-kernel/src/store/methods/gc.ts index 2e397b6aeb..f4a4d3d0d0 100644 --- a/packages/ocap-kernel/src/store/methods/gc.ts +++ b/packages/ocap-kernel/src/store/methods/gc.ts @@ -47,9 +47,17 @@ export function getGCMethods(ctx: StoreContext) { * object record and leaves `collectGarbage` reading a c-list entry that is no * longer there. * + * `expectedOwner` is required, and must match: disowning an object is only + * ever the owner's own doing. Taking it on trust would let one endpoint erase + * another's claim to an object it is still exporting. + * * @param kref - The object whose owner mapping is to be dropped. + * @param expectedOwner - The endpoint the caller believes owns `kref`. */ - function orphanKernelObject(kref: KRef): void { + function orphanKernelObject(kref: KRef, expectedOwner: EndpointId): void { + const owner = getOwner(kref); + owner === expectedOwner || + Fail`cannot orphan ${kref} for ${expectedOwner}: owned by ${owner ?? 'nobody'}`; ctx.kv.delete(getOwnerKey(kref)); ctx.maybeFreeKrefs.add(kref); } @@ -179,11 +187,15 @@ export function getGCMethods(ctx: StoreContext) { // process of being deleted. These two clauses are // mutually exclusive. if (ownerVatID && !terminated && !hasCListEntry(ownerVatID, kref)) { - // The owner still claims this object but no longer names it, having - // retired or abandoned the export itself. There is nobody to notify, - // and reading its reachable flag would throw, so treat it as - // orphaned and let the clause below dispose of it. - orphanKernelObject(kref); + // Should be unreachable: every path that tears down an owner's + // export entry orphans the object with it. Repair it so the + // collector can keep going, but say so — absorbing this in silence + // would hide whatever upstream broke the pairing. + ctx.logger?.error( + `${kref} is owned by live endpoint ${ownerVatID} which has no ` + + `c-list entry for it; treating it as orphaned`, + ); + orphanKernelObject(kref, ownerVatID); ownerVatID = undefined; } else if (ownerVatID && !terminated) { const vatConsidersReachable = getReachableFlag(ownerVatID, kref); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index 5ccd8d118c..14a084724a 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -371,8 +371,8 @@ export function getRefCountAuditMethods(ctx: StoreContext) { const violations = auditRefCounts(); if (violations.length > 0) { const report = formatRefCountViolations(violations); - // Logged as well as thrown: this fires from inside a crank, and whoever - // catches that has no way to render the report itself. + // Logged as well as thrown: if this is the last crank before the kernel + // goes idle, nobody sends another message and the log is the only record. ctx.logger?.error(`reference count invariant violated:\n${report}`); throw Error(`reference count invariant violated:\n${report}`); } diff --git a/packages/ocap-kernel/src/vats/VatManager.ts b/packages/ocap-kernel/src/vats/VatManager.ts index 47a0a9f771..0870a9dacf 100644 --- a/packages/ocap-kernel/src/vats/VatManager.ts +++ b/packages/ocap-kernel/src/vats/VatManager.ts @@ -139,15 +139,25 @@ export class VatManager { } catch (error) { // The worker is already running, so leaving it would strand a vat the // kernel has no record of. Tear it down before reporting the failure. - await this.stopVat(vatId, true).catch((stopError) => { + let stopFailure: unknown; + try { + await this.stopVat(vatId, true); + } catch (caught) { + stopFailure = caught; this.#logger.error( - `Failed to stop vat ${vatId} after incomplete launch:`, - stopError, + `Failed to stop vat ${vatId} after incomplete launch; its worker may still be running:`, + caught, ); - }); - throw new Error(`Failed to launch vat ${vatId} (${vatName})`, { - cause: error, - }); + } + // `stopVat` only tears down the worker. Whatever store records the + // partial launch did write — the endpoint counters, the root's c-list + // pair, its owner entry — are reclaimed by the terminated-vat cleanup, + // which never runs unless the vat is marked. + this.#kernelStore.markVatAsTerminated(vatId); + throw new Error( + `Failed to launch vat ${vatId} (${vatName})${stopFailure ? ' (cleanup also failed)' : ''}`, + { cause: error }, + ); } } diff --git a/packages/ocap-kernel/src/vats/VatSyscall.test.ts b/packages/ocap-kernel/src/vats/VatSyscall.test.ts index 6246e3bf75..bf425bc23b 100644 --- a/packages/ocap-kernel/src/vats/VatSyscall.test.ts +++ b/packages/ocap-kernel/src/vats/VatSyscall.test.ts @@ -30,6 +30,8 @@ describe('VatSyscall', () => { clearReachableFlag: vi.fn(), getReachableFlag: vi.fn(), forgetKref: vi.fn(), + // Only an owner may disown an object, so the cleanup syscalls check first + getOwner: vi.fn().mockReturnValue('v1'), orphanKernelObject: vi.fn(), getVatConfig: vi.fn(() => ({})), isVatActive: vi.fn(() => true), From 5c3a850d79036375848bb300714257be125bfb0d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 5 Aug 2026 19:21:43 +0200 Subject: [PATCH 30/34] fix(ocap-kernel): don't starve the run loop retrying a remote GC delivery Aborting a failed GC delivery restores the action to the durable set, and `processGCActionSet` is consulted ahead of all other run-queue work. For a vat that is fine, because terminating it is what stops the restored action from coming back. A remote cannot be terminated, so the same item would be selected every crank and nothing else would ever run. A remote is a separate kernel across a link that can drop messages anyway, and it reconciles on the next incarnation change, so its failures no longer abort. Also stop `orphanKernelObject` throwing on an object that is already orphaned. Disowning something nobody owns is a no-op, not an error: only a mismatch with a different, live owner is, which is the case the check exists for. Same for the syscall path. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelRouter.test.ts | 7 +++-- packages/ocap-kernel/src/KernelRouter.ts | 29 ++++++++++++------- .../src/garbage-collection/gc-handlers.ts | 7 +++-- packages/ocap-kernel/src/store/methods/gc.ts | 12 +++++--- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/packages/ocap-kernel/src/KernelRouter.test.ts b/packages/ocap-kernel/src/KernelRouter.test.ts index 650933297f..eed87fe4d7 100644 --- a/packages/ocap-kernel/src/KernelRouter.test.ts +++ b/packages/ocap-kernel/src/KernelRouter.test.ts @@ -860,7 +860,7 @@ describe('KernelRouter', () => { expect(result?.terminate?.vatId).toBe('v1'); }); - it('rolls back without terminating when a remote fails', async () => { + it('does not retry a remote that refuses the delivery', async () => { ( endpointHandle.deliverRetireImports as unknown as MockInstance ).mockRejectedValueOnce(Error('remote queue full')); @@ -871,7 +871,10 @@ describe('KernelRouter', () => { krefs: ['ko1'], }); - expect(result).toStrictEqual({ abort: true }); + // Aborting would restore the action, and GC actions are selected ahead + // of all other work, so a remote that keeps refusing would be handed + // this same item every crank and nothing else would ever run + expect(result).toStrictEqual({ didDelivery: 'r1' }); }); }); diff --git a/packages/ocap-kernel/src/KernelRouter.ts b/packages/ocap-kernel/src/KernelRouter.ts index 0deb9911da..cbd36b36a5 100644 --- a/packages/ocap-kernel/src/KernelRouter.ts +++ b/packages/ocap-kernel/src/KernelRouter.ts @@ -487,19 +487,28 @@ export class KernelRouter { try { return await endpoint[method](erefs); } catch (error) { - // The teardown above has to be undone with it: committing it while the - // endpoint still holds the erefs would leave the two disagreeing, and the - // endpoint would go on to mint fresh krefs for objects the kernel thinks - // it has let go of. Aborting restores both the entries and the action. + if (!isVatId(endpointId)) { + // A remote is a separate kernel across a link that can drop messages, + // so its protocol already has to tolerate one going missing — it + // reconciles on the next incarnation change. Retrying instead would + // starve the kernel: GC actions are selected ahead of all other work, + // so a remote that keeps refusing (a full send queue, say) would be + // handed the same item every crank and nothing else would ever run. + this.#logger?.error( + `Delivery of ${type} to remote ${endpointId} failed; the kernel has released ${JSON.stringify(live)} regardless:`, + error, + ); + return { didDelivery: endpointId }; + } + // A vat is local and reliable, so a refusal means it is broken. Undo the + // teardown rather than commit it: leaving the two disagreeing would have + // the vat mint fresh krefs for objects the kernel thinks it let go of. + // Aborting restores the entries and the action; terminating the vat is + // what stops that restored action from being retried forever. this.#logger?.error( - `Delivery of ${type} to ${endpointId} failed; rolling back the kernel's release of ${JSON.stringify(live)}:`, + `Delivery of ${type} to ${endpointId} failed; rolling back the kernel's release of ${JSON.stringify(live)} and terminating it:`, error, ); - if (!isVatId(endpointId)) { - // A remote gets reconciled by the incarnation-change path when it comes - // back; there is no worker to terminate. - return { abort: true }; - } return { abort: true, terminate: { diff --git a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts index 79ea340d86..15e254eb3b 100644 --- a/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts +++ b/packages/ocap-kernel/src/garbage-collection/gc-handlers.ts @@ -81,11 +81,12 @@ export function performExportCleanup( // Only an owner may give up an object. Nothing upstream of here checks that // the vref is even an export — `translateSyscallVtoK` maps import and // export directions alike — so without this a vat could disown an object - // belonging to a different, live vat. + // belonging to a different, live vat. An already-orphaned object is fine: + // there is no claim left to erase. const owner = kernelStore.getOwner(kref); - if (owner !== endpointId) { + if (owner !== undefined && owner !== endpointId) { throw Error( - `endpoint ${endpointId} issued ${action}Exports for ${kref}, which is owned by ${owner ?? 'nobody'}`, + `endpoint ${endpointId} issued ${action}Exports for ${kref}, which is owned by ${owner}`, ); } if (checkReachable) { diff --git a/packages/ocap-kernel/src/store/methods/gc.ts b/packages/ocap-kernel/src/store/methods/gc.ts index f4a4d3d0d0..92a093c968 100644 --- a/packages/ocap-kernel/src/store/methods/gc.ts +++ b/packages/ocap-kernel/src/store/methods/gc.ts @@ -47,17 +47,21 @@ export function getGCMethods(ctx: StoreContext) { * object record and leaves `collectGarbage` reading a c-list entry that is no * longer there. * - * `expectedOwner` is required, and must match: disowning an object is only - * ever the owner's own doing. Taking it on trust would let one endpoint erase - * another's claim to an object it is still exporting. + * Disowning an object is only ever the owner's own doing, so `expectedOwner` + * is required: taking it on trust would let one endpoint erase another's claim + * to an object it is still exporting. An object that is already orphaned is + * left alone — the caller and the kernel agree it has no owner. * * @param kref - The object whose owner mapping is to be dropped. * @param expectedOwner - The endpoint the caller believes owns `kref`. */ function orphanKernelObject(kref: KRef, expectedOwner: EndpointId): void { const owner = getOwner(kref); + if (owner === undefined) { + return; + } owner === expectedOwner || - Fail`cannot orphan ${kref} for ${expectedOwner}: owned by ${owner ?? 'nobody'}`; + Fail`cannot orphan ${kref} for ${expectedOwner}: owned by ${owner}`; ctx.kv.delete(getOwnerKey(kref)); ctx.maybeFreeKrefs.add(kref); } From 2cc1d85c088e713360ec35bddcaa2ef02e34cd3f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 10 Aug 2026 12:52:18 +0200 Subject: [PATCH 31/34] test(ocap-kernel): pin each refcount audit credit source to a literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean-audit cases prove each rule agrees with whatever the store did, which stays true if a rule and the code it mirrors are wrong by the same constant. Six of eight rules could have drifted and the suite would have stayed green. Each of the ten credit sources now pins its count and holder labels to literals and asserts drift in both directions: too low collects a live capability, too high leaks it. That closes the two coverage gaps as a side effect — a run-queue send's result promise, and a message parked on an unresolved promise, neither of which any test reached. Also states what the audit can and cannot find, which matters because its ground truth *is* the holder set: a count that disagrees with its holders is caught either way, but a holder that should have been torn down and wasn't justifies its own count at any value, so a leaked reference is invisible to it by construction. That is exactly the case the retained settled-promise c-list entry leaves behind, so the CHANGELOG no longer claims the audit would catch it. The `auditRefCounts` JSDoc no longer scopes the option as "intended for tests and debugging": it stands in for the invariant `collectGarbage` cannot assert, and is off by default only because it walks the whole store. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/Kernel.ts | 8 +- .../src/store/methods/refcount-audit.test.ts | 191 +++++++++++++++++- .../src/store/methods/refcount-audit.ts | 9 + 3 files changed, 205 insertions(+), 3 deletions(-) diff --git a/packages/ocap-kernel/src/Kernel.ts b/packages/ocap-kernel/src/Kernel.ts index c041d060be..f615ea6cbf 100644 --- a/packages/ocap-kernel/src/Kernel.ts +++ b/packages/ocap-kernel/src/Kernel.ts @@ -111,8 +111,12 @@ export class Kernel { * @param options.onRunLoopFailure - Optional handler called if the run loop dies. * @param options.auditRefCounts - If true, verify every kref's reference * counts against the references the kernel actually holds at the end of each - * crank, and throw on any mismatch. Intended for tests and debugging; the - * audit walks the whole store. + * crank, and throw on any mismatch. This is the check standing in for the + * accounting invariant `collectGarbage` still cannot assert (see the comment + * on its `retireExport` branch), so it is not optional + * instrumentation: it is off by default only because it walks the whole store + * every crank. Any kernel whose accounting is under test wants it on, and + * every kernel `kernel-test` builds enables it. */ // eslint-disable-next-line no-restricted-syntax private constructor( diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts index bccab4073e..868034e55c 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.test.ts @@ -1,3 +1,4 @@ +import type { KernelDatabase } from '@metamask/kernel-store'; import { describe, it, expect, beforeEach } from 'vitest'; import { makeMapKernelDatabase } from '../../../test/storage.ts'; @@ -6,6 +7,7 @@ import { makeKernelStore } from '../index.ts'; describe('reference count audit', () => { let kernelStore: ReturnType; + let kdb: KernelDatabase; /** * Register and initialize an endpoint so it can hold c-list entries. @@ -19,8 +21,35 @@ describe('reference count audit', () => { } } + /** + * Overwrite a kref's stored count, going around the store's own arithmetic so + * that drift can be introduced in either direction regardless of what the + * current count happens to be. + * + * @param kref - The kref whose count to overwrite. + * @param counts - The count text, in the store's encoding. + */ + function setStoredCount(kref: KRef, counts: string): void { + kdb.kernelKVStore.set(`${kref}.refCount`, counts); + } + + /** + * Shift every component of a count by the same amount. + * + * @param counts - The count text, in the store's encoding. + * @param delta - How far to shift each component. + * @returns The shifted count text. + */ + function shift(counts: string, delta: number): string { + return counts + .split(',') + .map((part) => `${Number(part) + delta}`) + .join(','); + } + beforeEach(() => { - kernelStore = makeKernelStore(makeMapKernelDatabase()); + kdb = makeMapKernelDatabase(); + kernelStore = makeKernelStore(kdb); kernelStore.markInitialized(); givenVats('v1', 'v2', 'v3'); }); @@ -172,6 +201,166 @@ describe('reference count audit', () => { }); }); + // The clean-audit cases above prove each rule agrees with whatever the store + // did, which stays true if a rule and the code it mirrors are wrong by the + // same constant. These pin each credit source to a literal count and holder + // label, and check drift in both directions: too low collects a live + // capability, too high leaks it. + describe('each credit source, on its own', () => { + const sources: { + what: string; + hold: () => KRef; + expected: string; + holders: string[]; + }[] = [ + { + what: 'an object import a vat still reaches', + hold: () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + return kref; + }, + expected: '1,1', + holders: ['v2 c-list import o-1'], + }, + { + what: 'an object import a vat has dropped but not retired', + hold: () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + return kref; + }, + expected: '0,1', + holders: ['v2 c-list import o-1'], + }, + { + what: 'a pinned object', + hold: () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.pinObject(kref); + return kref; + }, + expected: '1,1', + holders: ['pin'], + }, + { + what: "a run-queue send's target and slot", + hold: () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.enqueueRun({ + type: 'send', + target: kref, + message: { methargs: { body: '#[]', slots: [kref] }, result: null }, + }); + kernelStore.incrementRefCount(kref, 'queue|target'); + kernelStore.incrementRefCount(kref, 'queue|slot'); + return kref; + }, + expected: '2,2', + holders: ['run queue #1 send target', 'run queue #1 send slot'], + }, + { + what: "a run-queue send's result promise", + hold: () => { + const target = kernelStore.exportFromEndpoint('v1', 'o+1'); + const kpid = kernelStore.initKernelPromise()[0]; + kernelStore.enqueueRun({ + type: 'send', + target, + message: { methargs: { body: '#[]', slots: [] }, result: kpid }, + }); + kernelStore.incrementRefCount(target, 'queue|target'); + kernelStore.incrementRefCount(kpid, 'queue|result'); + return kpid; + }, + expected: '2', + holders: ['unsettled promise', 'run queue #1 send result'], + }, + { + what: 'a queued notification', + hold: () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.enqueueRun({ type: 'notify', endpointId: 'v2', kpid }); + kernelStore.incrementRefCount(kpid, 'notify'); + return kpid; + }, + expected: '3', + holders: [ + 'unsettled promise', + 'run queue #1 notify', + 'v1 c-list export p+1', + ], + }, + { + // `enqueuePromiseMessage` takes the references itself, which is the + // point of the transfer-don't-duplicate fix; incrementing here too + // would be the double-count it exists to prevent. + what: 'a message parked on an unresolved promise', + hold: () => { + const target = kernelStore.exportFromEndpoint('v1', 'o+1'); + const kpid = kernelStore.initKernelPromise()[0]; + kernelStore.enqueuePromiseMessage(kpid, { + methargs: { body: '#[]', slots: [target] }, + result: null, + }); + return kpid; + }, + expected: '2', + holders: ['unsettled promise', 'kp1 queue #1 target'], + }, + { + what: 'a promise nobody has settled yet', + hold: () => kernelStore.initKernelPromise()[0], + expected: '1', + holders: ['unsettled promise'], + }, + { + what: "a settled promise's resolution slot", + hold: () => { + const koid = kernelStore.exportFromEndpoint('v1', 'o+1'); + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.incrementRefCount(koid, 'resolve|slot'); + kernelStore.resolveKernelPromise(kpid, false, { + body: '#"$0"', + slots: [koid], + }); + return koid; + }, + expected: '1,1', + holders: ['kp1 resolution slot'], + }, + { + what: "a promise's own c-list entries", + hold: () => { + const kpid = kernelStore.exportFromEndpoint('v1', 'p+1'); + kernelStore.translateRefKtoE('v2', kpid, true); + return kpid; + }, + expected: '3', + holders: [ + 'unsettled promise', + 'v1 c-list export p+1', + 'v2 c-list import p-1', + ], + }, + ]; + + it.each(sources)('credits $what exactly', ({ hold, expected, holders }) => { + const kref = hold(); + + expect(kernelStore.auditRefCounts()).toStrictEqual([]); + + for (const delta of [1, -1]) { + const stored = shift(expected, delta); + setStoredCount(kref, stored); + expect(kernelStore.auditRefCounts()).toStrictEqual([ + { kind: 'mismatch', kref, stored, expected, holders }, + ]); + } + }); + }); + describe('assertRefCountsIfAuditing', () => { it('does nothing while auditing is off', () => { const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index 14a084724a..128cb38d7f 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -262,6 +262,15 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * Compare every kref's stored reference counts against the references the * kernel can be seen to hold. * + * What this can and cannot find is worth being precise about, because the + * ground truth here *is* the holder set. A count that disagrees with its + * holders is caught in either direction: too low, and a live capability can be + * collected; too high with no holder left, and the count itself is orphaned. + * But a holder that should have been torn down and wasn't justifies its own + * count — at any value — so a leaked *reference* is invisible to this by + * construction. A c-list entry that outlives what it names is the case that + * matters: see the settled-promise TODO in `KernelRouter`. + * * @returns The krefs whose counts disagree with ground truth, in kref order. */ function auditRefCounts(): RefCountViolation[] { From 0e80f43c774616934b5a87d2f8b791a83a004503 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 10 Aug 2026 13:37:46 +0200 Subject: [PATCH 32/34] fix(ocap-kernel): don't commit a GC release a restarting vat disagrees with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releasing the kernel's side of a garbage-collection action when the endpoint has vanished is right for an endpoint that is gone, and wrong for one that is merely out of reach. `restartVat` keeps the vat's c-list and takes the vat out of the kernel's vat table for as long as launching a worker and negotiating with it takes, so a GC action selected in that window found the vat absent, released entries the returning incarnation still holds, and committed — leaving the vat free to mint fresh krefs for objects the kernel thinks it let go of. That is the same divergence the failed-delivery path below rolls back to avoid. The endpoint is now resolved before anything is torn down, so the outcome is decided rather than discovered halfway through, and the release commits only where the endpoint is genuinely gone: a vat the store has marked terminated, whose cleanup tears the whole c-list down regardless, or a remote, which reconciles on its next incarnation. A vat that is absent yet not terminated fails the crank instead, which is what this path did before the release was added to it. This does not make a vat restart safe, and is not trying to: it stops the GC path from turning that window into silent corruption. The window itself needs the vat to stop being unreachable while it restarts — `restartVat` is an RPC handler mutating kernel state alongside a running run loop, which a send already resolves as a splat and a `notify` already dies on. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/src/KernelRouter.test.ts | 48 ++++++++++++++++++- packages/ocap-kernel/src/KernelRouter.ts | 42 ++++++++++++---- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/packages/ocap-kernel/src/KernelRouter.test.ts b/packages/ocap-kernel/src/KernelRouter.test.ts index eed87fe4d7..e88c54b52c 100644 --- a/packages/ocap-kernel/src/KernelRouter.test.ts +++ b/packages/ocap-kernel/src/KernelRouter.test.ts @@ -67,6 +67,7 @@ describe('KernelRouter', () => { forgetKref: vi.fn(), orphanKernelObject: vi.fn(), hasCListEntry: vi.fn().mockReturnValue(true), + isVatTerminated: vi.fn().mockReturnValue(false), createCrankSavepoint: vi.fn(), } as unknown as KernelStore; @@ -788,10 +789,13 @@ describe('KernelRouter', () => { expect(kernelStore.orphanKernelObject).not.toHaveBeenCalled(); }); - it('still releases the kernel side when the endpoint has vanished', async () => { + it('still releases the kernel side when a terminated vat has vanished', async () => { getEndpoint.mockImplementationOnce(() => { throw Error('vat v1 not found'); }); + ( + kernelStore.isVatTerminated as unknown as MockInstance + ).mockReturnValue(true); const result = await kernelRouter.deliver({ type: 'retireImports', @@ -809,6 +813,48 @@ describe('KernelRouter', () => { ); }); + it('still releases the kernel side when a remote has vanished', async () => { + getEndpoint.mockImplementationOnce(() => { + throw Error('remote r1 not found'); + }); + + const result = await kernelRouter.deliver({ + type: 'retireImports', + endpointId: 'r1', + krefs: ['ko1'], + }); + + expect(result).toStrictEqual({ didDelivery: 'r1' }); + expect(kernelStore.deleteCListEntry).toHaveBeenCalledWith( + 'r1', + 'ko1', + 'translated-ko1', + ); + }); + + it.each(['dropExports', 'retireExports', 'retireImports'] as const)( + 'refuses to release %s for a vat that is absent but not terminated', + async (actionType) => { + // A vat between incarnations still holds every one of these krefs, so + // committing the kernel's release would leave the two disagreeing. + getEndpoint.mockImplementationOnce(() => { + throw Error('vat v1 not found'); + }); + + await expect( + kernelRouter.deliver({ + type: actionType, + endpointId: 'v1', + krefs: ['ko1'], + }), + ).rejects.toThrow('vat v1 not found'); + + expect(kernelStore.clearReachableFlag).not.toHaveBeenCalled(); + expect(kernelStore.deleteCListEntry).not.toHaveBeenCalled(); + expect(kernelStore.orphanKernelObject).not.toHaveBeenCalled(); + }, + ); + it('skips krefs already cleaned up before delivery', async () => { ( kernelStore.hasCListEntry as unknown as MockInstance diff --git a/packages/ocap-kernel/src/KernelRouter.ts b/packages/ocap-kernel/src/KernelRouter.ts index cbd36b36a5..5117905f36 100644 --- a/packages/ocap-kernel/src/KernelRouter.ts +++ b/packages/ocap-kernel/src/KernelRouter.ts @@ -447,6 +447,39 @@ export class KernelRouter { if (live.length === 0) { return { didDelivery: endpointId }; } + // Resolved before anything is torn down, so a lookup that fails has nothing + // to undo, and so the two outcomes below are decided rather than discovered + // halfway through. + let endpoint: EndpointHandle | undefined; + try { + endpoint = this.#getEndpoint(endpointId); + } catch (error) { + // A vat absent from the kernel's vat table but not marked terminated is a + // vat between incarnations, and its c-list is whole: every kref here is one + // the returning incarnation still has in its own tables. `restartVat` + // takes a vat out of that table for as long as launching a worker and + // negotiating with it takes, so this is reachable, and releasing the + // kernel's side would commit exactly the disagreement the failed delivery + // below rolls back to avoid — the vat would mint fresh krefs for objects + // the kernel thinks it let go of. Fail the crank rather than commit that. + // Nothing here can make the restart safe: the action is already spent from + // the durable set, and a crank that neither delivers nor releases would + // simply be handed the same action again on the next one. + if ( + isVatId(endpointId) && + !this.#kernelStore.isVatTerminated(endpointId) + ) { + throw error; + } + // A terminated vat's cleanup tears its c-list down wholesale, and a remote + // reconciles on its next incarnation, so for those the release below is + // safe to commit — and has to be, since the action is already spent from + // the durable set. + this.#logger?.error( + `Endpoint ${endpointId} vanished before ${type} of ${JSON.stringify(live)}; releasing the kernel's side anyway:`, + error, + ); + } const erefs = this.#kernelStore.krefsToErefs(endpointId, live); // Telling an endpoint to let go is also the kernel letting go. Otherwise a // dropped export stays flagged reachable, so the same action gets derived @@ -469,14 +502,7 @@ export class KernelRouter { this.#kernelStore.orphanKernelObject(kref, endpointId); } }); - let endpoint: EndpointHandle; - try { - endpoint = this.#getEndpoint(endpointId); - } catch (error) { - this.#logger?.error( - `Endpoint ${endpointId} vanished before ${type} of ${JSON.stringify(live)}; released the kernel's side anyway:`, - error, - ); + if (!endpoint) { return { didDelivery: endpointId }; } const method = From 2bcfb098cce41a1816a6407b5dc2ccba2c3440ee Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:14:40 +0200 Subject: [PATCH 33/34] test(ocap-kernel): cover the export-ownership guard and the launch cleanup path Both are new here and neither was reached by a test. The ownership guard is the one that matters: nothing upstream of `performExportCleanup` checks that a vref is even an export, so without it a vat can disown another live vat's object. Removing the guard now fails two cases rather than none. `launchVat`'s registration failure is covered through the store calls it makes, since `VatManager` is hardened and cannot be spied on. Co-Authored-By: Claude Opus 5 (1M context) --- .../garbage-collection/gc-handlers.test.ts | 115 ++++++++++++++++++ .../ocap-kernel/src/vats/VatManager.test.ts | 49 ++++++++ 2 files changed, 164 insertions(+) create mode 100644 packages/ocap-kernel/src/garbage-collection/gc-handlers.test.ts diff --git a/packages/ocap-kernel/src/garbage-collection/gc-handlers.test.ts b/packages/ocap-kernel/src/garbage-collection/gc-handlers.test.ts new file mode 100644 index 0000000000..8b5231eec9 --- /dev/null +++ b/packages/ocap-kernel/src/garbage-collection/gc-handlers.test.ts @@ -0,0 +1,115 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../test/storage.ts'; +import { makeKernelStore } from '../store/index.ts'; +import type { VatConfig, VatId } from '../types.ts'; +import { performExportCleanup } from './gc-handlers.ts'; + +describe('performExportCleanup', () => { + let kernelStore: ReturnType; + + /** + * Register and initialize an endpoint so it can hold c-list entries. + * + * @param vatIds - The vats to bring into existence. + */ + function givenVats(...vatIds: VatId[]): void { + for (const vatId of vatIds) { + kernelStore.setVatConfig(vatId, { sourceSpec: 'x' } as VatConfig); + kernelStore.initEndpoint(vatId); + } + } + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + kernelStore.markInitialized(); + givenVats('v1', 'v2'); + }); + + // `checkReachable` is what separates a retire from an abandon; the ownership + // check precedes it, so both syscalls have to be covered. + const actions = [ + { name: 'retireExports', checkReachable: true }, + { name: 'abandonExports', checkReachable: false }, + ] as const; + + it.each(actions)( + 'lets an owner give up its own export via $name', + ({ checkReachable }) => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.clearReachableFlag('v1', kref); + + performExportCleanup([kref], checkReachable, 'v1', kernelStore); + + expect(kernelStore.getOwner(kref)).toBeUndefined(); + expect(kernelStore.hasCListEntry('v1', kref)).toBe(false); + }, + ); + + it.each(actions)( + 'refuses $name for an object owned by another endpoint', + ({ name, checkReachable }) => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + // v2 holds it as an import, which is what makes the kref nameable in a + // syscall from v2 at all. + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + + expect(() => + performExportCleanup([kref], checkReachable, 'v2', kernelStore), + ).toThrow(`endpoint v2 issued ${name} for ${kref}, which is owned by v1`); + + // v1's claim survives intact, entry and ownership both. + expect(kernelStore.getOwner(kref)).toBe('v1'); + expect(kernelStore.hasCListEntry('v1', kref)).toBe(true); + expect(kernelStore.hasCListEntry('v2', kref)).toBe(true); + }, + ); + + it.each(actions)( + 'allows $name for an already-orphaned object', + ({ checkReachable }) => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + kernelStore.translateRefKtoE('v2', kref, true); + kernelStore.clearReachableFlag('v2', kref); + kernelStore.clearReachableFlag('v1', kref); + kernelStore.forgetKref('v1', kref); + kernelStore.orphanKernelObject(kref, 'v1'); + + // No claim is left to erase, so there is nothing for the guard to protect. + expect(() => + performExportCleanup([kref], checkReachable, 'v2', kernelStore), + ).not.toThrow(); + expect(kernelStore.hasCListEntry('v2', kref)).toBe(false); + }, + ); + + it('refuses retireExports for an object the owner still reaches', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + + expect(() => performExportCleanup([kref], true, 'v1', kernelStore)).toThrow( + `retireExports but ${kref} is still reachable`, + ); + expect(kernelStore.getOwner(kref)).toBe('v1'); + }); + + it('abandons an export the owner still reaches', () => { + const kref = kernelStore.exportFromEndpoint('v1', 'o+1'); + + performExportCleanup([kref], false, 'v1', kernelStore); + + expect(kernelStore.getOwner(kref)).toBeUndefined(); + }); + + it.each(actions)( + 'refuses $name for a promise', + ({ name, checkReachable }) => { + const kpid = kernelStore.initKernelPromise()[0]; + kernelStore.exportFromEndpoint('v1', 'p+1'); + + expect(() => + performExportCleanup([kpid], checkReachable, 'v1', kernelStore), + ).toThrow(`endpoint v1 issued invalid ${name} for ${kpid}`); + }, + ); +}); diff --git a/packages/ocap-kernel/src/vats/VatManager.test.ts b/packages/ocap-kernel/src/vats/VatManager.test.ts index 21361f942c..e437d042b9 100644 --- a/packages/ocap-kernel/src/vats/VatManager.test.ts +++ b/packages/ocap-kernel/src/vats/VatManager.test.ts @@ -207,6 +207,55 @@ describe('VatManager', () => { expect((error as Error).cause).toBe(cause); }); + + it('tears the worker down when kernel-side registration fails', async () => { + const config = createMockVatConfig(); + const cause = new Error('initEndpoint threw'); + mockKernelStore.initEndpoint.mockImplementationOnce(() => { + throw cause; + }); + + const error = await vatManager + .launchVat(config, 'bob', 's1') + .catch((reason: unknown) => reason); + + expect((error as Error).message).toBe('Failed to launch vat v1 (bob)'); + expect((error as Error).cause).toBe(cause); + // The worker is already running by this point, so it has to be stopped, + // and the vat marked so the terminated-vat cleanup reclaims what the + // partial launch wrote. + expect(mockPlatformServices.terminate).toHaveBeenCalledWith( + 'v1', + expect.any(Error), + ); + expect(vatHandles[0]?.terminate).toHaveBeenCalled(); + expect(mockKernelStore.markVatAsTerminated).toHaveBeenCalledWith('v1'); + expect(vatManager.hasVat('v1')).toBe(false); + }); + + it('still marks the vat terminated when the cleanup itself fails', async () => { + const config = createMockVatConfig(); + const cause = new Error('setVatConfig threw'); + mockKernelStore.setVatConfig.mockImplementationOnce(() => { + throw cause; + }); + // `stopVat` unpins the root it was launched with, which is the first + // thing in the teardown that can fail. + mockKernelStore.unpinObject.mockImplementationOnce(() => { + throw new Error('worker will not die'); + }); + + const error = await vatManager + .launchVat(config, 'bob', 's1') + .catch((reason: unknown) => reason); + + expect((error as Error).message).toBe( + 'Failed to launch vat v1 (bob) (cleanup also failed)', + ); + // The launch failure, not the cleanup failure, is what the caller needs. + expect((error as Error).cause).toBe(cause); + expect(mockKernelStore.markVatAsTerminated).toHaveBeenCalledWith('v1'); + }); }); describe('runVat', () => { From 10997709793ecd87079d30669c3a4dea99f46047 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 13 Aug 2026 17:45:14 +0200 Subject: [PATCH 34/34] docs(ocap-kernel): add changelog entries for GC delivery hardening Also narrows the audit's Added entry: this branch turns `RefCountViolation` into a discriminated union, and the audit compares counts against the holders it finds, so a holder that should have been torn down but wasn't justifies its own count. "A leak" overstated what it can detect. Co-Authored-By: Claude Opus 5 (1M context) --- packages/ocap-kernel/CHANGELOG.md | 13 +++++++++++-- .../ocap-kernel/src/store/methods/refcount-audit.ts | 4 ++-- packages/ocap-kernel/src/store/types.ts | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index 4c363fd234..86794f9103 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -38,12 +38,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Caveat: a legitimate string argument that begins with `@@` followed by alphanumerics will be misinterpreted as a marker; wrap such literals inside an object - Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high (a leak) + - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high with no holder (an orphaned count). It compares counts against the holders it finds, so a holder that should have been torn down but wasn't justifies its own count and is not detectable this way - Only references visible in the kernel's own state are checkable, so a holder that keeps a kref outside them has to take a pin to be counted at all - `recomputeRefCounts` is a repair tool for a drifted store, offered to embedders and never run automatically: opening an existing store does not migrate it - - Exports the `RefCountViolation` type + - Exports the `RefCountViolation` type, a union over `kind: 'mismatch' | 'dangling'` - Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Add `getOcapURLObjects` and `retainForOcapURL` to the kernel store, and `VatManager.releaseVatRootPin` ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) +- Add `orphanKernelObject` to the kernel store, which drops an object's owner mapping and hands it to the collector ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) ### Changed @@ -96,6 +97,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A root is addressable while its vat lives whether or not anyone imports it; the old `(1, 1)` birth baseline was standing in for this - Garbage-collection action delivery now moves the kernel's own c-list: `dropExports` clears the owner's reachable flag and `retireExports`/`retireImports` tear the entry down ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Previously the owner's flag never cleared, so the same action could be re-derived, and retired entries outlived the objects they named +- Orphan a kernel object when its owner stops naming it (a delivered `retireExport`, or a `retireExports`/`abandonExports` syscall), so its `owner` and `refCount` records no longer outlive the c-list entry they were reachable through ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) + - They leaked, and the next collection to visit such a kref read a c-list entry that was no longer there and killed the run loop. Reproduces on `main`, so it predates this stack +- Reject a `retireExports`/`abandonExports` syscall for an object the calling endpoint does not own, instead of letting it erase another endpoint's claim to an object it is still exporting ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) + - Nothing upstream of `performExportCleanup` checked that the vref it was handed is even an export, and the audit could not see the damage, because an export entry carries no count +- Garbage-collection action delivery releases the kernel's side even when the endpoint has vanished, and rolls back rather than committing a release the endpoint was never told about ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) + - It releases only where the endpoint is genuinely gone: a terminated vat, whose cleanup tears the whole c-list down anyway, or a remote, which reconciles on its next incarnation. A vat that is absent yet not terminated is one `restartVat` has taken out of the kernel's reach while keeping its c-list, so the crank fails there instead of committing a release the returning incarnation would disagree with +- A failed garbage-collection delivery to a remote is logged and survived rather than escaping the crank and stopping the run loop ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) +- Tear down and mark for cleanup a vat whose worker launched but whose kernel-side registration then failed ([#1022](https://github.com/MetaMask/ocap-kernel/pull/1022)) - Charge a delivered message's target reference against the run-queue item's own target rather than the routed target, so a message routed through a resolved promise no longer decrements an object nobody charged while leaking the promise ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) - Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1020](https://github.com/MetaMask/ocap-kernel/pull/1020)) diff --git a/packages/ocap-kernel/src/store/methods/refcount-audit.ts b/packages/ocap-kernel/src/store/methods/refcount-audit.ts index 128cb38d7f..27a6d912f4 100644 --- a/packages/ocap-kernel/src/store/methods/refcount-audit.ts +++ b/packages/ocap-kernel/src/store/methods/refcount-audit.ts @@ -374,7 +374,7 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * via the `auditRefCounts` option, and run at the end of every crank. */ function assertRefCountsIfAuditing(): void { - if (!ctx.refCountAuditingEnabled) { + if (!ctx.auditRefCounts) { return; } const violations = auditRefCounts(); @@ -393,7 +393,7 @@ export function getRefCountAuditMethods(ctx: StoreContext) { * @param enabled - Whether to audit after every crank. */ function setRefCountAuditing(enabled: boolean): void { - ctx.refCountAuditingEnabled = enabled; + ctx.auditRefCounts = enabled; } return { diff --git a/packages/ocap-kernel/src/store/types.ts b/packages/ocap-kernel/src/store/types.ts index 909c070d8f..b9886c174f 100644 --- a/packages/ocap-kernel/src/store/types.ts +++ b/packages/ocap-kernel/src/store/types.ts @@ -28,7 +28,7 @@ export type StoreContext = { subclusters: StoredValue; // Holds Subcluster[] nextSubclusterId: StoredValue; // Holds string vatToSubclusterMap: StoredValue; // Holds Record - refCountAuditingEnabled: boolean; // If true, verify refcounts against ground truth every crank + auditRefCounts: boolean; // If set, verify refcounts against ground truth every crank logger?: Logger | undefined; };