diff --git a/.changeset/sqlite-commits-off-event-loop.md b/.changeset/sqlite-commits-off-event-loop.md new file mode 100644 index 0000000000..98621ff8f2 --- /dev/null +++ b/.changeset/sqlite-commits-off-event-loop.md @@ -0,0 +1,9 @@ +--- +"@schlessera/brain-ui-server": minor +"@schlessera/brain-ui-sdk": minor +"@schlessera/brain-backend-claude": patch +--- + +Changed: the hosted server commits turn-journal, Activity and native-lease SQLite writes on a write worker thread at `synchronous=FULL`, in submission order; the event loop awaits each commit instead of blocking in its fsync (#2814). +Changed (breaking for direct `TurnJournal` callers): journal writes that callers wait on (`bindSession`, `writeOperation`, Claude memory persistence, revert reservation, retained-version leases and retained streams) are asynchronous; `writeOperation` takes statements, and the journal exposes `ready` for its startup recovery. +Added: `StartTurnRequest.persistClaudeMemory` may return a promise; the Claude backend awaits it before persisting the next note or ending the turn. diff --git a/api-report/ui-server.txt b/api-report/ui-server.txt index 6f0599c165..a0b7a3f8eb 100644 --- a/api-report/ui-server.txt +++ b/api-report/ui-server.txt @@ -162,6 +162,8 @@ signatures (every public export and the types it is made of) subscriptionCount(): number; close(): void; } + ActivityWriteMethod (packages/ui-server/src/activity/store.ts) + export type ActivityWriteMethod = "writeLifecycle" | "startSpan" | "endSpan" | "patchSpan" | "appendEvent" | "heartbeat" | "cascadeClose" | "sweepOwnOrphans" | "sweepStale" | "rollupRun" | "prune"; AgentConfig (packages/ui-server/src/config/env.ts) export interface AgentConfig { backend: string | null; @@ -214,6 +216,12 @@ signatures (every public export and the types it is made of) retain(): () => void; release(): void; } + Awaitable (packages/ui-server/src/activity/store.ts) + type Awaitable = T | Promise; + AwaitableActivityStore (packages/ui-server/src/activity/store.ts) + export type AwaitableActivityStore = Omit & { + [K in ActivityWriteMethod]: (...args: Parameters) => Awaitable>; + }; BackendRegistry (packages/ui-server/src/agent/backend.ts) export interface BackendRegistry { getBackends(): Promise; @@ -455,7 +463,7 @@ signatures (every public export and the types it is made of) log?: Logger; meter?: Meter; activity?: { - store: Pick; + store: Pick; onWrite?: () => void; }; } @@ -661,6 +669,15 @@ signatures (every public export and the types it is made of) } InboxIntake (packages/ui-server/src/inbox/intake.ts) export type InboxIntake = ReturnType; + JournalStatement (packages/ui-server/src/brain/journal-store.ts) + export type JournalStatement = { + sql: string; + params?: SQLQueryBindings[]; + } | { + check: string; + params?: SQLQueryBindings[]; + refusal: string; + }; LABEL_MAX_CHARS (packages/ui-server/src/labels/normalise.ts) const LABEL_MAX_CHARS: 32; LABEL_RUN_NAME (packages/ui-server/src/labels/labeller.ts) @@ -1260,6 +1277,18 @@ signatures (every public export and the types it is made of) } TurnJournal (packages/ui-server/src/brain/turn-journal.ts) export type TurnJournal = ReturnType; + TurnJournalOptions (packages/ui-server/src/brain/turn-journal.ts) + export interface TurnJournalOptions { + brainPath: string; + dbPath: string; + objectsRoot: string; + capBytes?: number; + now?: () => number; + nativeManager: NativeJournalManager; + referencePool?: ReferenceWorktreeJournalPool; + referenceCheckpoints?: ReferenceWorktreeCheckpointLedger; + changed?: (turnId: string, sessionId: string, summary: TurnJournalSummary) => void; + } VoiceConfig (packages/ui-server/src/config/env.ts) export interface VoiceConfig { provider: string | null; @@ -1643,17 +1672,7 @@ signatures (every public export and the types it is made of) log?: Logger; }): BackendRegistry; createTurnJournal (packages/ui-server/src/brain/turn-journal.ts) - export function createTurnJournal(db: Database, options: { - brainPath: string; - dbPath: string; - objectsRoot: string; - capBytes?: number; - now?: () => number; - nativeManager: NativeJournalManager; - referencePool?: ReferenceWorktreeJournalPool; - referenceCheckpoints?: ReferenceWorktreeCheckpointLedger; - changed?: (turnId: string, sessionId: string, summary: TurnJournalSummary) => void; - }): { + export function createTurnJournal(db: Database, options: TurnJournalOptions): { attachWorktree(turnId: string, worktreeId: string, signal?: AbortSignal): Promise; begin(turnId: string, metadata: { actor: "interactive" | "user" | "voice"; @@ -1664,7 +1683,7 @@ signatures (every public export and the types it is made of) sessionId: null | string; }, stop: (error: Error) => void, signal?: AbortSignal): Promise; bindModel(turnId: string, sessionId: null | string, model: string): boolean; - bindSession(turnId: string, sessionId: string): void; + bindSession(turnId: string, sessionId: string): Promise; claudeMemoryTarget(turnId: string, path: string): null | { device: string; directory: string; @@ -1675,12 +1694,12 @@ signatures (every public export and the types it is made of) coverPaths(turnId: string, paths: readonly string[]): void; details(sessionId: string, turnId: string): TurnJournalDetails | null; finish(turnId: string): Promise; - finishReserved(turnId: string): void; + finishReserved(turnId: string): Promise; hasRetainedVersion(turnId: string, selected: JournalVersion): boolean; idle(): Promise; isHeld(): boolean; observe(): Promise; - observeUserPolicy(turnId: string, approvedWrite: () => void): Promise; + observeUserPolicy(turnId: string, approvedWrite: () => Promise | void): Promise; onObservation(observer: (turnId: string, facts: { changedPaths: number; observedBytes: number; @@ -1695,29 +1714,30 @@ signatures (every public export and the types it is made of) beforeMode: number; directory: string; note: string; - }, save: () => void): void; + }, save: () => void): Promise; + ready: Promise; recordClaudeMemory(turnId: string, path: string, target: { device: string; directory: string; inode: string; note: string; - }, before: JournalVersion | null, after: JournalVersion | null): void; + }, before: JournalVersion | null, after: JournalVersion | null): Promise; removalCoverage(paths: readonly string[], signal?: AbortSignal, absent?: boolean): Promise; - reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]): void; + reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]): Promise; retainBefore(turnId: string, path: string, signal?: AbortSignal): Promise; summary: (turnId: string) => TurnJournalSummary; version(sessionId: string, turnId: string, sequence: number, side: "after" | "before"): null | { bytes: (maxBytes: number) => Buffer; selected: JournalVersion; - stream(): { + stream(): Promise<{ body: ReadableStream; bytes: number; - }; + }>; }; - withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): T; - withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): T; + withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): Promise; + withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): Promise; worktrees(turnId: string): string[]; - writeOperation(write: () => T): T; + writeOperation(statements: readonly JournalStatement[]): Promise; }; inspect (packages/ui-server/src/inbox/resolve.ts) function inspect(principalId: string, msg: ClientInboxResolve): { diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 987b829c8e..a370b129d5 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -46,6 +46,7 @@ somebody already learn the hard way?* | [action-notifications.md](action-notifications.md) | Why Action notices use fixed cross-thread batches, client-local quiet/digest schedules, new waiting episodes and durable per-destination submission/coverage history. | | [hygiene-review.md](hygiene-review.md) | Why async hygiene review orders eligible findings by severity/known urgency, merges equivalent source reports and invalidates dispositions only on relevant evidence changes. | | [agent-observability.md](agent-observability.md) | Runs, spans, the Activity surface, the failure inbox and the digest. | +| [sqlite-write-worker.md](sqlite-write-worker.md) | Why journal, Activity and native-lease commits run at `synchronous=FULL` on a per-database write worker thread, in submission order, while the event loop awaits them. | | [cost-tracking.md](cost-tracking.md) | List price versus effective cost, and why unknown cost is never rendered as zero. | | [design-feedback.md](design-feedback.md) | The design defects building the kit found, the measurement behind each, and the ruling that settled it. Cited by section number from `packages/ui-kit` source. | | [geo-operations.md](geo-operations.md) | Why one independent concrete geo library owns shared operations, with strict travel compatibility and explicit recovered-track evidence. | diff --git a/docs/decisions/async-collaboration.md b/docs/decisions/async-collaboration.md index 39e6f497a8..fca36bba98 100644 --- a/docs/decisions/async-collaboration.md +++ b/docs/decisions/async-collaboration.md @@ -71,7 +71,7 @@ requires one enforcement mechanism with different named memberships for voice and unattended work. Keep that requirement when implementing availability control. Existing mandatory backend conformance supports safe rejection of an unsupported restricted request; silently ignoring it is forbidden. The current -optional inputs are (`enforceAllowedTools?: boolean`, `packages/ui-sdk/src/server/backend.ts:586`) and (`noGrantSurface?: boolean`, `packages/ui-sdk/src/server/backend.ts:610`). These are permission primitives, +optional inputs are (`enforceAllowedTools?: boolean`, `packages/ui-sdk/src/server/backend.ts:589`) and (`noGrantSurface?: boolean`, `packages/ui-sdk/src/server/backend.ts:613`). These are permission primitives, not a claim of filesystem or network containment. The ordinary Claude assembly loads project settings and appends bridge tools @@ -204,7 +204,7 @@ permission and unwinds the turn. Already completed side effects are not undone by an abort. Attempt staging and cleanup therefore need idempotent recovery, with filesystem compensation outside SQLite transactions. Ordinary permission parking still exists (`requestPermission: (req) => {`, -`packages/ui-server/src/ws/bridge.ts:310-403`); it is not a durable Action store. +`packages/ui-server/src/ws/bridge.ts:311-404`); it is not a durable Action store. Resolution validates the stored effect again, checks current authority, records one resolution and applies the guarded state transition in one transaction. @@ -241,7 +241,7 @@ suppressed-per-model Action; unknown usage cannot become a guessed zero. Subscription billing moves the turn counter even when spend is zero. The activity sum exposes unpriced runs (`sumEffectiveCost`, `packages/ui-server/src/activity/store.ts:179-194`) and settles only after execution -(`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:705`). +(`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:723`). Retrospective totals alone cannot enforce admission. The maintainer's [2026-10-01 ruling on #678](https://github.com/schlessera/brain-kit/issues/678#issuecomment-5926691626) diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index f88785b28c..81f5078030 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -64,7 +64,7 @@ next to it. of this binary. The only version probe in the server is for the `brain` CLI (`Probe the selected content CLI`, `packages/ui-server/src/brain/client.ts:142-210`, called at - `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:433`). + `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:434`). `brain doctor` runs `claude mcp list` from `PATH` (`which("claude")`, `packages/core/src/cli/commands/doctor.ts:501-505`) — the user's own Claude Code on their own machine, to check the MCP registration, not the server's @@ -212,7 +212,7 @@ unnecessary. `node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs`. The boot probe below turns either into a refusal to start, for the same reason a missing backend refuses to boot (`A missing (or unrecognized) agent backend`, - `packages/ui-server/src/app.ts:371-376`). + `packages/ui-server/src/app.ts:372-377`). - **The published range is still a caret, and that bounds what this repo can guarantee.** `@schlessera/brain-backend-claude` depends on `^0.3.241`, and a host resolves it in its own lockfile. A host can bump the SDK — and so the @@ -468,7 +468,7 @@ into them. billing now stays unknown for every origin; server credentials and an observed runtime version cannot establish a child's billing. The recorded root-span attribute is the source of truth - (`const attrBilling`, `packages/ui-server/src/activity/store.ts:537-538`); + (`const attrBilling`, `packages/ui-server/src/activity/store.ts:554-555`); [cost-tracking.md](cost-tracking.md#recorded-billing-is-the-rollup-source-of-truth-293) records the 2026-09-30 ruling and frozen-history semantics. - **Model discovery prefers the API key** and describes that as "mirroring the diff --git a/docs/decisions/cost-tracking.md b/docs/decisions/cost-tracking.md index 2020100248..579a7ba755 100644 --- a/docs/decisions/cost-tracking.md +++ b/docs/decisions/cost-tracking.md @@ -128,7 +128,7 @@ math and authoritative backend cost remain available independently; aggregates exclude unknown effective costs from their sum and count the runs as unpriced. Explicit valid subscription/API classifications keep their existing behavior. The rollup reads the root attribute and prices child usage for cron locally -(`const attrBilling`, `packages/ui-server/src/activity/store.ts:537-538`). +(`const attrBilling`, `packages/ui-server/src/activity/store.ts:554-555`). This is a change to the evidence used for new computations, not a historical backfill. First non-null billing, list cost, effective cost and estimate values @@ -156,7 +156,7 @@ still satisfies the interface. The host declares the billing mode with `CreateAppOptions.labeller.billing`; nothing infers it. The run is written so the existing rollup prices it without a rule of its -own (`const endRun`, `packages/ui-server/src/labels/labeller.ts:235-251`). The +own (`const endRun`, `packages/ui-server/src/labels/labeller.ts:242-257`). The reported usage and model become the root's `gen_ai.usage.per_model`, and the declared billing is written as `brain.billing_mode` **only beside usage and a model**. The rollup prices a subscription run at $0 whatever its tokens, so a diff --git a/docs/decisions/governed-writable-workers.md b/docs/decisions/governed-writable-workers.md index 8bd3c3fc04..fb1b2b767d 100644 --- a/docs/decisions/governed-writable-workers.md +++ b/docs/decisions/governed-writable-workers.md @@ -163,7 +163,7 @@ still holds. **Voice.** Ruling 1 makes voice `governed` too. Voice keeps its narrower membership (`VOICE_ALLOWED_TOOLS`, `packages/ui-backend-claude/src/tool-policy.ts:120`): no Bash, no raw `Write`/`Edit`, no archive, and approval returns false -(`const apply = createBrainApplication`, `packages/ui-server/src/ws/bridge.ts:613-618`). +(`const apply = createBrainApplication`, `packages/ui-server/src/ws/bridge.ts:614-619`). A voice turn therefore writes only through curated capture and update, now with a journal behind it. The writable mount gives voice no tool it lacked. Pi still refuses voice turns. @@ -1412,7 +1412,7 @@ limits from the same policy inputs and states independent hard limits. **What exists.** The audit trail is the Activity span record: tool name, the full input up to its cap, output and approval decisions -(`"tool_input"`, `packages/ui-server/src/activity/recorder.ts:301`). +(`"tool_input"`, `packages/ui-server/src/activity/recorder.ts:313`). **What changes:** @@ -2078,7 +2078,7 @@ every turn without a proven recipient: - **Auto-memory.** A denied Claude worker gets no saved project `memory/`, and no note it writes is saved, neither a new name nor a replacement of an existing one (`omitMemory`, - `packages/ui-backend-claude/src/worker-state.ts:284-284`). Its notes stay + `packages/ui-backend-claude/src/worker-state.ts:285-285`). Its notes stay in its tmpfs and are discarded with it, so no note can collide with a saved one and refuse the turn's transcript. The decision is taken from the host's actual masks (`sandbox.providerMasks.folders`), never from a @@ -3110,7 +3110,7 @@ attribution. If an existing same-session/turn taint event precedes an agent observation, the journal retains its source ID with that change. Replay never infers taint from a later event, and absence of evidence does not assert complete untrusted-input detection. User history operations are not tainted agent writes -(`async function observe`, `packages/ui-server/src/brain/turn-journal.ts:222-285`). +(`async function observe`, `packages/ui-server/src/brain/turn-journal.ts:249-312`). Claude auto-memory persists outside the authoritative brain. Its validated native copy-out wraps each changed direct `.md` note in host journal retention: @@ -3134,7 +3134,7 @@ providers or a claim of capturing intervening external writes. The copy-out boundary is `const write`, `packages/ui-backend-claude/src/worker-state.ts:210-224`; retention/finalization is `persistClaudeMemory`, `packages/ui-server/src/brain/turn-journal.ts:462-497`; destination-bound recovery -is `const target`, `packages/ui-server/src/brain/turn-revert.ts:106-116`. +is `const target`, `packages/ui-server/src/brain/turn-revert.ts:108-118`. ### Concurrent edits @@ -3167,7 +3167,7 @@ exclusive "mine" or "theirs" ownership. A conflict requires two distinct observed after-versions sharing an overlapping turn; one version remains shared attribution. User choices use the existing authenticated history operation, comparing current raw state with the reviewed latest observation before effects -(`resolveSelection`, `packages/ui-server/src/brain/turn-revert.ts:49-72`). +(`resolveSelection`, `packages/ui-server/src/brain/turn-revert.ts:50-73`). Keep both creates the other version beside the selected file with sync's numbered `-remote` convention, using exclusive descriptor-rooted creation, existing CAS leases and explicit coverage for its final generated name diff --git a/docs/decisions/http-api-boundary.md b/docs/decisions/http-api-boundary.md index 6439df2b5f..8d96b28f2a 100644 --- a/docs/decisions/http-api-boundary.md +++ b/docs/decisions/http-api-boundary.md @@ -46,11 +46,11 @@ explicit supported rows, not silently excluded by this distinction. The source audit on 2026-09-30 used `e0875269b21b707482545be9e94901acb897893b`. The public health mount precedes auth -(`createHealthRoutes`, `packages/ui-server/src/app.ts:847`), the API guard follows -public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:868`), and the -WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:1057`). +(`createHealthRoutes`, `packages/ui-server/src/app.ts:856`), the API guard follows +public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:877`), and the +WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:1066`). Static serving is conditional (`if (options.staticRoot)`, -`packages/ui-server/src/app.ts:1077-1098`). That order establishes access; +`packages/ui-server/src/app.ts:1086-1107`). That order establishes access; it does not select compatibility status. A hermetic real-app launch, with temporary brain root, in-memory database, diff --git a/docs/decisions/live-conversation.md b/docs/decisions/live-conversation.md index 3e2ab8fa72..2948c668f0 100644 --- a/docs/decisions/live-conversation.md +++ b/docs/decisions/live-conversation.md @@ -30,7 +30,7 @@ tool admission and execution, permissions, cancellation, terminal state and resynchronization. Engine task requests are advisory until mapped to committed user input. No native search, code-execution or alternate permission loop is enabled. Both adapters can delegate through the existing `AgentBackend` -(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:691-734`); +(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:694-737`); the host serializes work in a session and uses capability-checked follow-ups. The [voice-permission decision](voice-permission.md) binds this interface. diff --git a/docs/decisions/managed-worktrees.md b/docs/decisions/managed-worktrees.md index ac5ebb60c9..423e738822 100644 --- a/docs/decisions/managed-worktrees.md +++ b/docs/decisions/managed-worktrees.md @@ -125,8 +125,8 @@ A managed worktree is not inside the brain, so the brain's checkpoints never cover it. The turn journal observes each covered worktree as its own root, through that worktree's read-only checkpoints and deltas and the same observer and capture as the brain -(`attachWorktree(turnId: string`, `packages/ui-server/src/brain/turn-journal.ts:394-406`; -`// Each managed worktree is observed through its own checkpoints`, `packages/ui-server/src/brain/turn-journal.ts:236-249`). +(`attachWorktree(turnId: string`, `packages/ui-server/src/brain/turn-journal.ts:385-397`; +`// Each managed worktree is observed through its own checkpoints`, `packages/ui-server/src/brain/turn-journal.ts:266-280`). Coverage starts with a baseline checkpoint and inventory before the grant, and every journal observation, periodic, at a scope cap or at the turn's end, then observes the worktree too. diff --git a/docs/decisions/policy-write-boundary.md b/docs/decisions/policy-write-boundary.md index c84671e292..39420380c4 100644 --- a/docs/decisions/policy-write-boundary.md +++ b/docs/decisions/policy-write-boundary.md @@ -509,12 +509,12 @@ host (`function* launchAgentWorkerSteps`, the brain can stand in for it, and the shell reads a pipe a byte at a time, so the runtime's own input is untouched. The server copies the snapshot in through the held descriptor (`populate(root) {`, - `packages/ui-backend-claude/src/worker-state.ts:230-252`), then opens the + `packages/ui-backend-claude/src/worker-state.ts:231-253`), then opens the gate. A failed copy kills the worker unstarted and releases everything. 3. After the worker exits, its PID namespace is gone, and with it every process that could write the tmpfs. The held descriptor still reaches the detached mount, so the server walks it then (`persist(root, rewrite, memory) {`, - `packages/ui-backend-claude/src/worker-state.ts:270-324`) with the existing + `packages/ui-backend-claude/src/worker-state.ts:271-325`) with the existing rules: only this brain's native transcripts and `memory/*.md` notes, every directory checked with `lstat` before it is entered, every file opened with `O_NOFOLLOW` and refused on the descriptor when it is not a regular file, diff --git a/docs/decisions/scheduled-tasks.md b/docs/decisions/scheduled-tasks.md index 7176771829..19597cccf1 100644 --- a/docs/decisions/scheduled-tasks.md +++ b/docs/decisions/scheduled-tasks.md @@ -23,7 +23,7 @@ Source inspected at `b5c30c180df5584584f261b446c4d440380d2441`: (`else if (item.status === "scheduled"`, `packages/ui-server/src/inbox/actions.ts:196-197`). The runtime ticks every minute (`INBOX_TICK_MS`, `packages/ui-server/src/inbox/runtime.ts:11-13`). - Production app wiring supplies recovery and budgets, without a dispatcher - (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:486-502`). + (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:491-507`). A scripted nonempty ready-item test still reports no claim, attempt or Activity root. That proves disabled wiring, not safe execution of a schedule. - Headless plumbing checks backend support, a usable principal, lifetime and diff --git a/docs/decisions/session-handoff.md b/docs/decisions/session-handoff.md index adb2901758..42be2ded46 100644 --- a/docs/decisions/session-handoff.md +++ b/docs/decisions/session-handoff.md @@ -137,7 +137,7 @@ server observation. No second warning band or capacity change was selected. The common threshold and copy are (`stateRoomWarning`, `packages/ui-sdk/src/state-room.ts:4-6`) and (`stateRoomCopy`, `packages/ui-sdk/src/state-room.ts:9-13`). Claude observes the completed actual -copy (`populate(root) {`, `packages/ui-backend-claude/src/worker-state.ts:230-252`); +copy (`populate(root) {`, `packages/ui-backend-claude/src/worker-state.ts:231-253`); Pi retains its existing payload/reserve admission (`checkSessionSize`, `packages/ui-backend-pi/src/worker-protocol.ts:21-26`). diff --git a/docs/decisions/session-principals.md b/docs/decisions/session-principals.md index 23b2fadd05..e9df574748 100644 --- a/docs/decisions/session-principals.md +++ b/docs/decisions/session-principals.md @@ -96,13 +96,13 @@ Both reviewers verified every row below against the source. | Cookie verify | `auth.ts:265-296` | strict `^(\d+)\.(\d+)$`, safe integers, future/expired rejection, `cookieEpoch === sessionsEpoch(db)` | | Global epoch | `auth.ts:302-318` | `settings` row; missing = 0; non-integer **throws** | | Global revoke | `auth.ts:321-330` | `setSetting` (overflow-guarded) + `clients.closeAll(1008, …)` | -| Auth routes | `authRoutes`, `app.ts:853` | mounted **before** the guard at `authGuard(authMode`, `app.ts:868`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:808`) | -| Passkey management | `passkeyManagementRoutes`, `app.ts:878` | mounted **after** the guard ("Mount AFTER the auth guard") | -| WS guard | `"/ws"`, `app.ts:1057-1074` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | +| Auth routes | `authRoutes`, `app.ts:862` | mounted **before** the guard at `authGuard(authMode`, `app.ts:877`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:808`) | +| Passkey management | `passkeyManagementRoutes`, `app.ts:887` | mounted **after** the guard ("Mount AFTER the auth guard") | +| WS guard | `"/ws"`, `app.ts:1066-1083` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | | WS upgrade | `ws/connection.ts:262-264` | `createWsUpgrade` ignores the request context | | WS admission | `ws/connection.ts:84` | `clients.add(ws)` — the socket's identity is unknown | | Turn record | `ws/run-session.ts:136-144` | recorder built from `{turnId, sessionId, billing}`; `RunningTurn` (`ws/turns.ts:55-77`) has no actor | -| Follow-up queue | `QueuedFollowUp`, `ws/turns.ts:61-103`; `slot.queue.push(entry)`, `run-session.ts:1085` | queued entries re-mint `turnId` in the same slot and may come from another socket | +| Follow-up queue | `QueuedFollowUp`, `ws/turns.ts:61-103`; `slot.queue.push(entry)`, `run-session.ts:1088` | queued entries re-mint `turnId` in the same slot and may come from another socket | | Password login | `acquirePasswordVerification(key)`, `auth.ts:749` | argon2id verify, failure counting, in-flight reservation, then `issueSessionCookie` | | Passkey login | `issueLoginSession`, `packages/ui-server/src/middleware/passkeys.ts:448` | assertion verified, then `issueSessionCookie`; `row.id` is in scope | | Passkey **registration** | `INSERT INTO passkey_credentials`, `passkeys.ts:547` | inserts a credential; calls **neither** helper | @@ -204,7 +204,7 @@ credentials, not people. (`upsertRollup`, `activity/sql.ts:19-24`) — the rollup is what survives span pruning, so cost-by-actor dies at prune time without it. The precedent is profile/billing: a root-span attr that the rollup reads (`spanId: rootSpanId`, -`activity/recorder.ts:211-226`). Nullable for cron (`origin: "cron"`, +`activity/recorder.ts:222-237`). Nullable for cron (`origin: "cron"`, `activity/span-sink.ts:102`, `cron/run-job.ts:113`) and for pre-migration rows; nothing is backfilled. diff --git a/docs/decisions/sqlite-write-worker.md b/docs/decisions/sqlite-write-worker.md new file mode 100644 index 0000000000..2ab85f7f33 --- /dev/null +++ b/docs/decisions/sqlite-write-worker.md @@ -0,0 +1,121 @@ +# SQLite commits off the event loop + +Ruling 1 on [#2789](https://github.com/schlessera/brain-kit/issues/2789), built +in [#2814](https://github.com/schlessera/brain-kit/issues/2814). + +## What was wrong + +The hosted server committed every turn-journal, Activity and native-lease write +on its event loop at `synchronous=FULL`. A FULL commit waits for the WAL's +fsync. On an idle disk that is a few milliseconds; with three `dd … conv=fsync` +writers on the same filesystem it measured 230–730 ms. While the loop waits, +`/api/health` and WebSocket pings wait too, which is the +`HOSTED_ADMISSION_HEALTH_WITHIN_1_SECOND` failure of +[#2416](https://github.com/schlessera/brain-kit/issues/2416). The lag windows +held only a few milliseconds of CPU samples: the time was spent in fsync, not in +accounting. + +The commit is the durability guarantee, so it stays. What changes is which +thread waits for it. + +## What we do + +- **One write worker per database file.** `openSqliteWriter` + (`packages/ui-server/src/db/write-worker.ts`) starts a Bun worker thread that + opens its own connection at `synchronous=FULL`. `bun:sqlite` connections are + per thread, so the worker never shares the app's handle. The app connection + keeps every read; WAL lets those proceed while the worker commits. +- **Writers move as code, not as SQL replay.** Each writer registers a + procedure module on the worker: the journal store + (`packages/ui-server/src/brain/journal-store.ts`), the Activity store's writes + (`packages/ui-server/src/activity/store-writes.ts`) and the native lease store + (`packages/ui-server/src/brain/native-lease-store.ts`). A procedure is the + transaction body that used to run inline, with its reads inside the same + transaction. The journal's procedures run through the unchanged + `createJournalAccounting`, so capacity accounting is exactly what it was. +- **Submission order is commit order.** The worker runs one call at a time, in + arrival order, and every call is posted synchronously when it is made. The + journal additionally routes every call through its startup-recovery promise, + so nothing overtakes recovery. +- **FULL is enforced, not assumed.** The worker refuses a transaction whose + connection is below FULL just before it commits, which rolls it back, and + reports an autocommit statement that ran below FULL. Journal writes still set + FULL themselves. +- **App transactions and worker procedures take turns.** Two connections in + one process writing one WAL file misbehaved under fsync load. The first + native runs refused governed turns with `database is locked`: an app + `BEGIN IMMEDIATE` returned SQLITE_BUSY after about 37 ms despite a 5 s busy + timeout, while the worker was committing. Separately, a deferred transaction + that reads and then writes can lose its snapshot upgrade to a worker commit. + So every outermost transaction on the app connection waits until the worker + is between procedures, through a lock shared with the worker thread + (`takeTurns`). The worker likewise waits, and lets a waiting app + transaction go first. Each transaction keeps its own mode, so a read-only + deferred snapshot never takes the write lock. Making every app transaction + immediate was tried and rejected: it held the write lock through read-only + snapshots, and the inbox's foreign-writer convergence test failed. +- **Further database files join the same connection.** `openSqliteWriter(db, + { attach })` attaches each file to the worker connection and checks FULL on + all of them before every commit. That is how the worker owns the journal's own + `journal.db` (ruling A on + [#2789](https://github.com/schlessera/brain-kit/issues/2789)), while journal + procedures still reach `governance_events` in the UI database. +- **A turn waits for its Activity writes.** The recorder keeps a turn's writes + in call order and exposes `settled()`. The session slot awaits it before it + ends, and an autonomous turn awaits it before the transaction that releases + its lifetime and commits its terminal, so that terminal still commits inside + the transaction. +- **Refusals keep their class.** `JournalCapacityError`, + `JournalStorageUnavailable`, `JournalStatementRefused` and + `NativeBrainManagerError` are rebuilt on the event loop with their message and + fields, so `retain()` collection retries, `journal_capacity` and + `journal_unavailable` work as before. +- **Callers await what they need.** Journal methods that wrote synchronously + now return promises: `bindSession`, `bindModel`, `writeOperation` (which now + takes statements), `persistClaudeMemory`, `recordClaudeMemory`, + `reserveRevert`, `finishReserved`, `withRetainedRemoval`, + `withRetainedVersion` and `version().stream()`. Turn revert `start` and + storage removal `start`/`cancel` follow. `createTurnJournal` exposes `ready`, + which the app awaits before serving. + +## The exceptions + +- **A write inside the caller's transaction runs there.** The session catalog + writes title and unstreamed-usage spans inside its own transaction on the app + connection. The off-loop Activity store sees `db.inTransaction` and runs that + write synchronously on the caller's connection, so it stays atomic with the + caller's rows. That commit is the caller's, on the event loop, as before. +- **The in-memory database has no file** for a second connection. Its writer + runs the same procedures inline, in the same order, behind the same promises. +- **Rollup pricing in the worker** reads the same on-disk table and bundled + snapshot as the app's pricing instance and reloads the table whenever the + app's refresh replaces it. A refresh whose cache write failed leaves the + worker on the previous table until the next successful write. +- **Other app-connection writers stay on the loop**: governance events written + outside a journal transaction, the writer gate, turn budgets and scope, the + session catalog, inbox and settings. Each commits at FULL on the loop, and + while a worker procedure holds its turn, such a write also waits there for + the worker's commit to finish. Those writers are tracked separately + ([#2850](https://github.com/schlessera/brain-kit/issues/2850)). + +## Rejected + +- **Committing at `synchronous=NORMAL` and syncing the WAL afterwards off the + loop.** It would reach the same durability by the time the caller proceeds, + but it is a different durability level per commit, which the ruling excludes. +- **Replaying recorded SQL in the worker.** Journal transactions read what they + are about to write (capacity measurement, conflict detection, guards), so a + replay could commit a decision taken against a different database state. +- **Synchronous RPC from the loop into the worker (Atomics.wait) with only the + commit asynchronous.** Every statement would still block the loop, including + `BEGIN IMMEDIATE` and the journal's measurement under the same I/O pressure. + +## Testing note + +Bun 1.4.2's `expect(promise).rejects` and `.resolves` wait in a nested event +loop that does not deliver every worker or port message. An assertion on a +promise that needs a write-worker round trip posted during that wait never +settles. The same holds for a +`MessageChannel` port in the cases measured. Tests use `expectRejection` +(`packages/ui-server/tests/helpers/expect-rejection.ts`) or a plain `await` +instead; production code never waits that way. diff --git a/docs/decisions/voice-permission.md b/docs/decisions/voice-permission.md index 838b0e7685..63ae96558e 100644 --- a/docs/decisions/voice-permission.md +++ b/docs/decisions/voice-permission.md @@ -406,7 +406,7 @@ A microphone is a reflex surface with worse recognition than a keyboard. This costs nothing measurable: the server already refuses `always` for kind `command` requests (the block computing `remembers`, `dispatch.ts:438-444`, and the lookup computing -`remembered`, `ws/bridge.ts:320-340`), and 192 of 192 measured +`remembered`, `ws/bridge.ts:321-341`), and 192 of 192 measured approvals were kind `command`. ### When the announcement actually fires @@ -460,10 +460,10 @@ An approval given by voice must be as reviewable afterwards as one given by tapping a card. Most of that already exists: every decision is written as an append-only `approval_decision` event carrying the principal, the decision and the request kind, and it patches the span -(`onApprovalDecision`, `packages/ui-server/src/activity/recorder.ts:635-650`), +(`onApprovalDecision`, `packages/ui-server/src/activity/recorder.ts:651-666`), fed from the bridge's `recorded()` wrapper -(`const recorded`, `packages/ui-server/src/ws/bridge.ts:362-377`). +(`const recorded`, `packages/ui-server/src/ws/bridge.ts:363-378`). One thing is missing and is a follow-up: **the event does not record the modality.** A denial decided by a phrase a microphone heard and one decided by a @@ -590,7 +590,7 @@ matters because a reader cannot otherwise tell a live hazard from a fixed one. **Closed, by #141 (`42a4d86`), which closed #124.** A turn now declares `StartTurnRequest.enforceAllowedTools` -(`enforceAllowedTools?: boolean`, `packages/ui-sdk/src/server/backend.ts:586`) +(`enforceAllowedTools?: boolean`, `packages/ui-sdk/src/server/backend.ts:589`) — the declaration this record asked for, by that name — and under it: - The input-rewrite hooks no longer grant. `createAgentHook` and `createRtkHook` @@ -608,7 +608,7 @@ matters because a reader cannot otherwise tell a live hazard from a fixed one. purpose, so a grant that exists and is deliberately not applied can be logged — but refuses to answer from it or add to it for a tool the enforced allowlist left out (the lookup computing `remembered`, - `packages/ui-server/src/ws/bridge.ts:320-340`, + `packages/ui-server/src/ws/bridge.ts:321-341`, and the block computing `remembers`, `packages/ui-server/src/ws/dispatch.ts:438-444`). The evaluation happens diff --git a/docs/hosting/paid-cli-provider-qualification.md b/docs/hosting/paid-cli-provider-qualification.md index 1c73f1848c..cc4b125f89 100644 --- a/docs/hosting/paid-cli-provider-qualification.md +++ b/docs/hosting/paid-cli-provider-qualification.md @@ -135,7 +135,7 @@ usage stay unknown; never infer tokens from payload bytes or vector length. The current shared inference/egress counter measures application or opaque transport bytes. These units stay separate from provider token, character, image or time units. Activity's current token pricing uses input/output/cache -classes (`priceUsage`, `packages/ui-server/src/activity/store.ts:1058-1088`); +classes (`priceUsage`, `packages/ui-server/src/activity/store.ts:1075-1105`); it cannot silently price image modalities or enterprise billable characters as ordinary tokens. Image model-table costs remain estimates; unknown model/rate combinations remain unknown. No live invoice or price validation is implied by diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index 00f545a890..9059c5e4be 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3854,6 +3854,35 @@ change only after the save succeeds. Failed or interrupted finalization remains partial/unconfirmed. Other backends need not implement Claude native-memory persistence; this adds no agent history authority or new public request route. +The callback may return a promise (additive, #2814): the first-party server +commits its journal records on a write worker, off its event loop, and invokes +the save closure only after they commit. A backend awaits the returned value +before it persists the next note or ends the turn; a callback that returns +nothing behaves exactly as before. + +### Journal and Activity writes commit off the event loop (#2814) + +Ruling 1 on [#2789](https://github.com/schlessera/brain-kit/issues/2789) +moves the hosted server's journal, Activity and native-lease commits to a +write-worker thread at `synchronous=FULL`, and makes their callers +asynchronous. On the `TurnJournal` reachable through `WsHost.journal`: + +- `writeOperation` takes statements (`JournalStatement[]`) instead of a + callback, and resolves with each statement's change count once they commit + as one accounted write. +- `bindSession`, `persistClaudeMemory`, `recordClaudeMemory`, `reserveRevert`, + `finishReserved`, `withRetainedRemoval`, `withRetainedVersion` and + `version().stream()` return promises that settle after their commits. + `bindModel` still answers synchronously, from the active turn's binding. +- `ready` settles once startup recovery has committed. It rejects with the + refusal that `createTurnJournal` used to throw. + +`AwaitableActivityStore` is the Activity store as the hosted server holds it. +Its reads are the `ActivityStore` reads. Its writes may return promises that +settle after their commit. An `ActivityStore`, whose writes all commit before +they return, satisfies it. Refusal classes and transaction order are +unchanged. These shapes change at a pre-1.0 minor under that ruling. + ### Late Activity session association `ActivityStore.patchSpan` accepts an optional `sessionId` to bind an unnamed diff --git a/docs/investigations/live-conversation-investigation.md b/docs/investigations/live-conversation-investigation.md index afb1f4086d..256bd41998 100644 --- a/docs/investigations/live-conversation-investigation.md +++ b/docs/investigations/live-conversation-investigation.md @@ -245,7 +245,7 @@ Backend reuse is structurally feasible for **both** adapters. `startTurn` is already asynchronous and accepts a host AbortSignal. A busy same-session turn is queued; new committed details use `followUp` only when the backend advertises it, otherwise remain queued/reviewable -(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:691-734`). +(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:694-737`). Continued voice interaction never requires unsafe concurrent `startTurn`s. This is source evidence and design mapping, not a real provider/backend trial. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index 1bcf9c55b3..2121388787 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -43,7 +43,7 @@ session. Three consequences: 1. **Background work cannot ask.** Cron runs (`sync`, `validate`, `maintain`, module jobs) have no human attached. `bridge.requestPermission` parks a promise that nobody will - resolve (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:310-404`), so autonomous work is confined to + resolve (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:311-405`), so autonomous work is confined to whatever is pre-approved, and anything requiring judgment is not attempted. 2. **Inbound material has no path.** A forwarded email, a shared link, a captured note has nowhere to land that the agent will act on later. The PWA share target stages into @@ -150,7 +150,7 @@ Stated before the requirements because five of them derive from it. thread_id, item_id, seq)` **carries `thread_id` explicitly** — activity stores its scope in the change row for exactly this reason ((`CREATE TABLE IF NOT EXISTS activity_changes (`, `packages/ui-server/migrations/007_activity.sql:58-66`), - (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:877-906`)), and deriving scope through a join on a + (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:894-923`)), and deriving scope through a join on a mutable item is weaker and breaks tombstones. `change_id` is the global cursor, `seq` is per-thread order. Append-only checkpoints, unique resolution records, scheduler heartbeats and budget reservations also need explicit durable storage. The four @@ -221,12 +221,12 @@ Stated before the requirements because five of them derive from it. ### Execution -- R18. **The drain ticks in-server.** (`// manual triggers (triggerJob)`, `packages/ui-server/src/cron/scheduler.ts:10-13`) owns only manual triggers and - history, but (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:55-194`) already runs an interval tick with a boot sweep — +- R18. **The drain ticks in-server.** (`// manual triggers (triggerJob)`, `packages/ui-server/src/cron/scheduler.ts:11-14`) owns only manual triggers and + history, but (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:57-196`) already runs an interval tick with a boot sweep — that is the shape to copy, including its `close()` lifecycle. - R19. **The cron backstop has independent authorization before the general guard.** Mount it on the existing listener before - (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:868`). + (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:877`). Authorize a boot-minted ephemeral token, rotated each boot and stored in a 0600 runtime file, with the actual socket address as an additional check. Proxy headers cannot authorize it. The poke succeeds in every auth mode @@ -253,17 +253,17 @@ Stated before the requirements because five of them derive from it. `persistSession: false`; pi provides `SessionManager.inMemory`. Drive this mode with a synthetic checkpoint bridge and server-selected authority; the recorder accepts autonomous origin and keeps runtime identity separate from an interactive session - (`export function createTurnRecorder(`, `packages/ui-server/src/activity/recorder.ts:112-190`). + (`export function createTurnRecorder(`, `packages/ui-server/src/activity/recorder.ts:115-193`). - R25. **Autonomous work gets its own pool** (`MAX_AUTONOMOUS_RUNS`, default 2) — but a second counter alone does not deliver "interactive always wins". The host cap applies only when - starting WS sessions (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1295-1307`), and an autonomous + starting WS sessions (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1298-1310`), and an autonomous turn can hold a path write lock while an interactive turn waits or is denied at 30 seconds (`export function createTurnLockBinding(`, `packages/ui-backend-claude/src/turn-lock.ts:27-119`). Required: an admission controller with reserved interactive capacity and hybrid yield at an explicit denial-risk threshold (~20s of the 30s lock budget). Below it nothing yields; above it the holder checkpoints/unwinds/releases. Test both edges and independent paths. - R26. **Abort-and-redo needs a checkpoint primitive.** `requestPermission` parks a bare - promise (`return new Promise((resolve) => {`, `packages/ui-server/src/ws/bridge.ts:354-412`) — while blocked on it the model cannot write anything, so + promise (`return new Promise((resolve) => {`, `packages/ui-server/src/ws/bridge.ts:355-413`) — while blocked on it the model cannot write anything, so "writes its findings, then aborts" has nowhere to run. The autonomous bridge must, in one server-side step: capture the checkpoint, create the Action and block the item, unwind without a live approval promise, preserving the tested timeout path's abort-then-drain order (`abortController.abort();`, `packages/ui-server/src/ws/run-session.ts:393-399`). Aborting does **not** undo completed tool side @@ -343,7 +343,7 @@ Stated before the requirements because five of them derive from it. policy proposals, and discuss-session digest construction when those v2 features ship. Revision 1's "T2 is the entire bill" was false. - R41. **Budgets are enforced by reservation at claim time, not by summing history.** - `rollupRun` runs in `finish()` (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:705`), so cost exists only after a + `rollupRun` runs in `finish()` (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:723`), so cost exists only after a run ends: two runs can both start under the cap and finish over it, and unknown effective costs are excluded from the sum (`export function sumEffectiveCost(`, `packages/ui-server/src/activity/store.ts:179-194`) so the query **fails open**. Required: transactional reservations on claim, in-flight reservations counted, settlement at diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 68e20d17a2..f71ee2913d 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -28,7 +28,7 @@ than last. ## Problem Frame Collaboration is synchronous-only: background work cannot ask (`requestPermission` parks a -promise nobody resolves — (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:310-404`)), inbound material +promise nobody resolves — (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:311-405`)), inbound material has no path that survives until the user is present, and no decision accumulates into standing authority. See origin for the full frame. @@ -125,16 +125,16 @@ system gate. It is not a prerequisite to rebuild or a new independent package. ### Relevant Code and Patterns - **Cursors and transactional snapshots:** (`CREATE TABLE IF NOT EXISTS activity_changes (`, `packages/ui-server/migrations/007_activity.sql:58-66`) - and (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:877-906`). Reuse the algorithm; the + and (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:894-923`). Reuse the algorithm; the activity stream is bound to activity store methods and frame types. -- **Runtime lifecycle:** (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:55-194`). Cron's scheduler +- **Runtime lifecycle:** (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:57-196`). Cron's scheduler owns manual triggers/history; it does not tick the inbox. - **Two connections:** (`export function createUiDb(`, `packages/ui-server/src/db/client.ts:27-44`) sets WAL, foreign keys and a 5-second busy timeout. Claims are immediate transactions. -- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:868`) follows public routes; +- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:877`) follows public routes; (`export function authGuard(`, `packages/ui-server/src/middleware/auth.ts:189-249`) binds principals in each auth mode. An internal poke needs independent token authorization before this guard. -- **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:310-404`). Timeout unwind is +- **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:311-405`). Timeout unwind is (`abortController.abort();`, `packages/ui-server/src/ws/run-session.ts:393-399`). Durable escalation must checkpoint before unwinding; the existing ordinary bridge does not do that. - **Tool enforcement:** (`const enforcementHook: HookCallback`, `packages/ui-backend-claude/src/permission-hooks.ts:200-229`) closes measured bypasses. @@ -146,7 +146,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. - **Pi resources and extension gate:** (`export function createSessionResources(`, `packages/ui-backend-pi/src/session-resources.ts:29-152`) and (`export function createPermissionGate(`, `packages/ui-backend-pi/src/permission-gate.ts:96-125`). Built-ins are disabled, but ambient resources/extensions and in-process execution still need containment. -- **Cost timing:** (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:705`) settles after execution; +- **Cost timing:** (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:723`) settles after execution; (`export function sumEffectiveCost(`, `packages/ui-server/src/activity/store.ts:179-194`) explicitly counts unpriced runs. Reservations must cover in-flight work, not only this retrospective sum. - **Notifications:** (`CREATE TABLE IF NOT EXISTS notification_intents (`, `packages/ui-server/migrations/007_activity.sql:99-114`), @@ -156,7 +156,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. - **Share provenance/limits:** (`const result = await stageShareAt(`, `packages/ui-server/src/inbox/intake.ts:106`) assigns the source in server code. (`export const SHARE_MAX_TEXT_BYTES =`, `packages/ui-sdk/src/protocol.ts:2348`) bounds text, not binary uploads; extracted T1 context needs its own byte/token bound. -- **Interactive locks:** (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1295-1307`) gates WS starts; +- **Interactive locks:** (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1298-1310`) gates WS starts; (`export function createTurnLockBinding(`, `packages/ui-backend-claude/src/turn-lock.ts:27-119`) owns tool locks. Shared-target priority and cooperative yield belong to the keyed lock and backend lifecycle. @@ -632,7 +632,7 @@ write-once outcome discipline, cursor emission); migration comment style of `007 - Test: `packages/ui-server/tests/inbox-runtime.test.ts` **Approach:** -- 60s interval with a `close()` lifecycle, modeled on (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:55-194`), including +- 60s interval with a `close()` lifecycle, modeled on (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:57-196`), including its boot sweep - **Gate first**: a SQL count of ready items; zero means return immediately — no model call and **no Activity run** (AE2) @@ -717,7 +717,7 @@ recovered tick after killing the interval model-bearing operation; folded compaction is included in its existing call. **Patterns to follow:** the frozen-at-first-computation discipline of effective cost -(`function rollupRunInTx(runId: string) {`, `packages/ui-server/src/activity/store.ts:518-620`); env descriptor array for the new configuration values +(`function rollupRunInTx(runId: string) {`, `packages/ui-server/src/activity/store.ts:535-637`); env descriptor array for the new configuration values **Test scenarios:** - Covers AE11: two claims that would each fit but jointly exceed the cap — the second is @@ -1193,7 +1193,7 @@ and a server-selected tool policy. - Modify: `packages/ui-backend-claude/src/backend.ts` (`persistSession: false`, synthetic bridge) - Modify: `packages/ui-server/src/activity/recorder.ts` (server-selected - origin defaults to `"session"` at (`export function createTurnRecorder(`, `packages/ui-server/src/activity/recorder.ts:112-190`)) + origin defaults to `"session"` at (`export function createTurnRecorder(`, `packages/ui-server/src/activity/recorder.ts:115-193`)) - Test: `packages/ui-backend-claude/tests/autonomous-turn.test.ts` **Approach:** an autonomous request carrying persistence, tool policy, origin, and prompt @@ -1274,7 +1274,7 @@ full-v1 enablement; predicates and schema tests cannot substitute for it. **Approach:** **hybrid — reserve capacity normally, yield only at denial risk.** `MAX_AUTONOMOUS_RUNS` (default 2) is necessary but not sufficient: the host cap applies only at -WS session start (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1295-1307`) and an autonomous turn can hold a path write lock +WS session start (`const cap = host.maxConcurrentSessions();`, `packages/ui-server/src/ws/run-session.ts:1298-1310`) and an autonomous turn can hold a path write lock while an interactive turn waits or is denied at 30s (`export function createTurnLockBinding(`, `packages/ui-backend-claude/src/turn-lock.ts:27-119`). diff --git a/docs/plans/navigation-recovery.md b/docs/plans/navigation-recovery.md index d4ac1ce70c..570189bd09 100644 --- a/docs/plans/navigation-recovery.md +++ b/docs/plans/navigation-recovery.md @@ -391,15 +391,15 @@ implementation. Any PR moving these lines owns their citation repairs. - Session summary shape: (`export interface ChatSession {`, `packages/ui-sdk/src/protocol.ts:1568-1584`). -- Run identity: (`const runId = turn.turnId;`, `packages/ui-server/src/activity/recorder.ts:141-142`). +- Run identity: (`const runId = turn.turnId;`, `packages/ui-server/src/activity/recorder.ts:144-145`). - Outcome vocabulary: (`export type ActivitySpanOutcome =`, `packages/ui-sdk/src/protocol.ts:2930-2953`). - Activity resolution: (`.get("/activity/runs", (c) => {`, `packages/ui-server/src/routes/activity.ts:143-281`). -- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:868`). +- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:877`). -- Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:1062-1107`). +- Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:1065-1110`). - Approval target: (`function mutateLastAssistant(`, `packages/ui-react/src/stores/chat-state.ts:1224-1254`). diff --git a/docs/plans/storage-accounting.md b/docs/plans/storage-accounting.md index bad8078d32..619059e630 100644 --- a/docs/plans/storage-accounting.md +++ b/docs/plans/storage-accounting.md @@ -67,7 +67,7 @@ Source anchors, checked against the reviewed commit: The completed #94 and #398 provide existing stats/maintenance foundations. - C: snapshot fields (`snapshotOf`, `packages/core/src/lib/stats-history.ts:132-172`), thinning (`thin`, `packages/core/src/lib/stats-history.ts:232-244`). -- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:292-328`), +- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:293-329`), WAL store (`createUiDb`, `packages/ui-server/src/db/client.ts:27-44`), and private locations (`statePaths`, `packages/ui-server/src/brain/operational-state.ts:108-114`). - E: live/archive paths (`transcriptPaths`, `packages/ui-backend-claude/src/transcript-archive.ts:29-38`). @@ -85,7 +85,7 @@ Source anchors, checked against the reviewed commit: decisions distinguish the independent configured limits and mounts. Delivered #1703 adds scoped turn-temp usage/purge and promotion protection; those measurements do not include the separate runtime-state tmpfs. -- J: private object store (`createJournalObjects`, `packages/ui-server/src/brain/journal-objects.ts:34-49`) +- J: private object store (`createJournalObjects`, `packages/ui-server/src/brain/journal-objects.ts:35-50`) and D's path resolution. Existing journal/revert accounting owns reservations. - K: consistent export (`exportInboxSnapshot`, `packages/ui-server/src/inbox/snapshot.ts:289-306`) and [operational recovery](../inbox-recovery.md); delivered #686 and its diff --git a/docs/plans/storage-lifecycle.md b/docs/plans/storage-lifecycle.md index 9de0e2c351..2172678106 100644 --- a/docs/plans/storage-lifecycle.md +++ b/docs/plans/storage-lifecycle.md @@ -84,11 +84,11 @@ Source foundations for the matrix: - Claude publishes and verifies cold copies before deleting the live source (`export async function archiveIdleSessions`, `packages/ui-backend-claude/src/transcript-archive.ts:188-242`). - Journal collection requires no refs/reservations and only expires ended - turns (`function collectObjects`, `packages/ui-server/src/brain/turn-journal.ts:175-201`). + turns (`function collectObjects`, `packages/ui-server/src/brain/turn-journal.ts:215-235`). Retention is applied at admission - (`begin(turnId: string`, `packages/ui-server/src/brain/turn-journal.ts:347-380`). + (`begin(turnId: string`, `packages/ui-server/src/brain/turn-journal.ts:332-363`). Object copies reserve temporary bytes before writing - (`async putFileAsync`, `packages/ui-server/src/brain/journal-objects.ts:92-147`). + (`async putFileAsync`, `packages/ui-server/src/brain/journal-objects.ts:95-150`). - Turn-temp purge has its own scope and measured result (`export function purgeTurnTemp`, `packages/ui-sdk/src/server/worker-turn-temp.ts:450-533`), and tool validation/recording diff --git a/docs/process/testing.md b/docs/process/testing.md index a4d0473c5b..ce94183e92 100644 --- a/docs/process/testing.md +++ b/docs/process/testing.md @@ -246,6 +246,13 @@ Follow [the wait classification and clock rules](test-waits.md): distinguish actual awaited delays from failure caps and fixture holds, preserve negative temporal guarantees, and measure native exceptions separately. +Do not wrap a promise that waits on the SQLite write worker in +`expect(...).rejects` or `.resolves`: Bun 1.4.2 waits for those in a nested +event loop that does not deliver the worker's reply, so the test times out. +Await it and assert on the result, or use `expectRejection` +(`packages/ui-server/tests/helpers/expect-rejection.ts`) +([why](../decisions/sqlite-write-worker.md#testing-note)). + ## Tests that cannot fail A green test is evidence only if it could have been red. Each of these shapes diff --git a/packages/ui-backend-claude/src/worker-spawn.ts b/packages/ui-backend-claude/src/worker-spawn.ts index e480d7e08a..7fac6a6d89 100644 --- a/packages/ui-backend-claude/src/worker-spawn.ts +++ b/packages/ui-backend-claude/src/worker-spawn.ts @@ -232,9 +232,9 @@ export function createWorkerSpawn(brainPath: string, config: ExecWrapperConfig, if (promoter && recovered) await state.prepareCopyOut?.(handle.root, text => promoter.prepareRecovery(text, signal), signal); if (recovered) signal.throwIfAborted(); persistenceAttempted = true; - state.persist(handle.root, promoter?.rewrite, persistMemory); + await state.persist(handle.root, promoter?.rewrite, persistMemory); } catch (error) { - if (!persistenceAttempted) persistMemory?.(null, () => {}); + if (!persistenceAttempted) await persistMemory?.(null, () => {}); // The exhaustion message must not claim what the copy-out refused. if (failure?.errorClass === FIRST_PARTY_FAILURE_CLASSES.worker_state_exceeded) failure = exceededFailure(state.bytes, error); else if (!failure) failure = workerStateSaveFailure(error); diff --git a/packages/ui-backend-claude/src/worker-state.ts b/packages/ui-backend-claude/src/worker-state.ts index 88701a1084..cf0049214f 100644 --- a/packages/ui-backend-claude/src/worker-state.ts +++ b/packages/ui-backend-claude/src/worker-state.ts @@ -120,7 +120,7 @@ export interface ClaudeWorkerState { * Copies validated results out of the private tmpfs, after the worker's * namespace is gone; `rewrite` promotes newly appended answer links for display. */ - persist(root: string, rewrite?: (text: string) => string, memory?: import("@schlessera/brain-ui-sdk/server").StartTurnRequest["persistClaudeMemory"]): void; + persist(root: string, rewrite?: (text: string) => string, memory?: import("@schlessera/brain-ui-sdk/server").StartTurnRequest["persistClaudeMemory"]): void | Promise; /** Prepare only newly appended answer links before synchronous publication. */ prepareCopyOut?(root: string, prepare: (text: string) => Promise, signal?: AbortSignal): Promise; /** Releases the admission reservation; idempotent, called on every exit path. */ @@ -207,7 +207,7 @@ function plan(brainPath: string, env: Record, if (settingsSize + transcriptSize > allowance) throw new SessionTooLargeError(settingsSize + transcriptSize, allowance); const initial = new Map(), initialLength = new Map(); /** Write one changed file back, refusing to overwrite a saved file that changed meanwhile. */ - const write = (name: string, bytes: Buffer, what: "transcript" | "memory", memory?: import("@schlessera/brain-ui-sdk/server").StartTurnRequest["persistClaudeMemory"]): void => { + const write = (name: string, bytes: Buffer, what: "transcript" | "memory", memory?: import("@schlessera/brain-ui-sdk/server").StartTurnRequest["persistClaudeMemory"]): void | Promise => { const target = join(saved, name); directory(dirname(target)); const before = existsSync(target) ? regular(target, allowance) : null; const present = before ? hash(before) : undefined; @@ -218,8 +218,9 @@ function plan(brainPath: string, env: Record, try { writeFileSync(temp, bytes, { flag: "wx", mode: 0o600 }); directory(dirname(target)); renameSync(temp, target); } finally { rmSync(temp, { force: true }); } }; - if (what === "memory" && memory) memory({ directory: dirname(target), note: name.slice(name.lastIndexOf("/") + 1), before, beforeMode: before ? lstatSync(target).mode & 0o7777 : 0o600, after: bytes }, commit); - else commit(); + // The host may retain the note off its event loop: its record commits before `commit` runs. + if (what === "memory" && memory) return memory({ directory: dirname(target), note: name.slice(name.lastIndexOf("/") + 1), before, beforeMode: before ? lstatSync(target).mode & 0o7777 : 0o600, after: bytes }, commit); + commit(); }; let observation: StateRoomObservation | undefined; return { @@ -308,18 +309,30 @@ function plan(brainPath: string, env: Record, if (initial.get(name) === hash(bytes)) continue; write(name, bytes, "transcript"); } - } catch (error) { memory?.(null, () => {}); throw error; } + } catch (error) { + const reported = memory?.(null, () => {}); + if (reported instanceof Promise) return reported.then(() => { throw error; }, () => { throw error; }); + throw error; + } // Every session of the brain shares its memory. A note another turn // changed meanwhile is refused alone and named; the other notes still - // persist, and nothing is ever deleted. + // persist, and nothing is ever deleted. Notes persist one at a time; a + // host that records them asynchronously is awaited between notes. const refused: string[] = []; - for (const [name, bytes] of notes) { - try { write(name, bytes, "memory", memory); } catch (error) { - if (!(error instanceof WorkerHostError && error.message.includes("changed concurrently"))) throw error; - refused.push(name.slice(project.length + 1)); + const refuse = (error: unknown, name: string) => { + if (!(error instanceof WorkerHostError && error.message.includes("changed concurrently"))) throw error; + refused.push(name.slice(project.length + 1)); + }; + const persistNotes = (from: number): void | Promise => { + for (let index = from; index < notes.length; index++) { + const [name, bytes] = notes[index]!; + let saved: void | Promise; + try { saved = write(name, bytes, "memory", memory); } catch (error) { refuse(error, name); continue; } + if (saved instanceof Promise) return saved.then(() => {}, error => refuse(error, name)).then(() => persistNotes(index + 1)); } - } - if (refused.length) throw new WorkerHostError(`saved memory changed concurrently; refusing overwrite of ${refused.join(", ")}`); + if (refused.length) throw new WorkerHostError(`saved memory changed concurrently; refusing overwrite of ${refused.join(", ")}`); + }; + return persistNotes(0); }, }; } diff --git a/packages/ui-sdk/src/server/backend.ts b/packages/ui-sdk/src/server/backend.ts index f254b0848c..1f34f2931d 100644 --- a/packages/ui-sdk/src/server/backend.ts +++ b/packages/ui-sdk/src/server/backend.ts @@ -516,8 +516,11 @@ export interface StartTurnRequest { workerStartupView?: import("./worker-startup-view.js").WorkerStartupView; /** @internal Parent-owned lifecycle control; never serialized into a worker. */ workerStarted?: (worker: Pick) => void; - /** @internal Host-only observation of validated native memory copy-out, never a worker/client destination. */ - persistClaudeMemory?: (change: { directory: string; note: string; before: Uint8Array | null; beforeMode: number; after: Uint8Array } | null, save: () => void) => void; + /** + * @internal Host-only observation of validated native memory copy-out, never a worker/client destination. + * The host may record the note asynchronously; a backend awaits the result before the next note or exit. + */ + persistClaudeMemory?: (change: { directory: string; note: string; before: Uint8Array | null; beforeMode: number; after: Uint8Array } | null, save: () => void) => void | Promise; /** @internal Host-resolved filesystem view, never a client or model grant. */ workerSandbox?: import("./worker-sandbox.js").WorkerSandbox; /** diff --git a/packages/ui-server/src/activity/notify.ts b/packages/ui-server/src/activity/notify.ts index 232f31ad69..ae69230da8 100644 --- a/packages/ui-server/src/activity/notify.ts +++ b/packages/ui-server/src/activity/notify.ts @@ -25,7 +25,7 @@ import { isFailureOutcome } from "@schlessera/brain-ui-sdk/protocol"; import { getSetting, setSetting } from "../db/settings.js"; import { LABEL_RUN_NAME } from "../labels/labeller.js"; -import type { ActivityStore, SpanRow } from "./store.js"; +import type { AwaitableActivityStore, SpanRow } from "./store.js"; export type IntentKind = "failure" | "completion" | "stuck"; @@ -44,7 +44,7 @@ export interface NotificationIntent { export interface ActivityNotifierDeps { db: Database; - store: ActivityStore; + store: AwaitableActivityStore; /** Was anyone watching this run/session when the event was recorded? */ isWatched: (scope: { runId?: string; sessionId?: string }) => boolean; log?: Logger; diff --git a/packages/ui-server/src/activity/query.ts b/packages/ui-server/src/activity/query.ts index 26b96d7077..0b11e7e1d0 100644 --- a/packages/ui-server/src/activity/query.ts +++ b/packages/ui-server/src/activity/query.ts @@ -14,11 +14,11 @@ import type { ActivityQuery, ActivityQueryResult } from "@schlessera/brain-ui-sd import { queryRoadblockReport } from "../governance/roadblock-report.js"; import { latestActivityDigest } from "./digest.js"; import type { ActivityNotifier } from "./notify.js"; -import { rowToRunRollup, summarizeRollups, type ActivityStore, type SpanRow } from "./store.js"; +import { rowToRunRollup, summarizeRollups, type AwaitableActivityStore, type SpanRow } from "./store.js"; export function runActivityQuery( db: Database, - store: ActivityStore, + store: AwaitableActivityStore, query: ActivityQuery, notifier?: ActivityNotifier ): ActivityQueryResult { diff --git a/packages/ui-server/src/activity/recorder.ts b/packages/ui-server/src/activity/recorder.ts index 124a83ee1e..ca24eb9c3d 100644 --- a/packages/ui-server/src/activity/recorder.ts +++ b/packages/ui-server/src/activity/recorder.ts @@ -44,10 +44,11 @@ import { redactCredentials, type RuntimeStatus } from "./runtime-status.js"; import { inputTaintSource, outputTaintSource, type IngestionSource } from "./ingestion-taint.js"; import { recordFirstTaint } from "../governance/session-taint.js"; import { MAX_PROVIDER_UNAVAILABLE_PER_TURN, providerUnavailablePayloads } from "../governance/provider-unavailable.js"; -import type { ActivityStore, SpanOutcome, SpanUsage } from "./store.js"; +import type { AwaitableActivityStore, SpanOutcome, SpanUsage } from "./store.js"; +import { orderedWrites } from "./store-writes.js"; export interface TurnRecorderDeps { - store: ActivityStore; + store: AwaitableActivityStore; /** Where the runtime a turn reported, and its auth failures, are kept for /api/status. */ runtime?: Pick & Partial>; /** Called after any committed write so the live stream can pump. */ @@ -68,6 +69,8 @@ export interface TurnRecorderDeps { } export interface TurnRecorder { + /** Settles once every write this recorder has submitted has committed or failed. */ + settled(): Promise; setMainModel(model: string): void; setWorkerProfile(profile: GovernanceProfile): void; /** The run id (one run per turn). */ @@ -137,7 +140,7 @@ export function createTurnRecorder( profile?: GovernanceProfile | null; } ): TurnRecorder { - const { store, onWrite, log, runtime } = deps; + const { onWrite: wrote, log, runtime } = deps; const runId = turn.turnId; const rootSpanId = `${runId}:turn`; let sessionId = turn.sessionId ?? undefined; @@ -191,17 +194,25 @@ export function createTurnRecorder( call.endedAt = Date.now(); }; + const dropped = (err: unknown) => { + log?.emit({ + severityText: "WARN", + body: "activity recording failed; dropping", + attributes: { error: err instanceof Error ? err.message : String(err) }, + }); + }; const guard = (fn: () => void) => { try { fn(); } catch (err) { - log?.emit({ - severityText: "WARN", - body: "activity recording failed; dropping", - attributes: { error: err instanceof Error ? err.message : String(err) }, - }); + dropped(err); } }; + // The hosted store commits off the event loop (#2814): this turn's writes + // stay in call order, and the live stream pumps only after they commit. + const writes = orderedWrites(deps.store, dropped); + const store = writes.store; + const onWrite = wrote ? () => writes.after(wrote) : undefined; function ensureRoot() { if (rootStarted) return; @@ -244,6 +255,7 @@ export function createTurnRecorder( } return { + settled: () => writes.settled(), setMainModel(model) { if (!finished) mainModel = model; }, setWorkerProfile(profile) { turn.profile = profile; }, runId, @@ -395,13 +407,17 @@ export function createTurnRecorder( if (prefixRecorded || turn.origin === "autonomous" || typeof event.resumed !== "boolean" || !validPrefix(event.prefix)) break; ensureRoot(); prefixRecorded = true; - const prior = sessionId ? store.previousSessionRoot?.(rootSpanId, sessionId) : null; - store.patchSpan(rootSpanId, { attrs: { - "brain.cache.prefix": { tools: event.prefix.tools, system: event.prefix.system, model: event.prefix.model, - thinking: event.prefix.thinking, gitStatusIncluded: event.prefix.gitStatusIncluded, coverage: event.prefix.coverage }, - "brain.cache.resumed": event.resumed, - "brain.cache.changed": event.resumed ? changedPrefix(event.prefix, prior?.attrs["brain.cache.prefix"] as typeof event.prefix | undefined) : "unknown", - } }); + // The prior root is read in this turn's write order, once its own root has committed. + const session = sessionId; + writes.after(() => { + const prior = session ? deps.store.previousSessionRoot?.(rootSpanId, session) : null; + return deps.store.patchSpan(rootSpanId, { attrs: { + "brain.cache.prefix": { tools: event.prefix.tools, system: event.prefix.system, model: event.prefix.model, + thinking: event.prefix.thinking, gitStatusIncluded: event.prefix.gitStatusIncluded, coverage: event.prefix.coverage }, + "brain.cache.resumed": event.resumed, + "brain.cache.changed": event.resumed ? changedPrefix(event.prefix, prior?.attrs["brain.cache.prefix"] as typeof event.prefix | undefined) : "unknown", + } }); + }); onWrite?.(); break; } @@ -692,8 +708,10 @@ export function createTurnRecorder( "brain.usage.completed_responses": true, "brain.usage.at_tool_boundary": atToolBoundary, ...(reportedCost != null ? { "brain.usage.reported_cost_usd": reportedCost } : {}) }; } + const endedAt = Date.now(); store.endSpan(rootSpanId, { outcome, + endedAt, usage: resultUsage, attrs: { ...resultAttrs, ...(budgetStop ? {"brain.failure.class":budgetStop.errorClass} : {}) }, ...((outcome === "error" || outcome === "timeout") && (reason ?? budgetStop?.reason ?? resultFailureReason) !== undefined @@ -704,10 +722,7 @@ export function createTurnRecorder( store.cascadeClose(runId, "cancelled", `turn ${outcome}`); store.rollupRun(runId); // Kept now, not only when the status is next read (#254). - if (outcome === "success" && observedBilling === "subscription") { - const endedAt = store.getSpan(rootSpanId)?.endedAt; - if (typeof endedAt === "number") runtime?.subscriptionProven?.(endedAt); - } + if (outcome === "success" && observedBilling === "subscription") runtime?.subscriptionProven?.(endedAt); onWrite?.(); }); // Separately guarded: a classification or storage failure loses the diff --git a/packages/ui-server/src/activity/runtime.ts b/packages/ui-server/src/activity/runtime.ts index 355531d37a..43479c4c3d 100644 --- a/packages/ui-server/src/activity/runtime.ts +++ b/packages/ui-server/src/activity/runtime.ts @@ -13,7 +13,9 @@ import type { Database } from "bun:sqlite"; import type { Logger } from "@opentelemetry/api-logs"; import type { ActivityQuery, ActivityQueryResult } from "@schlessera/brain-ui-sdk/server"; -import { createActivityStore, type ActivityStore, type RollupPricing } from "./store.js"; +import { createActivityStore, type AwaitableActivityStore, type RollupPricing } from "./store.js"; +import { afterWrite, createOffloopActivityStore, observeWrite, type ActivityPricingConfig } from "./store-writes.js"; +import type { SqliteWriter } from "../db/write-worker.js"; import { createActivityStream, type ActivityStream } from "./stream.js"; import { createActivityNotifier, type ActivityNotifier } from "./notify.js"; import { createPushSender, type PushSender } from "./push-sender.js"; @@ -37,7 +39,7 @@ const ACTIVITY_HARD_CEILING_MS = 90 * 24 * 60 * 60 * 1000; const INTENT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; export interface ActivityRuntime { - store: ActivityStore; + store: AwaitableActivityStore; stream: ActivityStream; notifier: ActivityNotifier; pushSender: PushSender; @@ -62,10 +64,19 @@ export function createActivityRuntime( * env-derived default (the same instance shape, just not shared). */ pricing?: RollupPricing; + /** + * The app's write worker. Present, every standalone Activity write + * commits there, off the event loop (#2814), pricing its rollups from + * `pricingConfig`'s table. + */ + writer?: SqliteWriter; + pricingConfig?: ActivityPricingConfig; } ): ActivityRuntime { const { log } = deps; - const store = createActivityStore(db, deps.pricing ? { pricing: deps.pricing } : {}); + const store: AwaitableActivityStore = deps.writer + ? createOffloopActivityStore(db, deps.writer, { ...(deps.pricing ? { pricing: deps.pricing } : {}), ...(deps.pricingConfig ? { pricingConfig: deps.pricingConfig } : {}) }) + : createActivityStore(db, deps.pricing ? { pricing: deps.pricing } : {}); const stream = createActivityStream(store, log); // The notifier resumes its persisted cursor, including terminal changes @@ -80,36 +91,48 @@ export function createActivityRuntime( const actionNotices = createActionNotifier(db); // Boot sweep: close this server's orphans from a previous life (interrupted). - try { - const orphans = store.sweepOwnOrphans(); - if (orphans > 0) { - log.emit({ - severityText: "WARN", - body: "closed orphaned activity spans from a previous process", - attributes: { count: orphans }, - }); - } - } catch (err) { + const sweepFailed = (err: unknown) => { log.emit({ severityText: "WARN", body: "boot activity sweep failed", attributes: { error: err instanceof Error ? err.message : String(err) }, }); + }; + try { + afterWrite(store.sweepOwnOrphans(), orphans => { + if (orphans > 0) { + log.emit({ + severityText: "WARN", + body: "closed orphaned activity spans from a previous process", + attributes: { count: orphans }, + }); + } + }, sweepFailed); + } catch (err) { + sweepFailed(err); } let delivering = false; let lastPrune = 0; + const tickFailed = (err: unknown) => { + log.emit({ + severityText: "WARN", + body: "activity tick failed", + attributes: { error: err instanceof Error ? err.message : String(err) }, + }); + }; const tick = setInterval(() => { try { - const stale = store.sweepStale(ACTIVITY_STALE_AFTER_MS); - if (stale > 0) { - log.emit({ - severityText: "WARN", - body: "closed stale activity spans (writer went silent)", - attributes: { count: stale }, - }); - stream.pump(); - } + afterWrite(store.sweepStale(ACTIVITY_STALE_AFTER_MS), stale => { + if (stale > 0) { + log.emit({ + severityText: "WARN", + body: "closed stale activity spans (writer went silent)", + attributes: { count: stale }, + }); + stream.pump(); + } + }, tickFailed); notifier.tick(); // Action notices: join newly eligible decisions to their fixed window // and store any due client-local digest. Isolated so a failure here @@ -155,7 +178,7 @@ export function createActivityRuntime( if (typeof pricing?.ensureFresh === "function") { void pricing.ensureFresh().catch(() => {}); } - store.prune({ + observeWrite(store.prune({ // Full detail survives until the digest has covered it AND the // retention window has passed; the hard ceiling bounds growth even // if the digest job silently dies. The setting is read every pass @@ -163,15 +186,11 @@ export function createActivityRuntime( digestFloorAt: digestRetentionFloor(db), detailRetentionMs: getDetailRetentionDays(db, log) * 24 * 60 * 60 * 1000, hardCeilingMs: ACTIVITY_HARD_CEILING_MS, - }); + }), tickFailed); notifier.pruneAcknowledged(INTENT_RETENTION_MS); } } catch (err) { - log.emit({ - severityText: "WARN", - body: "activity tick failed", - attributes: { error: err instanceof Error ? err.message : String(err) }, - }); + tickFailed(err); } }, ACTIVITY_TICK_MS); // A Bun interval would otherwise keep a closed test app's process alive. diff --git a/packages/ui-server/src/activity/store-writes.ts b/packages/ui-server/src/activity/store-writes.ts new file mode 100644 index 0000000000..4c81924994 --- /dev/null +++ b/packages/ui-server/src/activity/store-writes.ts @@ -0,0 +1,139 @@ +/** + * Activity writes off the event loop (#2814). The write worker runs this + * module's store on its own connection at `synchronous=FULL`; the hosted + * server's store keeps reading on the app connection and submits every + * standalone write there, in call order. Each write keeps its own immediate + * transaction, so its atomicity and the order of writes are unchanged. + */ +import type { Database } from "bun:sqlite"; +import { statSync } from "node:fs"; + +import type { SqliteWriter, WriteProcedures } from "../db/write-worker.js"; +import { createModelPricing, pricingCachePath } from "../pricing/model-pricing.js"; +import { createActivityStore, MAX_EVENT_PAYLOAD_BYTES, type ActivityStore, type ActivityWriteMethod, type AwaitableActivityStore, type RollupPricing } from "./store.js"; + +export const url = import.meta.url; + +export const ACTIVITY_WRITE_METHODS = ["writeLifecycle", "startSpan", "endSpan", "patchSpan", "appendEvent", "heartbeat", + "cascadeClose", "sweepOwnOrphans", "sweepStale", "rollupRun", "prune"] as const satisfies readonly ActivityWriteMethod[]; + +/** The app's pricing configuration; the worker prices rollups from the same cache and snapshot. */ +export interface ActivityPricingConfig { brainPath: string; enabled?: boolean; ttlMs?: number } +export interface ActivityWriteOptions { writer: string; pricing?: ActivityPricingConfig } + +/** + * Worker side. Without a pricing configuration the store keeps its own + * env-derived default, as the cron wrapper's bare store does. With one, the + * worker reloads the shared on-disk table whenever the app's refresh + * replaces it, so a rollup prices from the table the app last published. + */ +export function setup(db: Database, options: ActivityWriteOptions): WriteProcedures { + const config = options.pricing; + let pricing: RollupPricing | undefined, identity = ""; + const current = (): RollupPricing => { + const cache = pricingCachePath(config!.brainPath); + let next = "absent"; + try { const info = statSync(cache); next = `${info.dev}:${info.ino}:${info.size}:${info.mtimeMs}`; } catch { /* The snapshot serves. */ } + if (!pricing || next !== identity) { pricing = createModelPricing({ ...config!, fetchImpl: (() => Promise.reject(new Error("The write worker never fetches pricing."))) as unknown as typeof fetch }); identity = next; } + return pricing; + }; + const store = createActivityStore(db, { writer: options.writer, ...(config ? { pricing: { resolve: (model, route) => current().resolve(model, route) } } : {}) }); + const procedures: WriteProcedures = {}; + for (const name of ACTIVITY_WRITE_METHODS) { + procedures[name] = ((...args: unknown[]) => { if (config) current(); return Reflect.apply(store[name] as (...values: unknown[]) => unknown, store, args); }) as WriteProcedures[string]; + } + return procedures; +} + +/** + * The payload as the store will cap and serialize it, in a form that crosses + * to the worker: what the store stores is unchanged. A payload over its cap is + * stored as a slice of its JSON either way, so its JSON is sent instead of a + * copy of the whole value. + */ +function transferablePayload(payload: unknown, cap?: number): unknown { + if (typeof payload === "string") return payload; + const json = JSON.stringify(payload ?? null); + return json.length > Math.min(cap ?? MAX_EVENT_PAYLOAD_BYTES, MAX_EVENT_PAYLOAD_BYTES) ? json : JSON.parse(json); +} + +/** + * The hosted server's store. Reads run on `db`. A standalone write is + * submitted to the write worker and resolves after its commit. A write the + * caller makes inside its own transaction on `db` runs there, synchronously, + * so it stays atomic with the caller's other rows. + */ +export function createOffloopActivityStore(db: Database, writer: SqliteWriter, options: { + writer?: string; pricing?: RollupPricing; pricingConfig?: ActivityPricingConfig; +} = {}): AwaitableActivityStore { + const identity = options.writer ?? `pid:${process.pid}:${Date.now()}`; + const local: ActivityStore = createActivityStore(db, { writer: identity, ...(options.pricing ? { pricing: options.pricing } : {}) }); + const channel = writer.channel({ url, setup }, { writer: identity, ...(options.pricingConfig ? { pricing: options.pricingConfig } : {}) }); + // Clock defaults are taken when the write is submitted, not when the + // worker reaches it, as a synchronous write took them. + const submitted: Partial unknown[]>> = { + startSpan: ([input]) => [{ ...(input as object), startedAt: (input as { startedAt?: number }).startedAt ?? Date.now() }], + endSpan: ([spanId, input]) => [spanId, { ...(input as object), endedAt: (input as { endedAt?: number }).endedAt ?? Date.now() }], + appendEvent: ([spanId, eventType, payload, ts, cap]) => [spanId, eventType, transferablePayload(payload, cap as number | undefined), ts ?? Date.now(), cap], + heartbeat: ([spanId, at]) => [spanId, at ?? Date.now()], + sweepStale: ([staleAfterMs, now]) => [staleAfterMs, now ?? Date.now()], + prune: ([prune]) => [{ ...(prune as object), now: (prune as { now?: number }).now ?? Date.now() }], + }; + const store = Object.create(local) as AwaitableActivityStore; + for (const name of ACTIVITY_WRITE_METHODS) { + Object.defineProperty(store, name, { enumerable: true, value: (...args: unknown[]) => db.inTransaction + ? Reflect.apply(local[name] as (...values: unknown[]) => unknown, local, args) + : channel.call(name, ...(submitted[name]?.(args) ?? args)) }); + } + return store; +} + +/** A write whose outcome the caller does not wait for: a failure is reported, never left unhandled. */ +export function observeWrite(result: unknown, failed: (error: unknown) => void): void { + if (result instanceof Promise) result.catch(failed); +} + +/** Runs `then` once the write has committed: at once for a synchronous store, after the commit otherwise. */ +export function afterWrite(result: T | Promise, then: (value: T) => void, failed: (error: unknown) => void): void { + if (result instanceof Promise) result.then(then, failed); + else then(result); +} + +/** + * One caller's writes as a queue. Each write is submitted once every earlier + * one has committed, and `after(job)` runs a read or a notification at its + * place in that order, so it sees every write queued before it. Over a + * synchronous store, or while nothing is pending, a call runs at once, as it + * always did. A failed write or job is reported and dropped, as a synchronous + * failure was. A caller that writes inside its own transaction awaits + * `settled()` first, so those writes run now, inside it. + */ +export function orderedWrites>(store: S, failed: (error: unknown) => void): { + store: S; after(job: () => unknown): void; settled(): Promise; +} { + let pending = 0, tail: Promise = Promise.resolve(); + const run = (job: () => unknown): Promise | undefined => { + try { + const result = job(); + if (result instanceof Promise) return result.then(() => {}, failed); + } catch (error) { failed(error); } + return undefined; + }; + const after = (job: () => unknown) => { + if (pending === 0) { + const running = run(job); + if (!running) return; + pending++; tail = running.finally(() => { pending--; }); + return; + } + pending++; tail = tail.then(() => run(job)).finally(() => { pending--; }); + }; + const view = Object.create(store) as S; + for (const name of ACTIVITY_WRITE_METHODS) { + if (typeof store[name] !== "function") continue; + Object.defineProperty(view, name, { enumerable: true, value: (...args: unknown[]) => { + after(() => Reflect.apply(store[name] as (...values: unknown[]) => unknown, store, args)); + } }); + } + return { store: view, after, settled: async () => { let waiting: Promise; do { waiting = tail; await waiting; } while (waiting !== tail); } }; +} diff --git a/packages/ui-server/src/activity/store.ts b/packages/ui-server/src/activity/store.ts index 82e14a9cd7..17d0470bbc 100644 --- a/packages/ui-server/src/activity/store.ts +++ b/packages/ui-server/src/activity/store.ts @@ -366,6 +366,23 @@ export interface ActivityStore { prune(options: PruneOptions): { runsPruned: number; spansDeleted: number }; } +type Awaitable = T | Promise; +/** The writes a store may commit off the event loop (#2814). */ +export type ActivityWriteMethod = "writeLifecycle" | "startSpan" | "endSpan" | "patchSpan" | "appendEvent" | "heartbeat" + | "cascadeClose" | "sweepOwnOrphans" | "sweepStale" | "rollupRun" | "prune"; + +/** + * The store as the hosted server holds it. Reads are synchronous on the + * caller's connection; a write's result may be a promise, because the server + * commits standalone writes in its write worker, off the event loop + * (`createOffloopActivityStore`). Each write is atomic, and writes commit in + * call order. An `ActivityStore` is one whose writes all commit before they + * return. + */ +export type AwaitableActivityStore = Omit & { + [K in ActivityWriteMethod]: (...args: Parameters) => Awaitable>; +}; + /** * The pricing dependency rollups price through. `resolve` MUST be synchronous * — it is called inside the rollup write transaction, which never awaits. diff --git a/packages/ui-server/src/activity/stream.ts b/packages/ui-server/src/activity/stream.ts index c155694849..95422ec855 100644 --- a/packages/ui-server/src/activity/stream.ts +++ b/packages/ui-server/src/activity/stream.ts @@ -33,7 +33,7 @@ import type { WSContext } from "../ws/clients.js"; import { sendTo } from "../ws/clients.js"; import type { ActivityChange, - ActivityStore, + AwaitableActivityStore, RunRollupRow, SpanEventRow, SpanRow, @@ -155,7 +155,7 @@ export interface ActivityStream { close(): void; } -export function createActivityStream(store: ActivityStore, log?: Logger): ActivityStream { +export function createActivityStream(store: AwaitableActivityStore, log?: Logger): ActivityStream { const subscriptions = new Map< unknown, { ws: WSContext; principalId: string; subscriptions: Subscription[] } @@ -199,7 +199,7 @@ export function createActivityStream(store: ActivityStore, log?: Logger): Activi } } - type StreamChange = ActivityChange | ReturnType[number]; + type StreamChange = ActivityChange | ReturnType[number]; function deltaFrame(change: StreamChange): ServerActivityDelta { if (change.kind === "lifecycle") return {type:"activity_delta",runId:change.runId,seq:change.seq,lifecycle:change.lifecycle}; return change.kind === "span" diff --git a/packages/ui-server/src/activity/turn-lifecycle.ts b/packages/ui-server/src/activity/turn-lifecycle.ts index c506eb437f..aa4995ecb6 100644 --- a/packages/ui-server/src/activity/turn-lifecycle.ts +++ b/packages/ui-server/src/activity/turn-lifecycle.ts @@ -1,9 +1,10 @@ import type { ActivityTurnLifecycle, ActivitySpanOutcome } from "@schlessera/brain-ui-sdk/protocol"; -import type { ActivityStore } from "./store.js"; +import type { AwaitableActivityStore } from "./store.js"; +import { orderedWrites } from "./store-writes.js"; export type LifecycleMilestone = "dequeuedAt" | "governedStartedAt" | "governedAcquiredAt" | "budgetStartedAt" | "budgetAdmittedAt" | "journalStartedAt" | "journalBeganAt" | "backendInvokedAt" | "firstFrameAt" | "failureObservedAt"; -export function createTurnLifecycle(store: ActivityStore, turnId: string, sessionId: string | null, +export function createTurnLifecycle(store: AwaitableActivityStore, turnId: string, sessionId: string | null, acceptedAt: number, onWrite: () => void, clock: () => number = Date.now) { const record: ActivityTurnLifecycle = { hostEvents: [], skippedPhases: [], lifecycleId: `lifecycle:${turnId}`, turnId, sessionId, backendRunId: null, acceptedAt, dequeuedAt: null, @@ -12,8 +13,10 @@ export function createTurnLifecycle(store: ActivityStore, turnId: string, sessio endedAt: null, outcome: null, reason: null, totalElapsedMs: null, timeToFirstFrameMs: null, queueWaitMs: null, governedAdmissionMs: null, budgetAdmissionMs: null, journalBeginMs: null, backendStartupMs: null, }; - // Observability, including fanout, must never fail the observed turn. - const persist = () => { try { store.writeLifecycle(record); onWrite(); } catch { /* best effort */ } }; + // Observability, including fanout, must never fail the observed turn. Each + // write carries the record as it is now and fans out once committed (#2814). + const writes = orderedWrites(store, () => { /* best effort */ }); + const persist = () => { writes.store.writeLifecycle(structuredClone(record)); writes.after(onWrite); }; persist(); const recordHostEvent = (kind: ActivityTurnLifecycle["hostEvents"][number]["kind"], principalId: string) => { if (record.endedAt !== null) return; diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index 758164e2ee..63dea75a38 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -91,7 +91,8 @@ import { import { principalManagementRoutes } from "./middleware/principals.js"; import { createModuleRoutes } from "./routes/modules.js"; import { createUiDb } from "./db/client.js"; -import { journalDatabasePath } from "./db/journal-database.js"; +import { openSqliteWriter } from "./db/write-worker.js"; +import { JOURNAL_SCHEMA, journalDatabasePath } from "./db/journal-database.js"; import { createInboxBridgeTools } from "./inbox/bridge-tools.js"; import { createInboxStore } from "./inbox/store.js"; import { createHygieneReview } from "./inbox/hygiene-review.js"; @@ -448,6 +449,10 @@ export async function createApp(options: CreateAppOptions = {}): Promise db.close()); + // Journal and Activity commits run on this worker's connection, off the event loop (#2814). + // It owns the journal's own file too (#2789), attached as it is here. + const writer = openSqliteWriter(db, { attach: [{ schema: JOURNAL_SCHEMA, path: journalDatabasePath(config.dbPath) }] }); + bootCleanup.push(() => writer.close()); reconciled(hostExecution.attach(db)); // Shared reference source (#2429) runs only where the native manager owns a reference pool. const referencePoolConfigured = await nativeManager.poolStatus().then(() => true, () => false); @@ -540,6 +545,8 @@ export async function createApp(options: CreateAppOptions = {}): Promise activity.close()); activity.runtime.setBoot(runtimeProbes); @@ -605,13 +612,15 @@ export async function createApp(options: CreateAppOptions = {}): Promise host.turnTimeoutMs }); - const journal = createTurnJournal(db, { nativeManager, brainPath: config.brainPath, dbPath: config.dbPath, objectsRoot: journalRoot, + const journal = createTurnJournal(db, { nativeManager, brainPath: config.brainPath, dbPath: config.dbPath, objectsRoot: journalRoot, writer, ...(referencePoolConfigured ? { referencePool: nativeManager, referenceCheckpoints: (() => { const owner = referenceWorktreeOwner(); return { record: (id: string, worktreeId: string) => recordReferenceWorktreeCheckpoint(db, owner, id, worktreeId), forget: (id: string) => forgetReferenceWorktreeCheckpoint(db, id) }; })() } : {}), changed(turnId, sessionId, summary) { host.clients.broadcast({ type: "turn_journal", sessionId, turnId, journal: { ...summary, ...(host.budgets?.saved(turnId) ? { budget: host.budgets.saved(turnId) } : {}) } }); } }); bootCleanup.push(() => journal.close()); + // Startup recovery commits, or refuses boot, before any turn can run. + await journal.ready; const policyProposals = createPolicyProposals({ root: config.brainPath, db, lock: { withLock: (key, fn, admission) => host.workerLock.withLock(key, fn, admission) }, observeUserPolicy: (turnId, write) => journal.observeUserPolicy(turnId, write), diff --git a/packages/ui-server/src/brain/journal-objects.ts b/packages/ui-server/src/brain/journal-objects.ts index fd0e0ad4cb..28e0b26bc2 100644 --- a/packages/ui-server/src/brain/journal-objects.ts +++ b/packages/ui-server/src/brain/journal-objects.ts @@ -7,11 +7,12 @@ import { promisify } from "node:util"; export interface JournalObjectReservation { readonly id?: string; /** Convert an in-flight reservation to one retained object, or release a deduplicated copy. */ - finish(hash: string, bytes: number, alreadyPresent: boolean): void; - cancel(): void; + finish(hash: string, bytes: number, alreadyPresent: boolean): void | Promise; + cancel(): void | Promise; } +/** A ledger may commit off the event loop (#2814): every step is awaited. */ export interface JournalObjectAccounting { - reserve(bytes: number, hash: string): JournalObjectReservation; + reserve(bytes: number, hash: string): JournalObjectReservation | Promise; } const DIRECTORY = constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW; @@ -34,12 +35,14 @@ export class JournalCapacityError extends Error { export function createJournalObjects(root: string, accounting: JournalObjectAccounting): { has(hash: string): boolean; putFileAsync(source: number, size: number, signal?: AbortSignal): Promise; - putBytes(bytes: Uint8Array): string; + putBytes(bytes: Uint8Array): Promise; read(hash: string, maxBytes: number): Buffer; verify(hash: string, bytes: number): boolean; - withVerifiedCopy(hash: string, bytes: number, apply: (copy: (destination: number) => void) => T): T; + /** The object stays open and verified until `apply` settles. */ + withVerifiedCopy(hash: string, bytes: number, apply: (copy: (destination: number) => void) => T | Promise): Promise; discardPending(id: string): void; - stream(hash: string, released: () => void): { body: ReadableStream; bytes: number }; + /** The body ends only after `released` settles, so a finished reader holds no lease. */ + stream(hash: string, released: () => void | Promise): { body: ReadableStream; bytes: number }; remove(hash: string): void; close(): void; } { @@ -64,10 +67,10 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco }; return { has, - putBytes(bytes) { + async putBytes(bytes) { assertOpen(); const digest = createHash("sha256").update(bytes).digest("hex"); - const reservation = accounting.reserve(bytes.length, digest); + const reservation = await accounting.reserve(bytes.length, digest); const temporary = `pending-${reservation.id ?? crypto.randomUUID()}`; let output: number | undefined, created = false, published = false; try { @@ -81,12 +84,12 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco try { fs.linkSync(path(temporary), path(digest)); published = true; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !has(digest)) throw error; duplicate = true; } unlinkSync(path(temporary)); created = false; fsyncSync(directory); - reservation.finish(digest, bytes.length, duplicate); + await reservation.finish(digest, bytes.length, duplicate); return digest; } catch (error) { if (output !== undefined) closeSync(output); if (created) unlinkSync(path(temporary)); - if (!published) reservation.cancel(); throw error; + if (!published) await reservation.cancel(); throw error; } }, async putFileAsync(source, size, signal) { @@ -122,7 +125,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco } const expectedHash = await digestSource(); signal?.throwIfAborted(); - const reservation = accounting.reserve(size, expectedHash); + const reservation = await accounting.reserve(size, expectedHash); const name = `pending-${reservation.id ?? crypto.randomUUID()}`; let output: number | undefined, present = false, published = false; try { @@ -136,12 +139,12 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !has(digest)) throw error; duplicate = true; } await unlinkAsync(path(name)); present = false; await syncAsync(directory); - reservation.finish(digest, size, duplicate); + await reservation.finish(digest, size, duplicate); return digest; } catch (error) { if (output !== undefined) await closeAsync(output); if (present) await unlinkAsync(path(name)); - if (!published) reservation.cancel(); + if (!published) await reservation.cancel(); throw error; } }, @@ -181,7 +184,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco try { unlinkSync(path(`pending-${id}`)); fsyncSync(directory); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } }, - withVerifiedCopy(hash, bytes, apply) { + async withVerifiedCopy(hash, bytes, apply) { const fd = openObject(hash); try { if (fstatSync(fd).size !== bytes) throw new Error("Retained version size changed."); @@ -192,7 +195,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco digest.update(buffer.subarray(0, count)); position += count; } if (digest.digest("hex") !== hash) throw new Error("Retained version checksum changed."); - return apply(destination => { + return await apply(destination => { let position = 0; while (position < bytes) { const count = readSync(fd, buffer, 0, Math.min(buffer.length, bytes - position), position); @@ -208,21 +211,21 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco const fd = openObject(hash), bytes = fstatSync(fd).size; let position = 0, ended = false; const digest = createHash("sha256"); - const finish = () => { if (!ended) { ended = true; closeSync(fd); released(); } }; + const finish = async () => { if (!ended) { ended = true; closeSync(fd); await released(); } }; const body = new ReadableStream({ - pull(controller) { + async pull(controller) { try { if (position === bytes) { if (digest.digest("hex") !== hash) throw new Error("Journal object integrity check failed."); - finish(); controller.close(); return; + await finish(); controller.close(); return; } const chunk = Buffer.alloc(Math.min(64 * 1024, bytes - position)); const count = readSync(fd, chunk, 0, chunk.length, position); if (!count) throw new Error("Journal object ended before its recorded size."); const observed = chunk.subarray(0, count); digest.update(observed); position += count; controller.enqueue(observed); - } catch (error) { finish(); controller.error(error); } + } catch (error) { await finish(); controller.error(error); } }, - cancel() { finish(); }, + async cancel() { await finish(); }, }); return { body, bytes }; }, diff --git a/packages/ui-server/src/brain/journal-store.ts b/packages/ui-server/src/brain/journal-store.ts new file mode 100644 index 0000000000..a6bc8d428e --- /dev/null +++ b/packages/ui-server/src/brain/journal-store.ts @@ -0,0 +1,224 @@ +/** + * The turn journal's writes, run on the write worker's connection (#2814). + * Each procedure is one accounted transaction of `createJournalAccounting`, + * committed at `synchronous=FULL` before its call resolves on the event loop. + * The host journal (`turn-journal.ts`) keeps everything else: observation, + * in-memory turn state and every read. + */ +import type { Database, SQLQueryBindings } from "bun:sqlite"; +import type { JournalObservationReference, JournalVersion } from "@schlessera/brain-ui-sdk/protocol"; + +import type { WireError, WriteProcedures } from "../db/write-worker.js"; +import { recordGovernanceEvent } from "../governance/events.js"; +import { createJournalAccounting, JournalStorageUnavailable } from "./journal-accounting.js"; +import { createJournalObjects, JournalCapacityError, type JournalObjectReservation } from "./journal-objects.js"; + +export const url = import.meta.url; + +export interface JournalStoreOptions { dbPath: string; objectsRoot: string; capBytes?: number } + +/** + * One statement of an accounted write. A `check` entry refuses the whole + * write with `refusal` unless its query returns a row. + */ +export type JournalStatement = { sql: string; params?: SQLQueryBindings[] } | { check: string; params?: SQLQueryBindings[]; refusal: string }; + +/** A `check` statement found no row: the write rolled back. */ +export class JournalStatementRefused extends Error {} + +export const journalStatement = (sql: string, ...params: SQLQueryBindings[]): JournalStatement => ({ sql, params }); +/** A `check` statement: the whole write is refused with `refusal` unless `sql` returns a row. */ +export const journalCheck = (sql: string, refusal: string, ...params: SQLQueryBindings[]): JournalStatement => ({ check: sql, params, refusal }); + +/** Journal refusals keep their class and message across the worker boundary. */ +export function reviveJournalError(wire: WireError): Error | undefined { + const error = wire.name === "JournalCapacityError" ? new JournalCapacityError(wire.message) + : wire.name === "JournalStorageUnavailable" ? new JournalStorageUnavailable("") + : wire.name === "JournalStatementRefused" ? new JournalStatementRefused(wire.message) : undefined; + if (error) error.message = wire.message; + return error; +} + +type TurnRow = { session_id: string | null; actor: "interactive" | "voice" | "user"; kind: "turn" | "revert"; model: string | null; profile: "strict" | "governed"; steering_inventory_missing: string; missing: string }; + +/** One observed change of a turn, as `observe()` classified it before admission. */ +export interface ObservedChange { path: string; scope: string; before: JournalVersion | null; after: JournalVersion | null } + +export interface ObservationRecord { + turnId: string; + /** The turn's last used change sequence; each recorded change takes the next. */ + sequence: number; + at: number; + userPolicy: boolean; + changed: ObservedChange[]; + steeringPaths: string[]; + /** Every turn observing this window, this one included. */ + overlaps: string[]; + /** Changes of this turn's managed worktrees, already in journal path form. */ + worktree: { path: string; scope: string; before: JournalVersion | null; after: JournalVersion | null }[]; + steeringUnavailable: string[]; + unreadable: { path: string; kind?: string; bytes?: number; ms: number; [key: string]: unknown }[]; +} + +export function setup(db: Database, options: JournalStoreOptions): WriteProcedures { + const accounting = createJournalAccounting(db, options.dbPath, { capBytes: options.capBytes }); + const objects = createJournalObjects(options.objectsRoot, accounting); + const reservations = new Map(); + const temporary = new Map(); + const pin = (turnId: string, entry: JournalVersion | null) => { + if (entry?.hash) db.query("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)").run(turnId, entry.hash); + }; + const turnContext = (turnId: string) => { + const row = db.query("SELECT * FROM turn_journal_turns WHERE turn_id=?").get(turnId) as TurnRow; + const taint = db.query("SELECT id FROM governance_events WHERE kind='taint' AND (turn_id=? OR (? IS NOT NULL AND session_id=?)) ORDER BY id LIMIT 1") + .get(turnId, row.session_id, row.session_id) as { id: number } | null; + return { row, taint }; + }; + return { + /** Statements in order, as one accounted transaction; the changes each made. */ + statements(list: JournalStatement[]): number[] { + return accounting.write(() => list.map(statement => { + if ("check" in statement) { + if (!db.query(statement.check).get(...(statement.params ?? []))) throw new JournalStatementRefused(statement.refusal); + return 0; + } + return db.query(statement.sql).run(...(statement.params ?? [])).changes; + })); + }, + bindModel(turnId: string, sessionId: string | null, model: string): boolean { + return accounting.write(() => { + const row = db.query("SELECT session_id,ended_at FROM turn_journal_turns WHERE turn_id=?").get(turnId) as { session_id: string | null; ended_at: number | null } | null; + if (!row || row.ended_at !== null || row.session_id !== sessionId) return false; + db.query("UPDATE turn_journal_turns SET model=? WHERE turn_id=? AND ended_at IS NULL").run(model, turnId); + return true; + }); + }, + collectObjects(): void { + accounting.write(() => { + const unused = db.query("SELECT hash FROM turn_journal_objects WHERE state='present' AND NOT EXISTS(SELECT 1 FROM turn_journal_object_refs r WHERE r.hash=turn_journal_objects.hash) AND NOT EXISTS(SELECT 1 FROM turn_journal_reservations r WHERE r.hash=turn_journal_objects.hash)").all() as { hash: string }[]; + for (const row of unused) { + if (objects.has(row.hash)) objects.remove(row.hash); + db.query("DELETE FROM turn_journal_objects WHERE hash=?").run(row.hash); + } + }); + }, + /** The changes of one turn's observation window; the turn's last sequence after them. */ + recordObservation(record: ObservationRecord): number { + const { turnId, at } = record; + const steering = new Set(record.steeringPaths); + let sequence = record.sequence; + accounting.write(() => { + const { row: metadata, taint } = turnContext(turnId); + for (const item of record.changed) { + sequence++; + const approvedPolicy = record.userPolicy && item.path === "context/policies/governance.md"; + const classification = approvedPolicy ? "policy" : steering.has(item.path) ? "steering" : "content"; + const actor = approvedPolicy || metadata.kind === "revert" ? "user" : "agent"; + const source = classification === "steering" && actor === "agent" && taint ? taint.id : null; + const overlappingTurnIds = record.overlaps.filter(id => id !== turnId); + db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,?,?,?,?)") + .run(turnId, sequence, item.path, item.scope, item.before ? JSON.stringify(item.before) : null, item.after ? JSON.stringify(item.after) : null, classification, actor, JSON.stringify(overlappingTurnIds), source); + if (classification === "steering") recordGovernanceEvent(db, { kind: "steering_change", sessionId: metadata.session_id, turnId, + actor: actor === "user" ? "user" : metadata.actor, model: metadata.model, profile: metadata.profile, at, dedupeKey: `${turnId}:${sequence}`, + payload: { path: item.path, sequence, beforeHash: item.before?.hash ?? null, afterHash: item.after?.hash ?? null, taintSourceEventId: source, overlappingTurnIds } }); + pin(turnId, item.before); pin(turnId, item.after); + const observed = db.query("SELECT sequence,after_json,overlaps FROM turn_journal_changes WHERE turn_id=? AND path=? ORDER BY sequence").all(turnId, item.path) as {sequence:number;after_json:string|null;overlaps:string}[]; + const participants = new Set(observed.flatMap(row => JSON.parse(row.overlaps) as string[])); + const conflicting = new Map(); + for (const other of participants) { + const shared = observed.filter(row => (JSON.parse(row.overlaps) as string[]).includes(other)); + if (new Set(shared.map(row => row.after_json ?? "null")).size < 2) continue; + for (const row of shared) conflicting.set(row.sequence, {turnId, sequence:row.sequence, side:"after"}); + } + if (conflicting.size) db.query("UPDATE turn_journal_changes SET conflict_observations=? WHERE turn_id=? AND path=?") + .run(JSON.stringify([...conflicting.values()].sort((a,b)=>a.sequence-b.sequence)),turnId,item.path); + } + // Intentional worktree edits are editable workspace content of their holding turn only. + for (const change of record.worktree) { + sequence++; + db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,?,?,?,?)") + .run(turnId, sequence, change.path, change.scope, change.before ? JSON.stringify(change.before) : null, + change.after ? JSON.stringify(change.after) : null, "content", metadata.kind === "revert" ? "user" : "agent", "[]", null); + pin(turnId, change.before); pin(turnId, change.after); + } + db.query("UPDATE turn_journal_turns SET steering_inventory_missing=? WHERE turn_id=?").run(JSON.stringify([...new Set([...(JSON.parse(metadata.steering_inventory_missing) as string[]), ...record.steeringUnavailable])]), turnId); + if (record.unreadable.length) { + const missing = new Map((JSON.parse(metadata.missing) as { path: string; reason: string }[]).map(item => [item.path, item])); + for (const item of record.unreadable) { + const { ms, ...known } = item; + missing.set(item.path, known); + const signature = `journal_reader:${item.kind ?? "unknown"}:${item.bytes ?? "unknown"}:${item.path}`.slice(0, 1000); + recordGovernanceEvent(db, { kind: "roadblock", sessionId: metadata.session_id, turnId, actor: metadata.actor, + model: metadata.model, profile: metadata.profile, at, dedupeKey: `journal-reader:${turnId}:${item.path}`, + payload: { category: "permission_denied", family: "journal_reader", signature, + failures: [{ toolCallId: null, signature, category: "permission_denied" }], resolution: null, + remedy: "capability", waste: { tokens: null, costUsd: null, ms }, eventIds: [] } }); + } + db.query("UPDATE turn_journal_turns SET state='partial',missing=? WHERE turn_id=?").run(JSON.stringify([...missing.values()]), turnId); + } + }); + return sequence; + }, + /** A saved or recovered native memory note, as one steering change at `sequence`. */ + recordClaudeMemory(turnId: string, sequence: number, path: string, target: { directory: string; device: string; inode: string; note: string }, + before: JournalVersion | null, after: JournalVersion | null, at: number): void { + accounting.write(() => { + const { row: metadata, taint: found } = turnContext(turnId); + const actor = metadata.actor === "user" ? "user" : "agent"; + const taint = actor === "agent" ? found : null; + db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,'steering',?,'[]',?)") + .run(turnId, sequence, path, "claude_memory", before ? JSON.stringify(before) : null, after ? JSON.stringify(after) : null, actor, taint?.id ?? null); + db.query("INSERT OR REPLACE INTO turn_journal_claude_memory VALUES (?,?,?,?,?,?,?,?,?)").run(turnId, path, target.directory, target.device, target.inode, target.note, before ? JSON.stringify(before) : null, after ? JSON.stringify(after) : null, "saved"); + pin(turnId, before); pin(turnId, after); + recordGovernanceEvent(db, { kind: "steering_change", sessionId: metadata.session_id, turnId, actor: metadata.actor, model: metadata.model, profile: metadata.profile, at, dedupeKey: `${turnId}:${sequence}`, + payload: { path, sequence, beforeHash: before?.hash ?? null, afterHash: after?.hash ?? null, taintSourceEventId: taint?.id ?? null, overlappingTurnIds: [] } }); + }); + }, + /** False when the turn has no record to mark. */ + markMemoryUnconfirmed(turnId: string, path: string): boolean { + return accounting.write(() => { + const row = db.query("SELECT missing FROM turn_journal_turns WHERE turn_id=?").get(turnId) as {missing:string} | null; + if (!row) return false; + const missing = (JSON.parse(row.missing) as {path:string;reason:string}[]).filter(item => item.path !== path); + missing.push({path,reason:"memory_persistence_unconfirmed"}); + db.query("UPDATE turn_journal_turns SET state='partial',missing=? WHERE turn_id=?").run(JSON.stringify(missing),turnId); + return true; + }); + }, + reserve(bytes: number, hash: string): string { + // This ledger commits on this connection: its reservation is synchronous. + const reservation = accounting.reserve(bytes, hash) as JournalObjectReservation; + reservations.set(reservation.id!, reservation); + return reservation.id!; + }, + finishReservation(id: string, hash: string, bytes: number, duplicate: boolean): void { + const reservation = reservations.get(id); + if (!reservation) throw new JournalStorageUnavailable("the content reservation is unknown"); + reservation.finish(hash, bytes, duplicate); reservations.delete(id); + }, + cancelReservation(id: string): void { + const reservation = reservations.get(id); + if (!reservation) throw new JournalStorageUnavailable("the content reservation is unknown"); + reservation.cancel(); reservations.delete(id); + }, + reserveTemporary(bytes: number): string { + const reservation = accounting.reserveTemporary(bytes); + temporary.set(reservation.id, reservation); + return reservation.id; + }, + releaseTemporary(id: string): void { + temporary.get(id)?.release(); temporary.delete(id); + }, + /** A read lease; with `ownerTurnId`, only while that ended turn still owns the object. */ + openRead(hash: string, ownerTurnId?: string): string { + return accounting.openRead(hash, ownerTurnId === undefined ? undefined : () => { + if (!db.query("SELECT 1 FROM turn_journal_object_refs r JOIN turn_journal_turns t ON t.turn_id=r.turn_id WHERE r.turn_id=? AND r.hash=? AND t.state IN ('complete','partial') AND t.ended_at IS NOT NULL").get(ownerTurnId, hash)) throw new JournalStorageUnavailable("original retention ownership expired before admission"); + }); + }, + releaseRead(id: string): void { accounting.releaseRead(id); }, + recover(): void { accounting.recover(objects); }, + usage(): number { return accounting.usage(); }, + close(): void { objects.close(); }, + }; +} + diff --git a/packages/ui-server/src/brain/native-brain-leases.ts b/packages/ui-server/src/brain/native-brain-leases.ts index 21079d9a52..02083fcb56 100644 --- a/packages/ui-server/src/brain/native-brain-leases.ts +++ b/packages/ui-server/src/brain/native-brain-leases.ts @@ -1,10 +1,13 @@ /** Concrete native-root ownership across app/cron crashes, separate from UI database creation. */ -import { Database } from "bun:sqlite"; +import { Database, type SQLQueryBindings } from "bun:sqlite"; import { randomUUID } from "node:crypto"; import { constants, closeSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, statSync, realpathSync } from "node:fs"; import { dirname, isAbsolute, join, relative } from "node:path"; import { NativeBrainManagerError, REFERENCE_POOL_CHECKPOINT_BYTES, type createNativeBrainManagerClient, type NativePoolRecord } from "@schlessera/brain-ui-sdk/internal"; import { processAlive, SELF } from "./writer-gate.js"; +import { openSqliteWriter } from "../db/write-worker.js"; +import * as nativeLeaseStore from "./native-lease-store.js"; +import { reviveNativeLeaseError, type NativeLeaseOwner } from "./native-lease-store.js"; type Client = ReturnType; type PoolLease = { id: string; purpose: "source" | "worktree" | "staging" | "checkpoint"; from_id: string | null; allowance: number; state: "creating" | "held" | "releasing" }; @@ -80,6 +83,11 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { } } catch (error) { db.close(); throw error; } const generation = randomUUID(); + // Every durable intent commits on the write worker, off the event loop (#2814). + const writer = openSqliteWriter(db); + const leases = writer.channel(nativeLeaseStore, { generation, boot, pidNamespace }, reviveNativeLeaseError); + try { await leases.ready; } catch (error) { await writer.close(); db.close(); throw error; } + const run = (sql: string, ...params: SQLQueryBindings[]) => leases.call("run", sql, params); let closed = false; const pending = new Set>(); function operation(run: () => Promise): Promise { @@ -98,26 +106,22 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { const mine = (lease: Lease) => lease.owner_pid === SELF.pid && lease.owner_start === SELF.start && lease.owner_generation === generation && lease.owner_boot === boot && lease.owner_pid_namespace === pidNamespace; async function releaseOwned(lease: Lease, signal?: AbortSignal): Promise<{ id: string; state: "deleted" }> { // This intent must survive a lost delete reply, and never uses admission's aborted signal. - db.query("UPDATE native_root_leases SET state='deleting' WHERE id=?").run(lease.id); + await run("UPDATE native_root_leases SET state='deleting' WHERE id=?", lease.id); if (lease.kind === "checkpoint" && db.query("SELECT 1 FROM native_root_leases WHERE kind='stage' AND stage_lower_id=? LIMIT 1").get(lease.id)) throw new NativeBrainManagerError("unknown", "A retained native stage still depends on this checkpoint; reconcile stage deletion first."); const result = await (lease.kind === "stage" ? raw.releaseStage(lease.id, signal) : raw.release(lease.id, signal)); if (result.id !== lease.id || result.state !== "deleted") throw new NativeBrainManagerError("unknown", "Native lease deletion receipt is unreadable; retain and reconcile the same identity."); - db.query("DELETE FROM native_root_leases WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=?").run(lease.id, lease.owner_pid, lease.owner_start, lease.owner_generation); + await run("DELETE FROM native_root_leases WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=?", lease.id, lease.owner_pid, lease.owner_start, lease.owner_generation); return result; } const client: Client = { ...raw, checkpoint(id, signal) { return operation(async () => { signal?.throwIfAborted(); - db.transaction(() => { - const lease = row(id); - if (lease && (!mine(lease) || lease.kind !== "checkpoint" || lease.state === "deleting")) throw new NativeBrainManagerError("unknown", "Checkpoint identity belongs to another live owner or deletion; no admission is authorized."); - if (!lease) db.query("INSERT INTO native_root_leases(id,owner_pid,owner_start,owner_generation,owner_boot,owner_pid_namespace,state) VALUES (?,?,?,?,?,?,'creating')").run(id, SELF.pid, SELF.start, generation, boot, pidNamespace); - }).immediate(); + await leases.call("checkpointIntent", id); const checkpoint = await raw.checkpoint(id, signal); if (checkpoint.id !== id || checkpoint.fsid !== options.fsid || checkpoint.sourceUuid !== options.brainUuid) throw new NativeBrainManagerError("unknown", "Native checkpoint reply does not match its durable lease identity."); - db.query("UPDATE native_root_leases SET state='sealed',checkpoint_uuid=? WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=? AND state='creating'").run(checkpoint.uuid, id, SELF.pid, SELF.start, generation); + await run("UPDATE native_root_leases SET state='sealed',checkpoint_uuid=? WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=? AND state='creating'", checkpoint.uuid, id, SELF.pid, SELF.start, generation); return checkpoint; }); }, delta(from, to, signal) { return operation(async () => { @@ -130,17 +134,10 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { }); }, prepareStage(id, checkpointId, allowanceBytes, signal) { return operation(async () => { signal?.throwIfAborted(); - let lowerUuid: string | null = null; - db.transaction(() => { - const lower = row(checkpointId), lease = row(id); - if (!lower || lower.kind !== "checkpoint" || !mine(lower) || lower.state !== "sealed" || !lower.checkpoint_uuid) throw new NativeBrainManagerError("unknown", "Native stage requires this owner's retained sealed checkpoint lease."); - if (lease && (!mine(lease) || lease.kind !== "stage" || lease.state === "deleting" || lease.stage_lower_id !== checkpointId || lease.stage_allowance !== allowanceBytes)) throw new NativeBrainManagerError("unknown", "Native stage identity belongs to another owner or deletion."); - lowerUuid = lower.checkpoint_uuid; - if (!lease) db.query("INSERT INTO native_root_leases(id,owner_pid,owner_start,owner_generation,owner_boot,owner_pid_namespace,state,kind,stage_lower_id,stage_allowance) VALUES (?,?,?,?,?,?,'creating','stage',?,?)").run(id, SELF.pid, SELF.start, generation, boot, pidNamespace, checkpointId, allowanceBytes); - }).immediate(); + const lowerUuid = await leases.call("stageIntent", id, checkpointId, allowanceBytes); const stage = await raw.prepareStage(id, checkpointId, allowanceBytes, signal); if (stage.id !== id || stage.checkpointId !== checkpointId || stage.checkpointUuid !== lowerUuid || stage.sourceUuid !== options.brainUuid || stage.allowanceBytes !== allowanceBytes) throw new NativeBrainManagerError("unknown", "Native stage reply does not match its durable lease identity."); - db.query("UPDATE native_root_leases SET state='sealed' WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=? AND state='creating'").run(id, SELF.pid, SELF.start, generation); + await run("UPDATE native_root_leases SET state='sealed' WHERE id=? AND owner_pid=? AND owner_start=? AND owner_generation=? AND state='creating'", id, SELF.pid, SELF.start, generation); return stage; }); }, releaseStage(id, signal) { return operation(async () => { @@ -151,18 +148,11 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { }); }, poolCreate(id, purpose, from, allowanceBytes, signal) { return operation(async () => { signal?.throwIfAborted(); - db.transaction(() => { - const lease = poolRow(id); - if (row(id)) throw new NativeBrainManagerError("unknown", "This identity already belongs to a checkpoint or stage lease."); - if (lease && (lease.state === "releasing" || lease.purpose !== purpose || lease.from_id !== from || lease.allowance !== allowanceBytes)) - throw new NativeBrainManagerError("unknown", "Reference pool identity is recorded with another purpose, source, allowance or release."); - if (from !== null && poolRow(from)?.state !== "held") throw new NativeBrainManagerError("unknown", "A reference snapshot needs this app's held source root."); - if (!lease) db.query("INSERT INTO native_pool_leases(id,purpose,from_id,allowance,state) VALUES (?,?,?,?,'creating')").run(id, purpose, from, allowanceBytes); - }).immediate(); + await leases.call("poolIntent", id, purpose, from, allowanceBytes); const record = await raw.poolCreate(id, purpose, from, allowanceBytes, signal); if (record.id !== id || record.purpose !== purpose || record.from !== from || record.allowanceBytes !== allowanceBytes || !["writable", "sealed"].includes(record.state)) throw new NativeBrainManagerError("unknown", "Reference pool reply does not match its durable lease identity."); - db.query("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='creating'").run(id); + await run("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='creating'", id); return record; }); }, poolSeal(id, signal) { return operation(async () => { @@ -185,18 +175,11 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { signal?.throwIfAborted(); // A worktree whose release never committed is assessable again (#2431). await settleRelease(from, signal); - db.transaction(() => { - const lease = poolRow(id), worktree = poolRow(from); - if (row(id)) throw new NativeBrainManagerError("unknown", "This identity already belongs to a checkpoint or stage lease."); - if (lease && (lease.state === "releasing" || lease.purpose !== "checkpoint" || lease.from_id !== from)) - throw new NativeBrainManagerError("unknown", "Reference pool identity is recorded with another purpose, source or release."); - if (worktree?.purpose !== "worktree" || worktree.state !== "held") throw new NativeBrainManagerError("unknown", "A worktree checkpoint needs this app's held worktree."); - if (!lease) db.query("INSERT INTO native_pool_leases(id,purpose,from_id,allowance,state) VALUES (?,'checkpoint',?,?,'creating')").run(id, from, REFERENCE_POOL_CHECKPOINT_BYTES); - }).immediate(); + await leases.call("poolCheckpointIntent", id, from, REFERENCE_POOL_CHECKPOINT_BYTES); const record = await raw.poolCheckpoint(id, from, signal); if (record.id !== id || record.purpose !== "checkpoint" || record.from !== from || record.state !== "sealed" || record.readOnly !== true) throw new NativeBrainManagerError("unknown", "Reference pool checkpoint reply does not match its durable lease identity."); - db.query("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='creating'").run(id); + await run("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='creating'", id); return record; }); }, poolDelta(from, to, signal) { return operation(async () => { @@ -226,14 +209,14 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { if (db.query("SELECT 1 FROM native_pool_leases WHERE from_id=? AND state='creating' LIMIT 1").get(id)) throw new NativeBrainManagerError("unavailable", "A reference snapshot is still being created from this root."); // Survives a lost reply; a retry continues the same identity's release. - db.query("UPDATE native_pool_leases SET state='releasing' WHERE id=?").run(id); + await run("UPDATE native_pool_leases SET state='releasing' WHERE id=?", id); let record: NativePoolRecord; releases.set(id, (releases.get(id) ?? 0) + 1); try { record = await raw.poolRelease(id, signal); } catch (error) { await settleRelease(id, undefined, 1).catch(() => {}); throw error; } finally { releases.set(id, releases.get(id)! - 1); if (!releases.get(id)) releases.delete(id); } if (record.id !== id || !["deleted", "reclaimed"].includes(record.state)) throw new NativeBrainManagerError("unknown", "Reference pool release receipt is unreadable; retain and reconcile the same identity."); - db.query("DELETE FROM native_pool_leases WHERE id=? AND state='releasing'").run(id); + await run("DELETE FROM native_pool_leases WHERE id=? AND state='releasing'", id); return record; }); }, release(id, signal) { return operation(async () => { @@ -264,7 +247,7 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { const record = await poolRecord(id, signal); // Another release of this identity may have started while the pool answered; its intent stands. if ((releases.get(id) ?? 0) !== own) return; - if (record && (record.state === "writable" || record.state === "sealed")) db.query("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='releasing'").run(id); + if (record && (record.state === "writable" || record.state === "sealed")) await run("UPDATE native_pool_leases SET state='held' WHERE id=? AND state='releasing'", id); } async function recoverDeadOwners(): Promise<{ released: string[]; retainedLive: number; retainedUnverifiable: number; unknown: string[]; retainedPool: number }> { return operation(async () => { @@ -295,7 +278,7 @@ export async function createNativeBrainLeaseManager(raw: Client, options: { try { const owned = db.query("SELECT * FROM native_root_leases WHERE owner_pid=? AND owner_start=? AND owner_generation=? ORDER BY (kind='stage') DESC,id").all(SELF.pid, SELF.start, generation) as Lease[]; for (const lease of owned) { try { await releaseOwned(lease); } catch (error) { failures.push(error); } } - } finally { db.close(); } + } finally { await writer.close(); db.close(); } if (failures.length) throw new AggregateError(failures, "Native lease close retains unknown root outcomes for later owner recovery."); })(); return closing; diff --git a/packages/ui-server/src/brain/native-lease-store.ts b/packages/ui-server/src/brain/native-lease-store.ts new file mode 100644 index 0000000000..6b36ed461d --- /dev/null +++ b/packages/ui-server/src/brain/native-lease-store.ts @@ -0,0 +1,72 @@ +/** + * Native root lease writes, run on the lease database's write worker at + * `synchronous=FULL` (#2814). Each durable intent commits before its call + * resolves, so the event loop awaits the fsync instead of blocking in it. + * The lease manager keeps every read and every manager request. + */ +import type { Database, SQLQueryBindings } from "bun:sqlite"; +import { NativeBrainManagerError } from "@schlessera/brain-ui-sdk/internal"; + +import type { WireError, WriteProcedures } from "../db/write-worker.js"; +import { SELF } from "./writer-gate.js"; + +export const url = import.meta.url; + +/** This owner's identity: the manager instance's generation on this boot and PID namespace. */ +export interface NativeLeaseOwner { generation: string; boot: string; pidNamespace: string } + +type Lease = { kind: "checkpoint" | "stage"; checkpoint_uuid: string | null; stage_lower_id: string | null; stage_allowance: number | null; owner_pid: number; owner_start: string; owner_generation: string; owner_boot: string; owner_pid_namespace: string; state: "creating" | "sealed" | "deleting" }; +type PoolLease = { purpose: string; from_id: string | null; allowance: number; state: "creating" | "held" | "releasing" }; + +export function reviveNativeLeaseError(wire: WireError): Error | undefined { + return wire.name === "NativeBrainManagerError" ? new NativeBrainManagerError(wire.fields.code as never, wire.message) : undefined; +} + +export function setup(db: Database, owner: NativeLeaseOwner): WriteProcedures { + const row = (id: string) => db.query("SELECT * FROM native_root_leases WHERE id=?").get(id) as Lease | null; + const poolRow = (id: string) => db.query("SELECT * FROM native_pool_leases WHERE id=?").get(id) as PoolLease | null; + const mine = (lease: Lease) => lease.owner_pid === SELF.pid && lease.owner_start === SELF.start && lease.owner_generation === owner.generation && lease.owner_boot === owner.boot && lease.owner_pid_namespace === owner.pidNamespace; + return { + /** One statement, committed on its own; the rows it changed. */ + run(sql: string, params: SQLQueryBindings[]): number { return db.query(sql).run(...params).changes; }, + checkpointIntent(id: string): void { + db.transaction(() => { + const lease = row(id); + if (lease && (!mine(lease) || lease.kind !== "checkpoint" || lease.state === "deleting")) throw new NativeBrainManagerError("unknown", "Checkpoint identity belongs to another live owner or deletion; no admission is authorized."); + if (!lease) db.query("INSERT INTO native_root_leases(id,owner_pid,owner_start,owner_generation,owner_boot,owner_pid_namespace,state) VALUES (?,?,?,?,?,?,'creating')").run(id, SELF.pid, SELF.start, owner.generation, owner.boot, owner.pidNamespace); + }).immediate(); + }, + /** The lower checkpoint's identity the stage must report. */ + stageIntent(id: string, checkpointId: string, allowanceBytes: number): string { + let lowerUuid = ""; + db.transaction(() => { + const lower = row(checkpointId), lease = row(id); + if (!lower || lower.kind !== "checkpoint" || !mine(lower) || lower.state !== "sealed" || !lower.checkpoint_uuid) throw new NativeBrainManagerError("unknown", "Native stage requires this owner's retained sealed checkpoint lease."); + if (lease && (!mine(lease) || lease.kind !== "stage" || lease.state === "deleting" || lease.stage_lower_id !== checkpointId || lease.stage_allowance !== allowanceBytes)) throw new NativeBrainManagerError("unknown", "Native stage identity belongs to another owner or deletion."); + lowerUuid = lower.checkpoint_uuid; + if (!lease) db.query("INSERT INTO native_root_leases(id,owner_pid,owner_start,owner_generation,owner_boot,owner_pid_namespace,state,kind,stage_lower_id,stage_allowance) VALUES (?,?,?,?,?,?,'creating','stage',?,?)").run(id, SELF.pid, SELF.start, owner.generation, owner.boot, owner.pidNamespace, checkpointId, allowanceBytes); + }).immediate(); + return lowerUuid; + }, + poolIntent(id: string, purpose: string, from: string | null, allowanceBytes: number): void { + db.transaction(() => { + const lease = poolRow(id); + if (row(id)) throw new NativeBrainManagerError("unknown", "This identity already belongs to a checkpoint or stage lease."); + if (lease && (lease.state === "releasing" || lease.purpose !== purpose || lease.from_id !== from || lease.allowance !== allowanceBytes)) + throw new NativeBrainManagerError("unknown", "Reference pool identity is recorded with another purpose, source, allowance or release."); + if (from !== null && poolRow(from)?.state !== "held") throw new NativeBrainManagerError("unknown", "A reference snapshot needs this app's held source root."); + if (!lease) db.query("INSERT INTO native_pool_leases(id,purpose,from_id,allowance,state) VALUES (?,?,?,?,'creating')").run(id, purpose, from, allowanceBytes); + }).immediate(); + }, + poolCheckpointIntent(id: string, from: string, allowanceBytes: number): void { + db.transaction(() => { + const lease = poolRow(id), worktree = poolRow(from); + if (row(id)) throw new NativeBrainManagerError("unknown", "This identity already belongs to a checkpoint or stage lease."); + if (lease && (lease.state === "releasing" || lease.purpose !== "checkpoint" || lease.from_id !== from)) + throw new NativeBrainManagerError("unknown", "Reference pool identity is recorded with another purpose, source or release."); + if (worktree?.purpose !== "worktree" || worktree.state !== "held") throw new NativeBrainManagerError("unknown", "A worktree checkpoint needs this app's held worktree."); + if (!lease) db.query("INSERT INTO native_pool_leases(id,purpose,from_id,allowance,state) VALUES (?,'checkpoint',?,?,'creating')").run(id, from, allowanceBytes); + }).immediate(); + }, + }; +} diff --git a/packages/ui-server/src/brain/revert-files.ts b/packages/ui-server/src/brain/revert-files.ts index 66a9fadf8b..b94f43783d 100644 --- a/packages/ui-server/src/brain/revert-files.ts +++ b/packages/ui-server/src/brain/revert-files.ts @@ -71,8 +71,8 @@ export function restoreJournalPath(brain: string, path: string, expected: Journa } /** Same sync naming, but raw bytes and exclusive descriptor-rooted creation. */ -export function createJournalSibling(brain: string, source: string, desired: JournalVersion, copy: (fd: number) => void, - prepare: (path: string) => void, occupied: (path: string) => void): string { +export async function createJournalSibling(brain: string, source: string, desired: JournalVersion, copy: (fd: number) => void, + prepare: (path: string) => void | Promise, occupied: (path: string) => void | Promise): Promise { if (process.platform !== "linux" || !reversibleJournalPath(source) || desired.kind !== "file" || desired.ignored || !desired.hash) throw new Error("sibling_version_unavailable"); const descriptors: number[] = [], parents: {parent:number;name:string;child:number}[] = []; let file: number | undefined; @@ -91,9 +91,9 @@ export function createJournalSibling(brain: string, source: string, desired: Jou const candidate=`${stem}-remote${number===1?"":`-${number}`}${extension}`, path=[...parts,candidate].join("/"); verify(); if (lstatSync(at(parent,candidate),{throwIfNoEntry:false})) continue; // Reserve durable operation metadata and journal coverage before effects. - prepare(path); verify(); + await prepare(path); verify(); try { file=openSync(at(parent,candidate),constants.O_WRONLY|constants.O_CREAT|constants.O_EXCL|constants.O_NOFOLLOW|constants.O_NONBLOCK,desired.mode); } - catch(error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") { occupied(path); continue; } throw error; } + catch(error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") { await occupied(path); continue; } throw error; } verify(); const opened=fstatSync(file), named=lstatSync(at(parent,candidate)); if(!opened.isFile()||opened.nlink!==1||opened.dev!==named.dev||opened.ino!==named.ino)throw new Error("changed_entry"); copy(file); fchmodSync(file,desired.mode); fsyncSync(file); fsyncSync(parent); return path; diff --git a/packages/ui-server/src/brain/turn-journal.ts b/packages/ui-server/src/brain/turn-journal.ts index b344ab8f19..7842cbd341 100644 --- a/packages/ui-server/src/brain/turn-journal.ts +++ b/packages/ui-server/src/brain/turn-journal.ts @@ -1,9 +1,13 @@ /** Concrete host journal: observations, durable ownership and completed-turn collection. */ -import type { Database } from "bun:sqlite"; +import type { Database, SQLQueryBindings } from "bun:sqlite"; import type { JournalVersion, TurnJournalDetails, TurnJournalSummary, JournalOutwardAction, JournalObservationReference } from "@schlessera/brain-ui-sdk/protocol"; -import { createJournalAccounting, JournalStorageUnavailable } from "./journal-accounting.js"; +import { openSqliteWriter, type SqliteWriter } from "../db/write-worker.js"; +import { JOURNAL_SCHEMA, journalDatabasePath } from "../db/journal-database.js"; +import { installEscapeGuard, JournalStorageUnavailable } from "./journal-accounting.js"; import { replaySteeringEvents } from "./journal-governance.js"; import { createJournalObjects, JournalCapacityError } from "./journal-objects.js"; +import * as journalStore from "./journal-store.js"; +import { reviveJournalError, type JournalStatement, type JournalStoreOptions } from "./journal-store.js"; import type { JournalSnapshotEntry } from "./journal-snapshot.js"; import { createNativeJournalCapture, type NativeJournalPolicy } from "./native-journal-capture.js"; import type { NativeJournalManager } from "./native-journal-observer.js"; @@ -15,7 +19,7 @@ import { constants, lstatSync, realpathSync } from "node:fs"; import { open, unlink } from "node:fs/promises"; import { join } from "node:path"; -import { queryGovernanceEvents, recordGovernanceEvent } from "../governance/events.js"; +import { queryGovernanceEvents } from "../governance/events.js"; const RETENTION_MS = 30 * 24 * 60 * 60 * 1000; /** @@ -39,13 +43,14 @@ const storageError = (error: unknown): Error => error instanceof JournalCapacity ? error : new JournalStorageUnavailable(error instanceof Error ? error.message : String(error)); type TurnRow = { turn_id: string; session_id: string | null; actor: "interactive" | "voice" | "user"; kind: "turn" | "revert"; source_turn_id: string | null; model: string | null; profile: "strict" | "governed"; policy_hash: string | null; started_at: number; ended_at: number | null; state: TurnJournalSummary["state"]; summary: string | null; missing: string; steering_inventory_missing: string; failure: string }; type ChangeRow = { sequence: number; path: string; scope: string; before_json: string | null; after_json: string | null; class: "content" | "steering" | "policy"; actor: "agent" | "user"; taint_source_event_id: number | null; conflict_observations: string; overlaps: string }; +const statement = (sql: string, ...params: SQLQueryBindings[]): JournalStatement => ({ sql, params }); const version = (entry: JournalSnapshotEntry | undefined): JournalVersion | null => entry ? { kind: entry.kind, mode: entry.mode, bytes: entry.bytes, ignored: entry.ignored, ...(entry.hash ? { hash: entry.hash } : {}), ...(entry.target !== undefined ? { target: entry.target } : {}), } : null; const identity = (entry: JournalSnapshotEntry | undefined) => JSON.stringify(entry ? { ...version(entry), ...(entry.ignored ? { stamp: entry.stamp } : {}) } : null); -export function createTurnJournal(db: Database, options: { +export interface TurnJournalOptions { brainPath: string; dbPath: string; objectsRoot: string; capBytes?: number; now?: () => number; nativeManager: NativeJournalManager; /** Managed worktrees (#2430): each is its own separately checkpointed root. */ @@ -53,26 +58,57 @@ export function createTurnJournal(db: Database, options: { /** Owner records for worktree observation checkpoints, so startup recovery releases only an ended process's. */ referenceCheckpoints?: ReferenceWorktreeCheckpointLedger; changed?: (turnId: string, sessionId: string, summary: TurnJournalSummary) => void; -}) { + /** @internal The app's write worker; absent, the journal opens its own for `db`'s file. */ + writer?: SqliteWriter; +} + +export function createTurnJournal(db: Database, options: TurnJournalOptions) { const now = options.now ?? Date.now; - const accounting = createJournalAccounting(db, options.dbPath, { capBytes: options.capBytes }); - const objects = createJournalObjects(options.objectsRoot, accounting); - accounting.recover(objects); - replaySteeringEvents(db); - for (const row of db.query("SELECT turn_id,pid,pid_start FROM turn_journal_turns WHERE ended_at IS NULL").all() as { turn_id: string; pid: number; pid_start: string }[]) { - if (!processAlive(row.pid, row.pid_start)) accounting.write(() => db.query("UPDATE turn_journal_turns SET state='partial',ended_at=?,missing=? WHERE turn_id=?").run(now(), JSON.stringify([{ path: ".", reason: "interrupted_observation" }]), row.turn_id)); + // Every journal write is an accounted transaction on the write worker's + // connection, committed at synchronous=FULL off the event loop and in + // submission order (#2814). This connection only reads; its own guard + // refuses any journal row it would change. + const writer = options.writer ?? openSqliteWriter(db, { attach: [{ schema: JOURNAL_SCHEMA, path: journalDatabasePath(options.dbPath) }] }); + const ownWriter = !options.writer; + const ledger = writer.channel(journalStore, { dbPath: options.dbPath, objectsRoot: options.objectsRoot, ...(options.capBytes !== undefined ? { capBytes: options.capBytes } : {}) }, reviveJournalError); + if (writer.offloop) installEscapeGuard(db); + /** Startup recovery commits before any later write; `ready` reports its refusal. */ + const ready = (async () => { + await ledger.ready; + await ledger.call("recover"); + // Governance events live in the UI database; a commit split between the + // two files is completed from the journal's rows (#2789). + replaySteeringEvents(db); + const interrupted = (db.query("SELECT turn_id,pid,pid_start FROM turn_journal_turns WHERE ended_at IS NULL").all() as { turn_id: string; pid: number; pid_start: string }[]) + .filter(row => !processAlive(row.pid, row.pid_start)); + for (const row of interrupted) await ledger.call("statements", [statement("UPDATE turn_journal_turns SET state='partial',ended_at=?,missing=? WHERE turn_id=?", now(), JSON.stringify([{ path: ".", reason: "interrupted_observation" }]), row.turn_id)]); + })(); + ready.catch(() => {}); + // Every later call passes through `ready` the same way, so calls reach the + // worker, and commit, in the order they were made. + const call = (name: string, ...args: unknown[]): Promise => ready.then(() => ledger.call(name, ...args)); + const write = (statements: JournalStatement[]) => call("statements", statements); + const objects = createJournalObjects(options.objectsRoot, { async reserve(bytes, hash) { + const id = await call("reserve", bytes, hash); + return { id, finish: (actual: string, size: number, duplicate: boolean) => call("finishReservation", id, actual, size, duplicate), cancel: () => call("cancelReservation", id) }; + } }); + /** Writes no admission awaits; `idle()` waits for them too. */ + const unserialized = new Set>(); + function tracked(work: Promise): Promise { + unserialized.add(work); void work.finally(() => unserialized.delete(work)).catch(() => {}); + return work; } - const active = new Map void; sequence: number }>(); + /** Active turns; `sessionId` is the binding their record holds once earlier writes commit. */ + const active = new Map void; sequence: number; sessionId: string | null }>(); const retainVersion = async (source: number, bytes: number, signal?: AbortSignal) => { - const hash = await retainAsync(() => objects.putFileAsync(source, bytes, signal)); + const hash = await retain(() => objects.putFileAsync(source, bytes, signal)); // Pin the shared observation before another admission can collect objects. - retain(() => accounting.write(() => { - for (const turnId of active.keys()) db.query("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)").run(turnId, hash); - })); + await retain(() => write([...active.keys()].map(turnId => statement("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)", turnId, hash)))); return hash; }; const readIdentity = { brainPath: options.brainPath, async createTemporary(maximum: number) { - const reservation = await retainAsync(async () => accounting.reserveTemporary(maximum)); + const id = await retain(() => call("reserveTemporary", maximum)); + const reservation = { id, release: () => call("releaseTemporary", id) }; const path = join(options.objectsRoot, `pending-${reservation.id}`); let file: Awaited> | undefined; const discard = async () => { @@ -89,7 +125,7 @@ export function createTurnJournal(db: Database, options: { await file.close(); await discard(); } - reservation.release(); throw error; + await reservation.release(); throw error; } } }; const capture = createNativeJournalCapture(options.nativeManager, retainVersion, {}, readIdentity); @@ -108,10 +144,12 @@ export function createTurnJournal(db: Database, options: { tail = work.catch(() => {}); return work; } - async function retainAsync(operation: () => Promise): Promise { + /** A capacity refusal collects the oldest ended turn and retries, until none is left. */ + async function retain(operation: () => Promise): Promise { + await ready; for (;;) { try { return await operation(); } - catch (error) { if (!(error instanceof JournalCapacityError) || !collectOne()) throw error; } + catch (error) { if (!(error instanceof JournalCapacityError) || !await collectOne()) throw error; } } } const periodicObservers = new Set<{ before(): void; after(completed: boolean): void }>(); @@ -142,18 +180,20 @@ export function createTurnJournal(db: Database, options: { } }); } - function fail(turnId: string, error: unknown) { + /** The turn is partial at once; its durable marker commits after. */ + async function fail(turnId: string, error: unknown) { const reason = error instanceof JournalCapacityError ? "journal_capacity" : "journal_unavailable"; failures.set(turnId, reason); // Admission preallocates this fixed-size terminal slot. It never needs a // larger row when an observation has exhausted available capacity. - try { accounting.write(() => db.query("UPDATE turn_journal_turns SET state='partial',failure=? WHERE turn_id=?").run(reason.padEnd(64), turnId)); } + try { await write([statement("UPDATE turn_journal_turns SET state='partial',failure=? WHERE turn_id=?", reason.padEnd(64), turnId)]); } catch { /* Busy storage remains recording on disk; restart marks it interrupted, never complete. */ } publish(turnId); } - function releaseReads() { - for (const id of pendingReadReleases) { accounting.releaseRead(id); pendingReadReleases.delete(id); } + async function releaseReads() { + for (const id of pendingReadReleases) { await call("releaseRead", id); pendingReadReleases.delete(id); } } + const releaseRead = (lease: string) => call("releaseRead", lease).catch(() => { pendingReadReleases.add(lease); }); function summary(turnId: string): TurnJournalSummary { const row = db.query("SELECT * FROM turn_journal_turns WHERE turn_id=?").get(turnId) as TurnRow | null; @@ -172,67 +212,57 @@ export function createTurnJournal(db: Database, options: { const row = db.query("SELECT session_id FROM turn_journal_turns WHERE turn_id=?").get(turnId) as { session_id: string | null } | null; if (row?.session_id) options.changed?.(turnId, row.session_id, summary(turnId)); } - function collectObjects() { - releaseReads(); - accounting.write(() => { - const unused = db.query("SELECT hash FROM turn_journal_objects WHERE state='present' AND NOT EXISTS(SELECT 1 FROM turn_journal_object_refs r WHERE r.hash=turn_journal_objects.hash) AND NOT EXISTS(SELECT 1 FROM turn_journal_reservations r WHERE r.hash=turn_journal_objects.hash)").all() as { hash: string }[]; - for (const row of unused) { - if (objects.has(row.hash)) objects.remove(row.hash); - db.query("DELETE FROM turn_journal_objects WHERE hash=?").run(row.hash); - } - }); + async function collectObjects() { + await releaseReads(); + await call("collectObjects"); } - function expire(turnId: string, removeSummary = false) { + async function expire(turnId: string, removeSummary = false) { const before = summary(turnId); - accounting.write(() => { - db.query("DELETE FROM turn_journal_changes WHERE turn_id=?").run(turnId); - db.query("DELETE FROM turn_journal_claude_memory WHERE turn_id=?").run(turnId); - db.query("DELETE FROM turn_journal_manifests WHERE turn_id=?").run(turnId); - db.query("DELETE FROM turn_journal_object_refs WHERE turn_id=?").run(turnId); - if (removeSummary) db.query("DELETE FROM turn_journal_turns WHERE turn_id=? AND ended_at IS NOT NULL").run(turnId); - else db.query("UPDATE turn_journal_turns SET state='expired',summary=? WHERE turn_id=? AND ended_at IS NOT NULL").run(JSON.stringify({ changedFiles: before.changedFiles, steeringFiles: before.steeringFiles, sharedAttribution: before.sharedAttribution }), turnId); - }); - failures.delete(turnId); collectObjects(); publish(turnId); + await write([ + statement("DELETE FROM turn_journal_changes WHERE turn_id=?", turnId), + statement("DELETE FROM turn_journal_claude_memory WHERE turn_id=?", turnId), + statement("DELETE FROM turn_journal_manifests WHERE turn_id=?", turnId), + statement("DELETE FROM turn_journal_object_refs WHERE turn_id=?", turnId), + removeSummary ? statement("DELETE FROM turn_journal_turns WHERE turn_id=? AND ended_at IS NOT NULL", turnId) + : statement("UPDATE turn_journal_turns SET state='expired',summary=? WHERE turn_id=? AND ended_at IS NOT NULL", JSON.stringify({ changedFiles: before.changedFiles, steeringFiles: before.steeringFiles, sharedAttribution: before.sharedAttribution }), turnId), + ]); + failures.delete(turnId); await collectObjects(); publish(turnId); } - function collectOne(): boolean { + async function collectOne(): Promise { const row = db.query("SELECT turn_id,state FROM turn_journal_turns WHERE ended_at IS NOT NULL ORDER BY (state='expired'),ended_at LIMIT 1").get() as { turn_id: string; state: string } | null; if (!row) return false; - expire(row.turn_id, row.state === "expired"); return true; + await expire(row.turn_id, row.state === "expired"); return true; } - function retain(operation: () => T): T { - for (;;) { - try { return operation(); } - catch (error) { if (!(error instanceof JournalCapacityError) || !collectOne()) throw error; } - } - } - function finalizeRefusedAdmission(turnId: string) { - retain(() => accounting.write(() => { + async function finalizeRefusedAdmission(turnId: string) { + await retain(() => write([ // The preallocated terminal slot shrinks more than ended_at grows. // A refused worker made no changes, so initial-only CAS pins are unneeded. - db.query("UPDATE turn_journal_turns SET state='partial',failure=?,ended_at=? WHERE turn_id=?").run(failures.get(turnId) ?? "journal_unavailable", now(), turnId); - db.query("DELETE FROM turn_journal_manifests WHERE turn_id=?").run(turnId); - db.query("DELETE FROM turn_journal_object_refs WHERE turn_id=?").run(turnId); - })); - failedAdmissions.delete(turnId); collectObjects(); publish(turnId); - } - function pin(turnId: string, entry: JournalVersion | null) { - if (entry?.hash) db.query("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)").run(turnId, entry.hash); + statement("UPDATE turn_journal_turns SET state='partial',failure=?,ended_at=? WHERE turn_id=?", failures.get(turnId) ?? "journal_unavailable", now(), turnId), + statement("DELETE FROM turn_journal_manifests WHERE turn_id=?", turnId), + statement("DELETE FROM turn_journal_object_refs WHERE turn_id=?", turnId), + ])); + failedAdmissions.delete(turnId); await collectObjects(); publish(turnId); } + const pin = (turnId: string, entry: JournalVersion | null): JournalStatement[] => + entry?.hash ? [statement("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)", turnId, entry.hash)] : []; async function observe(userPolicy: boolean = false): Promise { - releaseReads(); + await releaseReads(); const overlaps = [...active.keys()]; const changed = new Map(); - const stopActive = (error: unknown) => { + const stopActive = async (error: unknown) => { + const recorded: Promise[] = []; for (const [id, turn] of active) { // One failed notification must not leave another participant running. - try { fail(id, error); } catch { /* Existing partial slot remains authoritative. */ } + // Existing partial slot remains authoritative if the marker cannot be recorded. + recorded.push(fail(id, error).catch(() => {})); try { turn.stop(error instanceof Error ? error : new Error(String(error))); } catch { /* Continue stopping every participant. */ } } + await Promise.all(recorded); }; let observation: Awaited>; try { observation = await capture.observe(new Map([...active].map(([id, turn]) => [id, turn.policy]))); } - catch (error) { observationFailures++; const failure = storageError(error); stopActive(failure); throw failure; } + catch (error) { observationFailures++; const failure = storageError(error); await stopActive(failure); throw failure; } // Each managed worktree is observed through its own checkpoints, for the one turn that holds it. const separate: { worktreeId: string; turnId: string; observation: Awaited> }[] = []; for (const [worktreeId, root] of [...worktrees]) { @@ -243,8 +273,9 @@ export function createTurnJournal(db: Database, options: { // Only the holding turn depends on this root: it stops, and its record stays partial. worktrees.delete(worktreeId); void root.capture.close().catch(() => {}); const failure = storageError(error); - try { fail(root.turnId, failure); } catch { /* The partial slot remains authoritative. */ } + const recorded = fail(root.turnId, failure).catch(() => { /* The partial slot remains authoritative. */ }); try { holder.stop(failure); } catch { /* Continue with the other participants. */ } + await recorded; } } try { @@ -255,60 +286,14 @@ export function createTurnJournal(db: Database, options: { } const steering = observation.steering; for (const [turnId, turn] of active) { - const metadata = db.query("SELECT * FROM turn_journal_turns WHERE turn_id=?").get(turnId) as TurnRow; - const taint = db.query("SELECT id FROM governance_events WHERE kind='taint' AND (turn_id=? OR (? IS NOT NULL AND session_id=?)) ORDER BY id LIMIT 1") - .get(turnId, metadata.session_id, metadata.session_id) as { id: number } | null; - retain(() => accounting.write(() => { - for (const item of changed.values()) { - const sequence = ++turn.sequence; - const approvedPolicy = userPolicy && item.path === "context/policies/governance.md"; - const classification = approvedPolicy ? "policy" : steering.paths.has(item.path) ? "steering" : "content"; - const actor = approvedPolicy || metadata.kind === "revert" ? "user" : "agent"; - const source = classification === "steering" && actor === "agent" && taint ? taint.id : null; - const overlappingTurnIds = overlaps.filter(id => id !== turnId); - db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,?,?,?,?)") - .run(turnId, sequence, item.path, item.scope, item.before ? JSON.stringify(item.before) : null, item.after ? JSON.stringify(item.after) : null, classification, actor, JSON.stringify(overlappingTurnIds), source); - if (classification === "steering") recordGovernanceEvent(db, { kind: "steering_change", sessionId: metadata.session_id, turnId, - actor: actor === "user" ? "user" : metadata.actor, model: metadata.model, profile: metadata.profile, at: now(), dedupeKey: `${turnId}:${sequence}`, - payload: { path: item.path, sequence, beforeHash: item.before?.hash ?? null, afterHash: item.after?.hash ?? null, taintSourceEventId: source, overlappingTurnIds } }); - pin(turnId, item.before); pin(turnId, item.after); - const observed = db.query("SELECT sequence,after_json,overlaps FROM turn_journal_changes WHERE turn_id=? AND path=? ORDER BY sequence").all(turnId, item.path) as {sequence:number;after_json:string|null;overlaps:string}[]; - const participants = new Set(observed.flatMap(row => JSON.parse(row.overlaps) as string[])); - const conflicting = new Map(); - for (const other of participants) { - const shared = observed.filter(row => (JSON.parse(row.overlaps) as string[]).includes(other)); - if (new Set(shared.map(row => row.after_json ?? "null")).size < 2) continue; - for (const row of shared) conflicting.set(row.sequence, {turnId, sequence:row.sequence, side:"after"}); - } - if (conflicting.size) db.query("UPDATE turn_journal_changes SET conflict_observations=? WHERE turn_id=? AND path=?") - .run(JSON.stringify([...conflicting.values()].sort((a,b)=>a.sequence-b.sequence)),turnId,item.path); - } - // Intentional worktree edits are editable workspace content of their holding turn only. - for (const root of separate) if (root.turnId === turnId) for (const change of root.observation.changes.get(turnId) ?? []) { - if (identity(change.before) === identity(change.after)) continue; - const before = version(change.before), after = version(change.after), sequence = ++turn.sequence; - db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,?,?,?,?)") - .run(turnId, sequence, referenceWorktreeJournalPath(root.worktreeId, change.path), REFERENCE_WORKTREE_JOURNAL_SCOPE, before ? JSON.stringify(before) : null, - after ? JSON.stringify(after) : null, "content", metadata.kind === "revert" ? "user" : "agent", "[]", null); - pin(turnId, before); pin(turnId, after); - } - db.query("UPDATE turn_journal_turns SET steering_inventory_missing=? WHERE turn_id=?").run(JSON.stringify([...new Set([...(JSON.parse(metadata.steering_inventory_missing) as string[]), ...steering.unavailable])]), turnId); - const unreadable = [...observation.unreadable, ...separate.filter(root => root.turnId === turnId).flatMap(root => root.observation.unreadable - .map(item => ({ ...item, path: referenceWorktreeJournalPath(root.worktreeId, item.path === "." ? "" : item.path) })))]; - if (unreadable.length) { - const missing = new Map((JSON.parse(metadata.missing) as { path: string; reason: string }[]).map(item => [item.path, item])); - for (const item of unreadable) { - const { ms, ...known } = item; - missing.set(item.path, known); - const signature = `journal_reader:${item.kind ?? "unknown"}:${item.bytes ?? "unknown"}:${item.path}`.slice(0, 1000); - recordGovernanceEvent(db, { kind: "roadblock", sessionId: metadata.session_id, turnId, actor: metadata.actor, - model: metadata.model, profile: metadata.profile, at: now(), dedupeKey: `journal-reader:${turnId}:${item.path}`, - payload: { category: "permission_denied", family: "journal_reader", signature, - failures: [{ toolCallId: null, signature, category: "permission_denied" }], resolution: null, - remedy: "capability", waste: { tokens: null, costUsd: null, ms }, eventIds: [] } }); - } - db.query("UPDATE turn_journal_turns SET state='partial',missing=? WHERE turn_id=?").run(JSON.stringify([...missing.values()]), turnId); - } + const worktree = separate.filter(root => root.turnId === turnId).flatMap(root => (root.observation.changes.get(turnId) ?? []) + .filter(change => identity(change.before) !== identity(change.after)) + .map(change => ({ path: referenceWorktreeJournalPath(root.worktreeId, change.path), scope: REFERENCE_WORKTREE_JOURNAL_SCOPE, before: version(change.before), after: version(change.after) }))); + const unreadable = [...observation.unreadable, ...separate.filter(root => root.turnId === turnId).flatMap(root => root.observation.unreadable + .map(item => ({ ...item, path: referenceWorktreeJournalPath(root.worktreeId, item.path === "." ? "" : item.path) })))]; + turn.sequence = await retain(() => call("recordObservation", { + turnId, sequence: turn.sequence, at: now(), userPolicy, changed: [...changed.values()], steeringPaths: [...steering.paths], + overlaps, worktree, steeringUnavailable: [...steering.unavailable], unreadable, })); publish(turnId); // Ignored versions have metadata but no content hash: count their largest @@ -332,7 +317,7 @@ export function createTurnJournal(db: Database, options: { } catch (error) { observationFailures++; const failure = storageError(error); - stopActive(failure); + await stopActive(failure); await observation.abort().catch(() => {}); for (const root of separate) await root.observation.abort().catch(() => {}); throw failure; @@ -348,23 +333,21 @@ export function createTurnJournal(db: Database, options: { signal?.throwIfAborted(); try { await capture.initialize(signal); } catch (error) { if (signal?.aborted) throw error; throw storageError(error); } - for (const refused of failedAdmissions) finalizeRefusedAdmission(refused); + for (const refused of failedAdmissions) await finalizeRefusedAdmission(refused); await observe(); signal?.throwIfAborted(); const expired = db.query("SELECT turn_id FROM turn_journal_turns WHERE ended_at IS NOT NULL AND ended_at accounting.write(() => { - db.query("INSERT INTO turn_journal_turns (turn_id,session_id,actor,model,profile,policy_hash,pid,pid_start,started_at,state) VALUES (?,?,?,?,?,?,?,?,?,'recording') ON CONFLICT(turn_id) DO UPDATE SET started_at=excluded.started_at WHERE turn_journal_turns.kind='revert' AND turn_journal_turns.ended_at IS NULL") - .run(turnId, metadata.sessionId, metadata.actor, metadata.model, metadata.profile, metadata.policyHash, SELF.pid, SELF.start, now()); - })); + for (const row of expired) await expire(row.turn_id, true); + await retain(() => write([statement("INSERT INTO turn_journal_turns (turn_id,session_id,actor,model,profile,policy_hash,pid,pid_start,started_at,state) VALUES (?,?,?,?,?,?,?,?,?,'recording') ON CONFLICT(turn_id) DO UPDATE SET started_at=excluded.started_at WHERE turn_journal_turns.kind='revert' AND turn_journal_turns.ended_at IS NULL", + turnId, metadata.sessionId, metadata.actor, metadata.model, metadata.profile, metadata.policyHash, SELF.pid, SELF.start, now())])); try { const policy = await capture.policy(metadata.coveredPaths); signal?.throwIfAborted(); - active.set(turnId, { policy, stop, sequence: 0 }); publish(turnId); + active.set(turnId, { policy, stop, sequence: 0, sessionId: metadata.sessionId }); publish(turnId); if (!scopeTimer) { scopeTimer = setInterval(() => { void held(() => observe()).catch(error => { - for (const [id, turn] of active) { fail(id, error); turn.stop(error instanceof Error ? error : new Error(String(error))); } + for (const [id, turn] of active) { void fail(id, error).catch(() => {}); turn.stop(error instanceof Error ? error : new Error(String(error))); } }); }, 15_000); scopeTimer.unref?.(); @@ -372,19 +355,27 @@ export function createTurnJournal(db: Database, options: { } catch (error) { failures.set(turnId, error instanceof JournalCapacityError ? "journal_capacity" : "journal_unavailable"); failedAdmissions.add(turnId); - try { finalizeRefusedAdmission(turnId); } catch { /* finish/next admission retries without reviving the worker. */ } + try { await finalizeRefusedAdmission(turnId); } catch { /* finish/next admission retries without reviving the worker. */ } publish(turnId); throw error instanceof JournalCapacityError || error instanceof JournalStorageUnavailable ? error : new JournalStorageUnavailable("native journal admission could not be prepared"); } }, true); }, - bindSession(turnId: string, sessionId: string) { if (!active.has(turnId)) return; accounting.write(() => db.query("UPDATE turn_journal_turns SET session_id=? WHERE turn_id=?").run(sessionId, turnId)); publish(turnId); }, + async bindSession(turnId: string, sessionId: string): Promise { + const turn = active.get(turnId); + if (!turn) return; + turn.sessionId = sessionId; + await tracked(write([statement("UPDATE turn_journal_turns SET session_id=? WHERE turn_id=?", sessionId, turnId)])); publish(turnId); + }, + /** + * Decided at once from the active turn's binding, so the caller adopts the + * model before its next event; the record commits after every earlier write. + */ bindModel(turnId: string, sessionId: string | null, model: string): boolean { - if (!model || !active.has(turnId)) return false; - const row = db.query("SELECT session_id,ended_at FROM turn_journal_turns WHERE turn_id=?").get(turnId) as { session_id: string | null; ended_at: number | null } | null; - if (!row || row.ended_at !== null || row.session_id !== sessionId) return false; - accounting.write(() => db.query("UPDATE turn_journal_turns SET model=? WHERE turn_id=? AND ended_at IS NULL").run(model, turnId)); - publish(turnId); return true; + const turn = active.get(turnId); + if (!model || !turn || turn.sessionId !== sessionId) return false; + void tracked(call("bindModel", turnId, sessionId, model)).then(bound => { if (bound) publish(turnId); }, () => {}); + return true; }, observe(): Promise { return held(() => observe()); }, /** @@ -407,33 +398,39 @@ export function createTurnJournal(db: Database, options: { /** Worktrees an active turn's journal observes. */ worktrees(turnId: string): string[] { return [...worktrees].filter(([, root]) => root.turnId === turnId).map(([id]) => id); }, isHeld(): boolean { return holds > 0; }, - async idle(): Promise { let waiting: Promise; do { waiting = tail; await waiting; } while (waiting !== tail); }, + async idle(): Promise { + let waiting: Promise; + do { waiting = tail; await waiting; } while (waiting !== tail); + await Promise.allSettled([...unserialized]); + }, + /** Settles once startup recovery has run; rejects with its refusal. */ + ready, /** @internal Only the authenticated history service selects generated sibling coverage. */ coverPaths(turnId: string, paths: readonly string[]): void { const turn=active.get(turnId); if(!turn)throw new JournalStorageUnavailable("history coverage requires an active observation window"); turn.policy.cover(paths); }, /** The caller retains approval/current-principal authority; this never revives a completed turn. */ - observeUserPolicy(turnId: string, approvedWrite: () => void): Promise { return held(async () => { + observeUserPolicy(turnId: string, approvedWrite: () => void | Promise): Promise { return held(async () => { if (!active.has(turnId) || summary(turnId).state !== "recording") throw new JournalStorageUnavailable("approved policy write requires an active complete observation window"); await observe(); if (summary(turnId).state !== "recording") throw new JournalStorageUnavailable("policy observation became incomplete before the approved write"); - try { approvedWrite(); } finally { await observe(true); } + try { await approvedWrite(); } finally { await observe(true); } }); }, finish(turnId: string) { return held(async () => { - if (!active.has(turnId)) { if (failedAdmissions.has(turnId)) finalizeRefusedAdmission(turnId); return; } + if (!active.has(turnId)) { if (failedAdmissions.has(turnId)) await finalizeRefusedAdmission(turnId); return; } try { await observe(); } finally { active.delete(turnId); await dropWorktrees(turnId); if (!active.size) { clearInterval(scopeTimer); scopeTimer = undefined; } - accounting.write(() => { - if (failures.has(turnId)) db.query("UPDATE turn_journal_turns SET state='partial',failure=? WHERE turn_id=?").run(failures.get(turnId)!.padEnd(64), turnId); - db.query("UPDATE turn_journal_turns SET ended_at=?,failure=trim(failure),state=CASE WHEN state='partial' THEN state ELSE 'complete' END WHERE turn_id=?").run(now(), turnId); - db.query("DELETE FROM turn_journal_manifests WHERE turn_id=?").run(turnId); - db.query("DELETE FROM turn_journal_object_refs WHERE turn_id=? AND NOT EXISTS (SELECT 1 FROM turn_journal_changes c WHERE c.turn_id=? AND (json_extract(c.before_json,'$.hash')=turn_journal_object_refs.hash OR json_extract(c.after_json,'$.hash')=turn_journal_object_refs.hash)) AND NOT EXISTS (SELECT 1 FROM turn_journal_claude_memory m WHERE m.turn_id=? AND m.state!='saved' AND (json_extract(m.before_json,'$.hash')=turn_journal_object_refs.hash OR json_extract(m.after_json,'$.hash')=turn_journal_object_refs.hash))").run(turnId, turnId, turnId); - }); - collectObjects(); publish(turnId); + await write([ + ...(failures.has(turnId) ? [statement("UPDATE turn_journal_turns SET state='partial',failure=? WHERE turn_id=?", failures.get(turnId)!.padEnd(64), turnId)] : []), + statement("UPDATE turn_journal_turns SET ended_at=?,failure=trim(failure),state=CASE WHEN state='partial' THEN state ELSE 'complete' END WHERE turn_id=?", now(), turnId), + statement("DELETE FROM turn_journal_manifests WHERE turn_id=?", turnId), + statement("DELETE FROM turn_journal_object_refs WHERE turn_id=? AND NOT EXISTS (SELECT 1 FROM turn_journal_changes c WHERE c.turn_id=? AND (json_extract(c.before_json,'$.hash')=turn_journal_object_refs.hash OR json_extract(c.after_json,'$.hash')=turn_journal_object_refs.hash)) AND NOT EXISTS (SELECT 1 FROM turn_journal_claude_memory m WHERE m.turn_id=? AND m.state!='saved' AND (json_extract(m.before_json,'$.hash')=turn_journal_object_refs.hash OR json_extract(m.after_json,'$.hash')=turn_journal_object_refs.hash))", turnId, turnId, turnId), + ]); + await collectObjects(); publish(turnId); } }); }, /** Ordinary current coverage only; never adds selected paths to ignored content. */ @@ -449,7 +446,7 @@ export function createTurnJournal(db: Database, options: { if (!turn || summary(turnId).state !== "recording") throw new JournalStorageUnavailable("A complete active journal is required before removal."); const before = version(await capture.retainBefore(path, turn.policy, signal)); if (!before || before.ignored || before.kind !== "file" || !before.hash) return null; - accounting.write(() => pin(turnId, before)); + await write(pin(turnId, before)); if (!objects.verify(before.hash, before.bytes)) throw new JournalStorageUnavailable("The retained original could not be verified."); return before; }); }, @@ -458,8 +455,11 @@ export function createTurnJournal(db: Database, options: { if (!db.query("SELECT 1 FROM turn_journal_object_refs WHERE turn_id=? AND hash=?").get(turnId, selected.hash)) return false; try { return objects.verify(selected.hash, selected.bytes); } catch { return false; } }, - /** Validated copy-out is the only native source of an external destination. */ - persistClaudeMemory(turnId: string, change: { directory: string; note: string; before: Uint8Array | null; beforeMode: number; after: Uint8Array } | null, save: () => void): void { + /** + * Validated copy-out is the only native source of an external destination. + * Every retained object and record commits before `save()` runs. + */ + async persistClaudeMemory(turnId: string, change: { directory: string; note: string; before: Uint8Array | null; beforeMode: number; after: Uint8Array } | null, save: () => void): Promise { if (!active.has(turnId)) throw new JournalStorageUnavailable("memory persistence requires an active turn"); const path = change ? `claude-memory://${change.note}` : "claude-memory://"; try { @@ -467,31 +467,26 @@ export function createTurnJournal(db: Database, options: { if (!change.note.endsWith(".md") || /[\\/\u0000-\u001f\u007f]/u.test(change.note) || [".", ".."].includes(change.note)) throw new Error("invalid memory note"); const directory = realpathSync(change.directory), info = lstatSync(directory); if (directory !== change.directory || !info.isDirectory() || directory === options.brainPath || directory.startsWith(`${options.brainPath}/`)) throw new Error("invalid external memory store"); - const retained = (bytes: Uint8Array | null, mode: number): JournalVersion | null => bytes === null ? null : { - kind: "file", mode, bytes: bytes.length, hash: retain(() => objects.putBytes(bytes)), ignored: false, + const retained = async (bytes: Uint8Array | null, mode: number): Promise => bytes === null ? null : { + kind: "file", mode, bytes: bytes.length, hash: await retain(() => objects.putBytes(bytes)), ignored: false, }; - const before = retained(change.before, change.beforeMode); + const before = await retained(change.before, change.beforeMode); // Pin the first object before a second admission can collect under pressure. - retain(() => accounting.write(() => pin(turnId, before))); - const after = retained(change.after, 0o600); - retain(() => accounting.write(() => pin(turnId, after))); - retain(() => accounting.write(() => db.query("INSERT OR REPLACE INTO turn_journal_claude_memory VALUES (?,?,?,?,?,?,?,?,?)") - .run(turnId, path, directory, String(info.dev), String(info.ino), change.note, before ? JSON.stringify(before) : null, JSON.stringify(after), "pending"))); + await retain(() => write(pin(turnId, before))); + const after = await retained(change.after, 0o600); + await retain(() => write(pin(turnId, after))); + await retain(() => write([statement("INSERT OR REPLACE INTO turn_journal_claude_memory VALUES (?,?,?,?,?,?,?,?,?)", + turnId, path, directory, String(info.dev), String(info.ino), change.note, before ? JSON.stringify(before) : null, JSON.stringify(after), "pending")])); save(); - this.recordClaudeMemory(turnId, path, { directory, device: String(info.dev), inode: String(info.ino), note: change.note }, before, after); + await this.recordClaudeMemory(turnId, path, { directory, device: String(info.dev), inode: String(info.ino), note: change.note }, before, after); } catch (error) { // A note-specific refusal does not erase other successfully retained // changes. If even this bounded missing receipt cannot fit, use the // admission's preallocated partial terminal slot instead. try { - retain(() => accounting.write(() => { - const row = db.query("SELECT missing FROM turn_journal_turns WHERE turn_id=?").get(turnId) as {missing:string} | null; - if (!row) throw error; - const missing = (JSON.parse(row.missing) as {path:string;reason:string}[]).filter(item => item.path !== path); - missing.push({path,reason:"memory_persistence_unconfirmed"}); - db.query("UPDATE turn_journal_turns SET state='partial',missing=? WHERE turn_id=?").run(JSON.stringify(missing),turnId); - })); publish(turnId); - } catch { fail(turnId, error); } + if (!await retain(() => call("markMemoryUnconfirmed", turnId, path))) throw error; + publish(turnId); + } catch { await fail(turnId, error); } if (change) throw error; } }, @@ -500,55 +495,43 @@ export function createTurnJournal(db: Database, options: { return db.query("SELECT directory,device,inode,note FROM turn_journal_claude_memory WHERE turn_id=? AND path=?").get(turnId, path) as { directory: string; device: string; inode: string; note: string } | null; }, /** @internal Record a successful native save or authenticated recovery with existing retained objects. */ - recordClaudeMemory(turnId: string, path: string, target: { directory: string; device: string; inode: string; note: string }, before: JournalVersion | null, after: JournalVersion | null) { + async recordClaudeMemory(turnId: string, path: string, target: { directory: string; device: string; inode: string; note: string }, before: JournalVersion | null, after: JournalVersion | null): Promise { const turn = active.get(turnId); if (!turn) throw new JournalStorageUnavailable("memory observation requires an active turn"); - const metadata = db.query("SELECT * FROM turn_journal_turns WHERE turn_id=?").get(turnId) as TurnRow; - const actor = metadata.actor === "user" ? "user" : "agent"; - const taint = actor === "agent" ? db.query("SELECT id FROM governance_events WHERE kind='taint' AND (turn_id=? OR (? IS NOT NULL AND session_id=?)) ORDER BY id LIMIT 1").get(turnId, metadata.session_id, metadata.session_id) as { id: number } | null : null; const sequence = ++turn.sequence; - retain(() => accounting.write(() => { - db.query("INSERT INTO turn_journal_changes (turn_id,sequence,path,scope,before_json,after_json,class,actor,overlaps,taint_source_event_id) VALUES (?,?,?,?,?,?,'steering',?,'[]',?)") - .run(turnId, sequence, path, "claude_memory", before ? JSON.stringify(before) : null, after ? JSON.stringify(after) : null, actor, taint?.id ?? null); - db.query("INSERT OR REPLACE INTO turn_journal_claude_memory VALUES (?,?,?,?,?,?,?,?,?)").run(turnId, path, target.directory, target.device, target.inode, target.note, before ? JSON.stringify(before) : null, after ? JSON.stringify(after) : null, "saved"); - pin(turnId, before); pin(turnId, after); - recordGovernanceEvent(db, { kind: "steering_change", sessionId: metadata.session_id, turnId, actor: metadata.actor, model: metadata.model, profile: metadata.profile, at: now(), dedupeKey: `${turnId}:${sequence}`, - payload: { path, sequence, beforeHash: before?.hash ?? null, afterHash: after?.hash ?? null, taintSourceEventId: taint?.id ?? null, overlappingTurnIds: [] } }); - })); + await retain(() => call("recordClaudeMemory", turnId, sequence, path, target, before, after, now())); publish(turnId); }, - /** All operation metadata uses the journal's existing physical allocation owner. */ - writeOperation(write: () => T): T { return accounting.write(write); }, - reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]) { + /** All operation metadata uses the journal's existing physical allocation owner: one accounted write of these statements. */ + writeOperation(statements: readonly JournalStatement[]): Promise { return write([...statements]); }, + async reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]): Promise { const source = db.query("SELECT * FROM turn_journal_turns WHERE turn_id=? AND ended_at IS NOT NULL AND state IN ('complete','partial')").get(sourceTurnId) as TurnRow | null; if (!source) throw new JournalStorageUnavailable("source journal is unavailable"); - accounting.write(() => { - db.query("INSERT INTO turn_journal_turns (turn_id,session_id,actor,model,profile,policy_hash,pid,pid_start,started_at,state,kind,source_turn_id) VALUES (?,?,'user',NULL,?,?,?,?,?,'recording','revert',?)") - .run(turnId, source.session_id, source.profile, source.policy_hash, SELF.pid, SELF.start, now(), sourceTurnId); - for (const entry of versions) pin(turnId, entry); - }); + await write([ + statement("INSERT INTO turn_journal_turns (turn_id,session_id,actor,model,profile,policy_hash,pid,pid_start,started_at,state,kind,source_turn_id) VALUES (?,?,'user',NULL,?,?,?,?,?,'recording','revert',?)", + turnId, source.session_id, source.profile, source.policy_hash, SELF.pid, SELF.start, now(), sourceTurnId), + ...versions.flatMap(entry => pin(turnId, entry)), + ]); }, - finishReserved(turnId: string) { + async finishReserved(turnId: string): Promise { if (active.has(turnId)) throw new Error("active revert must finish its observation"); - accounting.write(() => { - db.query("UPDATE turn_journal_turns SET ended_at=?,state='partial',failure='revert_not_observed' WHERE turn_id=? AND ended_at IS NULL").run(now(), turnId); - db.query("DELETE FROM turn_journal_object_refs WHERE turn_id=?").run(turnId); - }); collectObjects(); + await write([ + statement("UPDATE turn_journal_turns SET ended_at=?,state='partial',failure='revert_not_observed' WHERE turn_id=? AND ended_at IS NULL", now(), turnId), + statement("DELETE FROM turn_journal_object_refs WHERE turn_id=?", turnId), + ]); await collectObjects(); }, /** Acquire the read lease while the selected source still owns its original. */ - withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): T { + async withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): Promise { if (!selected.hash || selected.ignored || selected.kind !== "file") throw new JournalStorageUnavailable("retained original is unavailable"); - const lease = accounting.openRead(selected.hash, () => { - if (!db.query("SELECT 1 FROM turn_journal_object_refs r JOIN turn_journal_turns t ON t.turn_id=r.turn_id WHERE r.turn_id=? AND r.hash=? AND t.state IN ('complete','partial') AND t.ended_at IS NOT NULL").get(turnId, selected.hash!)) throw new JournalStorageUnavailable("original retention ownership expired before admission"); - }); - try { return objects.withVerifiedCopy(selected.hash, selected.bytes, apply); } - finally { try { accounting.releaseRead(lease); } catch { pendingReadReleases.add(lease); } } + const lease = await call("openRead", selected.hash, turnId); + try { return await objects.withVerifiedCopy(selected.hash, selected.bytes, apply); } + finally { await releaseRead(lease); } }, - withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): T { + async withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): Promise { if (!selected || selected.kind === "symlink") return apply(() => {}); if (!selected.hash || selected.ignored || selected.kind !== "file") throw new JournalStorageUnavailable("retained version is unavailable"); - const lease = accounting.openRead(selected.hash); - try { return objects.withVerifiedCopy(selected.hash, selected.bytes, apply); } - finally { try { accounting.releaseRead(lease); } catch { pendingReadReleases.add(lease); } } + const lease = await call("openRead", selected.hash); + try { return await objects.withVerifiedCopy(selected.hash, selected.bytes, apply); } + finally { await releaseRead(lease); } }, summary, details(sessionId: string, turnId: string): TurnJournalDetails | null { @@ -569,13 +552,17 @@ export function createTurnJournal(db: Database, options: { const details = this.details(sessionId, turnId), change = details?.changes.find(item => item.sequence === sequence); const selected = change?.[side]; if (!selected?.hash || details?.summary.state === "expired") return null; - return { selected, bytes: (maxBytes: number) => objects.read(selected.hash!, maxBytes), stream() { - const lease = accounting.openRead(selected.hash!); - const release = () => { try { accounting.releaseRead(lease); } catch { pendingReadReleases.add(lease); } }; + return { selected, bytes: (maxBytes: number) => objects.read(selected.hash!, maxBytes), async stream() { + const lease = await call("openRead", selected.hash!); + const release = () => releaseRead(lease); try { return objects.stream(selected.hash!, release); } catch (error) { release(); throw error; } } }; }, - async close() { clearInterval(scopeTimer); try { await tail; await dropWorktrees(); await capture.close(); } finally { objects.close(); } }, + async close() { + clearInterval(scopeTimer); + try { await tail; await Promise.allSettled([...unserialized]); await dropWorktrees(); await capture.close(); } + finally { objects.close(); await ledger.close(); if (ownWriter) await writer.close(); } + }, }; } diff --git a/packages/ui-server/src/brain/turn-revert.ts b/packages/ui-server/src/brain/turn-revert.ts index 46e1dc99fa..6cd5a76d07 100644 --- a/packages/ui-server/src/brain/turn-revert.ts +++ b/packages/ui-server/src/brain/turn-revert.ts @@ -6,6 +6,7 @@ import type { JournalVersion, TurnRevertPlan, TurnRevertReceipt, TurnRevertPath, import { BRAIN_LOCK_KEY, type KeyedLock } from "@schlessera/brain-ui-sdk/server"; import { GIT_LOCK_KEY } from "@schlessera/brain-ui-sdk/internal"; import type { TurnJournal } from "./turn-journal.js"; +import { journalStatement } from "./journal-store.js"; import type { WriterGate } from "./writer-gate.js"; import { processAlive, SELF } from "./writer-gate.js"; import { reversibleJournalPath, restoreJournalPath, createJournalSibling } from "./revert-files.js"; @@ -74,7 +75,8 @@ export function createTurnReverts(db: Database, options: { function operation(id: string, principalId: string, sessionId: string, sourceTurnId: string): Operation | null { const row = db.query("SELECT * FROM turn_journal_revert_operations WHERE id=? AND principal_id=? AND session_id=? AND source_turn_id=?").get(id, principalId, sessionId, sourceTurnId) as Operation | null; if (row && ["waiting", "applying"].includes(row.state) && !processAlive(row.pid, row.pid_start)) { - journal.writeOperation(() => db.query("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=? WHERE id=?").run(row.state === "waiting" ? "failed" : "unknown", Date.now(), row.state === "waiting" ? "interrupted_before_write" : "interrupted_partial_work", id)); + // The repair is idempotent: a reader before it commits repairs again. + void journal.writeOperation([journalStatement("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=? WHERE id=?", row.state === "waiting" ? "failed" : "unknown", Date.now(), row.state === "waiting" ? "interrupted_before_write" : "interrupted_partial_work", id)]).catch(() => {}); row.error = row.state === "waiting" ? "interrupted_before_write" : "interrupted_partial_work"; row.state = row.state === "waiting" ? "failed" : "unknown"; } return row; @@ -97,7 +99,7 @@ export function createTurnReverts(db: Database, options: { const source = journal.summary(row.journal_turn_id); await journal.begin(row.journal_turn_id, { sessionId: row.session_id, actor: "user", model: null, profile: source.profile ?? "strict", policyHash: source.policyHash, coveredPaths: selected.filter(item => !journal.claudeMemoryTarget(row.source_turn_id, item.path)).map(item => item.path) }, error => { observationFailure = error; }); observed = true; - journal.writeOperation(() => db.query("UPDATE turn_journal_revert_operations SET state='applying' WHERE id=?").run(row.id)); + await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_operations SET state='applying' WHERE id=?", row.id)]); for (const path of selected) { if (path.state !== "eligible") continue; if (observationFailure || !options.isAuthorized(row.principal_id)) throw observationFailure ?? new RevertUnavailable("authorization_changed"); @@ -108,27 +110,27 @@ export function createTurnReverts(db: Database, options: { const info = lstatSync(target.directory); if (!info.isDirectory() || realpathSync(target.directory) !== target.directory || String(info.dev) !== target.device || String(info.ino) !== target.inode) state = "conflict"; else { - state = journal.withRetainedVersion(path.before, copy => restoreJournalPath(target.directory, target.note, path.after, path.before, copy)); - if (state === "restored") journal.recordClaudeMemory(row.journal_turn_id, path.path, target, path.after, path.before); + state = await journal.withRetainedVersion(path.before, copy => restoreJournalPath(target.directory, target.note, path.after, path.before, copy)); + if (state === "restored") await journal.recordClaudeMemory(row.journal_turn_id, path.path, target, path.after, path.before); } - } else state = journal.withRetainedVersion(path.before, copy => restoreJournalPath(options.brainPath, path.path, path.after, path.before, copy)); + } else state = await journal.withRetainedVersion(path.before, copy => restoreJournalPath(options.brainPath, path.path, path.after, path.before, copy)); code = state === "conflict" ? "changed_since_observation" : ""; } catch { state = "failed"; code = "restoration_unavailable"; } - journal.writeOperation(() => db.query("UPDATE turn_journal_revert_paths SET state=?,code=? WHERE operation_id=? AND path=?").run(state, code.padEnd(64), row.id, path.path)); + await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_paths SET state=?,code=? WHERE operation_id=? AND path=?", state, code.padEnd(64), row.id, path.path)]); path.state = state; path.reason = code || null; await journal.observe(); if(state === "restored" && path.other) { let sibling:Selected|undefined; try { - journal.withRetainedVersion(path.other, copy=>createJournalSibling(options.brainPath,path.path,path.other!,copy, - candidate=>{ + await journal.withRetainedVersion(path.other, copy=>createJournalSibling(options.brainPath,path.path,path.other!,copy, + async candidate=>{ sibling={path:candidate,before:path.other!,after:null,beforeSequence:path.otherSequence!,afterSequence:0,state:"eligible",reason:null,quarantined:false}; - journal.writeOperation(()=>db.query("INSERT INTO turn_journal_revert_paths (operation_id,path,before_json,after_json,before_sequence,after_sequence,state,code) VALUES (?,?,?,NULL,?,0,'eligible',?)").run(row.id,candidate,JSON.stringify(path.other),path.otherSequence!,"".padEnd(64))); + await journal.writeOperation([journalStatement("INSERT INTO turn_journal_revert_paths (operation_id,path,before_json,after_json,before_sequence,after_sequence,state,code) VALUES (?,?,?,NULL,?,0,'eligible',?)",row.id,candidate,JSON.stringify(path.other),path.otherSequence!,"".padEnd(64))]); journal.coverPaths(row.journal_turn_id,[candidate]); - }, candidate=>{journal.writeOperation(()=>db.query("DELETE FROM turn_journal_revert_paths WHERE operation_id=? AND path=?").run(row.id,candidate));sibling=undefined;})); + }, async candidate=>{await journal.writeOperation([journalStatement("DELETE FROM turn_journal_revert_paths WHERE operation_id=? AND path=?",row.id,candidate)]);sibling=undefined;})); sibling!.state="restored"; - } catch { if(sibling){sibling.state="failed";sibling.reason="sibling_creation_unavailable";}else{path.state="failed";path.reason="sibling_creation_unavailable";journal.writeOperation(()=>db.query("UPDATE turn_journal_revert_paths SET state='failed',code=? WHERE operation_id=? AND path=?").run("sibling_creation_unavailable".padEnd(64),row.id,path.path));} } - if(sibling){selected.push(sibling);journal.writeOperation(()=>db.query("UPDATE turn_journal_revert_paths SET state=?,code=? WHERE operation_id=? AND path=?").run(sibling!.state,(sibling!.reason??"").padEnd(64),row.id,sibling!.path));} + } catch { if(sibling){sibling.state="failed";sibling.reason="sibling_creation_unavailable";}else{path.state="failed";path.reason="sibling_creation_unavailable";await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_paths SET state='failed',code=? WHERE operation_id=? AND path=?","sibling_creation_unavailable".padEnd(64),row.id,path.path)]);} } + if(sibling){selected.push(sibling);await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_paths SET state=?,code=? WHERE operation_id=? AND path=?",sibling!.state,(sibling!.reason??"").padEnd(64),row.id,sibling!.path)]);} await journal.observe(); } } @@ -138,17 +140,16 @@ export function createTurnReverts(db: Database, options: { })); await journal.finish(row.journal_turn_id); const partial = selected.some(path => ["conflict", "failed", "excluded"].includes(path.state)) || journal.summary(row.journal_turn_id).state !== "complete"; - journal.writeOperation(() => db.query("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=trim(error) WHERE id=?").run(partial ? "partial" : "complete", Date.now(), row.id)); + await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=trim(error) WHERE id=?", partial ? "partial" : "complete", Date.now(), row.id)]); } catch (error) { const reason = (error as {reason?:string}).reason === "journal_capacity" ? "journal_capacity" : (error as {reason?:string}).reason === "journal_storage_unavailable" ? "journal_unavailable" : error instanceof RevertUnavailable ? "authorization_changed" : "operation_unavailable"; - try { if (observed) await journal.finish(row.journal_turn_id); else journal.finishReserved(row.journal_turn_id); } catch { /* An interrupted journal remains partial, never undoable by assumption. */ } - try { journal.writeOperation(() => db.query("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=? WHERE id=?").run(observed ? "unknown" : "failed", Date.now(), reason, row.id)); } catch { /* Readback will retain the original pending identity. */ } + try { if (observed) await journal.finish(row.journal_turn_id); else await journal.finishReserved(row.journal_turn_id); } catch { /* An interrupted journal remains partial, never undoable by assumption. */ } + try { await journal.writeOperation([journalStatement("UPDATE turn_journal_revert_operations SET state=?,completed_at=?,error=? WHERE id=?", observed ? "unknown" : "failed", Date.now(), reason, row.id)]); } catch { /* Readback will retain the original pending identity. */ } } finally { hold?.release(); if (selected.some(path => path.state === "restored" || path.state === "failed")) { try { options.refresh(); } catch { /* The ordinary index refresh may be retried; file effects remain recorded. */ } } } } - return { - plan(sessionId: string, turnId: string, action: "revert" | "restore" = "revert"): TurnRevertPlan { return { turnId, action, paths: select(sessionId, turnId, action).map(publicPath) }; }, - start(principalId: string, sessionId: string, sourceTurnId: string, request: TurnRevertRequest): TurnRevertReceipt { - if (shutdown.signal.aborted || !options.isAuthorized(principalId)) throw new RevertUnavailable("A current user authorization is required."); + let starting: Promise = Promise.resolve(); + async function begin(principalId: string, sessionId: string, sourceTurnId: string, request: TurnRevertRequest): Promise { + if (shutdown.signal.aborted || !options.isAuthorized(principalId)) throw new RevertUnavailable("A current user authorization is required."); if (!/^[a-zA-Z0-9_-]{16,96}$/.test(request.id)) throw new RevertUnavailable("Invalid operation identity."); const action = request.action ?? "revert"; if(!["revert","restore","keep_version","keep_both"].includes(action))throw new RevertUnavailable("Invalid history action."); @@ -163,16 +164,24 @@ export function createTurnReverts(db: Database, options: { const all = selection ? resolveSelection(sessionId,sourceTurnId,request) : select(sessionId,sourceTurnId,action as "revert"|"restore"), selected = request.path !== undefined ? all.filter(item=>item.path===request.path) : all; if (!selected.some(item=>item.state==="eligible")) throw new RevertUnavailable("No retained eligible paths are available."); const journalTurnId = `revert_${request.id}`; - journal.reserveRevert(journalTurnId,sourceTurnId,selected.flatMap(item=>[item.before,item.after,...(item.other?[item.other]:[])])); - try { journal.writeOperation(() => { - db.query("INSERT INTO turn_journal_revert_operations (id,principal_id,session_id,source_turn_id,journal_turn_id,fingerprint,action,state,pid,pid_start,requested_at,error,selection) VALUES (?,?,?,?,?,?,?,'waiting',?,?,?,?,?)") - .run(request.id,principalId,sessionId,sourceTurnId,journalTurnId,fingerprint,action,SELF.pid,SELF.start,Date.now(),"".padEnd(64),JSON.stringify(selection??{})); - for (const item of selected) db.query("INSERT INTO turn_journal_revert_paths (operation_id,path,before_json,after_json,before_sequence,after_sequence,state,code) VALUES (?,?,?,?,?,?,?,?)") - .run(request.id,item.path,item.before ? JSON.stringify(item.before) : null,item.after ? JSON.stringify(item.after) : null,item.beforeSequence,item.afterSequence,item.state,(item.reason ?? "").padEnd(64)); - }); } catch (error) { journal.finishReserved(journalTurnId); throw error; } + await journal.reserveRevert(journalTurnId,sourceTurnId,selected.flatMap(item=>[item.before,item.after,...(item.other?[item.other]:[])])); + try { await journal.writeOperation([ + journalStatement("INSERT INTO turn_journal_revert_operations (id,principal_id,session_id,source_turn_id,journal_turn_id,fingerprint,action,state,pid,pid_start,requested_at,error,selection) VALUES (?,?,?,?,?,?,?,'waiting',?,?,?,?,?)", + request.id,principalId,sessionId,sourceTurnId,journalTurnId,fingerprint,action,SELF.pid,SELF.start,Date.now(),"".padEnd(64),JSON.stringify(selection??{})), + ...selected.map(item => journalStatement("INSERT INTO turn_journal_revert_paths (operation_id,path,before_json,after_json,before_sequence,after_sequence,state,code) VALUES (?,?,?,?,?,?,?,?)", + request.id,item.path,item.before ? JSON.stringify(item.before) : null,item.after ? JSON.stringify(item.after) : null,item.beforeSequence,item.afterSequence,item.state,(item.reason ?? "").padEnd(64))), + ]); } catch (error) { await journal.finishReserved(journalTurnId); throw error; } const row = operation(request.id,principalId,sessionId,sourceTurnId)!; const work = execute(row,selected).finally(()=>running.delete(request.id)); running.set(request.id,work); return receipt(request.id,principalId,sessionId,sourceTurnId)!; + } + return { + plan(sessionId: string, turnId: string, action: "revert" | "restore" = "revert"): TurnRevertPlan { return { turnId, action, paths: select(sessionId, turnId, action).map(publicPath) }; }, + /** Starts run one at a time, so an identity is reserved once even across their commits. */ + start(principalId: string, sessionId: string, sourceTurnId: string, request: TurnRevertRequest): Promise { + const started = starting.then(() => begin(principalId, sessionId, sourceTurnId, request)); + starting = started.catch(() => {}); + return started; }, receipt, latest(principalId: string, sessionId: string, sourceTurnId: string) { diff --git a/packages/ui-server/src/cron/activity.ts b/packages/ui-server/src/cron/activity.ts index 0d471fd882..40fac6346c 100644 --- a/packages/ui-server/src/cron/activity.ts +++ b/packages/ui-server/src/cron/activity.ts @@ -1,6 +1,7 @@ import type { Database } from "bun:sqlite"; -import type { ActivityStore } from "../activity/store.js"; +import type { AwaitableActivityStore } from "../activity/store.js"; +import { observeWrite } from "../activity/store-writes.js"; import { resolveSystemPrincipal } from "../db/principals.js"; /** A cron run's root span in the activity store. */ @@ -15,14 +16,15 @@ export interface CronSpan { * wrapper (`run-job.ts`) and the in-process scheduler — so a status read * finds a run the same way whichever path ran it. */ -export function startCronSpan(db: Database, store: ActivityStore, name: string): CronSpan { +export function startCronSpan(db: Database, store: AwaitableActivityStore, name: string, failed: (error: unknown) => void = () => {}): CronSpan { const principal = resolveSystemPrincipal(db, { identity: "scheduled-jobs", label: "Scheduled jobs", }); const runId = `cron-${name}-${Date.now()}`; const rootSpanId = `${runId}:root`; - store.startSpan({ + // A store that commits off the event loop reports a failed write to `failed`. + observeWrite(store.startSpan({ spanId: rootSpanId, runId, name: `cron ${name}`, @@ -30,6 +32,6 @@ export function startCronSpan(db: Database, store: ActivityStore, name: string): origin: "cron", jobName: name, principalId: principal.id, - }); + }), failed); return { runId, rootSpanId }; } diff --git a/packages/ui-server/src/cron/scheduler.ts b/packages/ui-server/src/cron/scheduler.ts index 41be7fd036..12d3839c53 100644 --- a/packages/ui-server/src/cron/scheduler.ts +++ b/packages/ui-server/src/cron/scheduler.ts @@ -2,8 +2,9 @@ import type { Database } from "bun:sqlite"; import type { Logger } from "@opentelemetry/api-logs"; import { BrainSyncError, type BrainClient } from "../brain/client.js"; import { syncActivityAttrs } from "../brain/sync-result.js"; -import type { ActivityStore } from "../activity/store.js"; +import type { AwaitableActivityStore } from "../activity/store.js"; import { startCronSpan, type CronSpan } from "./activity.js"; +import { observeWrite } from "../activity/store-writes.js"; // Scheduling is owned by the container crontab (config/crontab) — sync at // 02:00, validate at 03:00, job scrape at 04:00. This module only provides @@ -77,7 +78,7 @@ export function createCronScheduler(deps: { * The activity store each run's root span goes into, as the container cron * wrapper writes it; absent records only `cron_runs`. */ - activity?: ActivityStore; + activity?: AwaitableActivityStore; }): CronScheduler { const { db, brain, log, activity } = deps; @@ -114,25 +115,29 @@ export function createCronScheduler(deps: { // Fail-open, as the wrapper's tracking is: the activity record never // stops the job it observes. let span: CronSpan | null = null; - try { - if (activity) span = startCronSpan(db, activity, job.name); - } catch (err) { + const unavailable = (err: unknown) => { log?.emit({ severityText: "WARN", body: "job activity unavailable", attributes: { job: job.name, error: err instanceof Error ? err.message : String(err) }, }); + }; + try { + if (activity) span = startCronSpan(db, activity, job.name, unavailable); + } catch (err) { + unavailable(err); } const attrs: Record = {}; const endSpan = (error?: string) => { if (!activity || !span) return; try { - activity.endSpan(span.rootSpanId, { + // Committed in this order; a failure stays unobserved, as before. + observeWrite(activity.endSpan(span.rootSpanId, { outcome: error === undefined ? "success" : "error", reason: error, attrs, - }); - activity.rollupRun(span.runId); + }), () => {}); + observeWrite(activity.rollupRun(span.runId), () => {}); } catch { // Observability must not break the observed. } diff --git a/packages/ui-server/src/db/write-worker-thread.ts b/packages/ui-server/src/db/write-worker-thread.ts new file mode 100644 index 0000000000..d85bb63dc6 --- /dev/null +++ b/packages/ui-server/src/db/write-worker-thread.ts @@ -0,0 +1,128 @@ +/** + * The write worker's thread (#2814): one write connection, opened at + * `synchronous=FULL`, running registered procedures one at a time in the + * order their calls arrived. Nothing here decides what is written. + */ +import { Database } from "bun:sqlite"; + +import { encodeError, WORKER_TURN, type WriteModule, type WriteProcedures } from "./write-worker.js"; + +declare const self: Worker; + + +const FULL = 2; +let db: Database | undefined; +/** Shared with the app connection: see `takeTurns` in write-worker.ts. */ +let turn: Int32Array | undefined; +/** Runs `work` while the app connection is outside any transaction; a waiting app transaction goes first. */ +function inTurn(work: () => T): T { + if (!turn) return work(); + while (Atomics.load(turn, 1) !== 0 || Atomics.compareExchange(turn, 0, 0, WORKER_TURN) !== 0) Atomics.wait(turn, 0, Atomics.load(turn, 0), 10); + try { return work(); } finally { Atomics.store(turn, 0, 0); Atomics.notify(turn, 0); } +} +const channels = new Map(); + +/** Every database file this connection writes: `main` and each attached schema, as last checked. */ +let schemas = ["main"]; + +/** + * A commit below FULL, in any of the connection's files, is refused before it + * happens: the transaction rolls back. The files are read each time, so one a + * procedure attaches itself (the journal ledger attaches `journal.db`) counts. + */ +function assertFull(connection: Database) { + schemas = (connection.query("PRAGMA database_list").all() as { name: string }[]).map(row => row.name).filter(name => name !== "temp"); + for (const schema of schemas) { + const level = (connection.query(`PRAGMA "${schema}".synchronous`).get() as { synchronous: number }).synchronous; + if (level !== FULL) throw new Error(`SQLITE_WRITE_BELOW_FULL: a write-worker commit would run at synchronous=${level} in ${schema}.`); + } +} + +function open(path: string, attach: { schema: string; path: string }[]): Database { + const connection = new Database(path, { readwrite: true, create: false }); + try { + connection.exec("PRAGMA busy_timeout = 5000"); + for (const database of attach) { + if (!/^[a-z_][a-z0-9_]*$/.test(database.schema) || ["main", "temp"].includes(database.schema)) throw new Error("Invalid attached schema name."); + connection.query(`ATTACH DATABASE ? AS "${database.schema}"`).run(database.path); + } + schemas = ["main", ...attach.map(database => database.schema)]; + for (const schema of schemas) { + connection.exec(`PRAGMA "${schema}".journal_mode = WAL`); + connection.exec(`PRAGMA "${schema}".synchronous = FULL`); + } + connection.exec("PRAGMA foreign_keys = ON"); + assertFull(connection); + // Every transaction checks its level last, just before COMMIT. + const transaction = connection.transaction.bind(connection); + connection.transaction = ((run: (...args: unknown[]) => unknown) => transaction(function (this: unknown, ...args: unknown[]) { + const value = run.apply(this, args); + assertFull(connection); + return value; + })) as Database["transaction"]; + } catch (error) { connection.close(); throw error; } + return connection; +} + +async function handle(message: { id: number; kind: string; [key: string]: unknown }): Promise { + switch (message.kind) { + case "open": + turn = message.turn as Int32Array | undefined; + db = inTurn(() => open(message.path as string, (message.attach as { schema: string; path: string }[] | undefined) ?? [])); + return undefined; + case "register": { + try { + if (!db) throw new Error("The SQLite write worker has no open database."); + const module = await import(message.module as string) as { default?: WriteModule } & Partial>; + const setup = module.setup ?? module.default?.setup; + if (typeof setup !== "function") throw new Error("A write procedure module must export setup()."); + const connection = db; + channels.set(message.channel as number, inTurn(() => setup(connection, message.options))); + } catch (error) { channels.set(message.channel as number, { failed: error }); throw error; } + return undefined; + } + case "call": { + const procedures = channels.get(message.channel as number); + if (!procedures) throw new Error("Unknown write channel."); + if ("failed" in procedures) throw procedures.failed; + const name = message.name as string, procedure = (procedures as WriteProcedures)[name]; + if (typeof procedure !== "function" || name === "close") throw new Error(`Unknown write procedure ${name}.`); + let value: unknown, failure: { error: unknown } | undefined; + try { value = inTurn(() => (procedure as (...args: unknown[]) => unknown)(...(message.args as unknown[]))); } + catch (error) { failure = { error }; } + // An autocommit statement has no transaction wrapper to check; a + // procedure that lowered the level fails and the next call is FULL again. + try { assertFull(db!); } catch (error) { if (!db!.inTransaction) for (const schema of schemas) db!.exec(`PRAGMA "${schema}".synchronous = FULL`); throw error; } + if (failure) throw failure.error; + if (value instanceof Promise) throw new Error(`Write procedure ${name} must complete synchronously.`); + if (db!.inTransaction) throw new Error(`Write procedure ${name} left a transaction open.`); + return value; + } + case "release": { + const procedures = channels.get(message.channel as number); + channels.delete(message.channel as number); + if (procedures && !("failed" in procedures)) procedures.close?.(); + return undefined; + } + case "idle": + return undefined; + case "close": + for (const procedures of channels.values()) if (!("failed" in procedures)) { try { procedures.close?.(); } catch { /* Closing continues. */ } } + channels.clear(); + db?.close(); db = undefined; + return undefined; + default: + throw new Error(`Unknown write worker message ${message.kind}.`); + } +} + +// One message at a time, in arrival order; a registration's import finishes +// before any later call starts. +let queue: Promise = Promise.resolve(); +self.onmessage = (event: MessageEvent<{ id: number; kind: string }>) => { + const message = event.data; + queue = queue.then(async () => { + try { postMessage({ id: message.id, ok: true, value: await handle(message) }); } + catch (error) { postMessage({ id: message.id, ok: false, error: encodeError(error) }); } + }); +}; diff --git a/packages/ui-server/src/db/write-worker.ts b/packages/ui-server/src/db/write-worker.ts new file mode 100644 index 0000000000..622bbb6b78 --- /dev/null +++ b/packages/ui-server/src/db/write-worker.ts @@ -0,0 +1,237 @@ +/** + * SQLite commits off the event loop (#2814). + * + * A writer owns one dedicated write connection to a database file, opened in + * a worker thread at `synchronous=FULL`. Each writing component registers a + * procedure module there and calls its procedures by name. The worker runs + * every call to completion, commit included, strictly in submission order, so + * the event loop awaits a commit's outcome instead of blocking in its fsync. + * Reads stay on the caller's own connection; WAL lets them proceed while the + * worker commits. + * + * An in-memory database has no file a second connection could open, so its + * writer runs the same procedures inline on the caller's connection, in the + * same order, behind the same asynchronous calls. + */ +import type { Database } from "bun:sqlite"; + +/** Every procedure is synchronous: it runs and commits before the next call starts. */ +export type WriteProcedures = Record unknown> & { close?: () => void }; + +/** + * A procedure module: `url` is its own `import.meta.url`, which the worker + * imports; `setup` runs once per channel on the write connection. + */ +export interface WriteModule { + url: string; + setup(db: Database, options: O): WriteProcedures; +} + +/** An error crossing the thread boundary: its class name and own fields. */ +export interface WireError { name: string; message: string; fields: Record } + +export interface WriteChannel { + /** Settles once the module's setup has run on the write connection. */ + readonly ready: Promise; + call(name: string, ...args: unknown[]): Promise; + close(): Promise; +} + +export interface SqliteWriter { + /** True when commits run in the worker thread, false for an inline in-memory writer. */ + readonly offloop: boolean; + channel(module: WriteModule, options: O, revive?: (error: WireError) => Error | undefined): WriteChannel; + /** Settles once every call submitted before it has settled. */ + idle(): Promise; + close(): Promise; +} + +export class SqliteWriterClosed extends Error { + constructor(message = "The SQLite write worker is closed.") { super(message); } +} + +export function encodeError(error: unknown): WireError { + if (!(error instanceof Error)) return { name: "Error", message: String(error), fields: {} }; + const fields: Record = {}; + for (const [key, value] of Object.entries(error)) if (["string", "number", "boolean"].includes(typeof value) || value === null) fields[key] = value; + return { name: error.constructor?.name || error.name, message: error.message, fields }; +} + +function decodeError(wire: WireError, revive?: (error: WireError) => Error | undefined): Error { + const error = revive?.(wire) ?? Object.assign(new Error(wire.message), { name: wire.name }); + for (const [key, value] of Object.entries(wire.fields)) { + if (!(key in error)) Object.defineProperty(error, key, { value, enumerable: true, writable: true, configurable: true }); + } + return error; +} + +const THREAD = new URL(`./write-worker-thread${import.meta.url.endsWith(".ts") ? ".ts" : ".js"}`, import.meta.url).href; + +/** A further database file the write connection owns, attached under `schema` (#2789's `journal.db`). */ +export interface AttachedDatabase { schema: string; path: string } + +/** + * Opens the writer for `db`'s file, or an inline writer when `db` is in + * memory. The worker connection attaches each `attach` file and commits it at + * FULL as well; an inline writer uses `db`'s own attachments. + */ +export function openSqliteWriter(db: Database, options: { attach?: readonly AttachedDatabase[] } = {}): SqliteWriter { + const path = db.filename; + if (!path || path === ":memory:") return inlineWriter(db); + const turn = new Int32Array(new SharedArrayBuffer(8)); + takeTurns(db, turn); + return workerWriter(path, options.attach ?? [], turn); +} + +/** `turn[0]`: who is writing now. `turn[1]`: app writes waiting for their turn. */ +const FREE = 0, APP = 1; +export const WORKER_TURN = 2; + +/** + * The app connection and the worker never write at the same time (#2814). + * Two connections in one process, both writing one WAL file, were measured failing an app `BEGIN IMMEDIATE` with SQLITE_BUSY after about + * 37 ms despite a 5 s busy timeout, and a deferred transaction that reads and + * then writes can lose its snapshot upgrade to a worker commit. So each + * outermost transaction on `db` waits until the worker is between procedures, + * then holds the turn until it ends, and so does every autocommit write; the + * worker waits for its turn the same way, and lets a waiting app write go + * first. A transaction keeps its own mode: a read-only deferred snapshot never + * takes the write lock. + */ +function takeTurns(db: Database, turn: Int32Array) { + // The app thread holds its turn once, however deeply its SQLite calls nest. + let depth = 0; + const taking = (work: (...args: A) => R) => (...args: A): R => { + if (depth > 0 || db.inTransaction) return work(...args); + Atomics.add(turn, 1, 1); + try { while (Atomics.compareExchange(turn, 0, FREE, APP) !== FREE) Atomics.wait(turn, 0, WORKER_TURN, 50); } + finally { Atomics.sub(turn, 1, 1); } + depth++; + try { return work(...args); } + finally { depth--; Atomics.store(turn, 0, FREE); Atomics.notify(turn, 0); } + }; + const transaction = db.transaction.bind(db); + db.transaction = ((run: (...args: never[]) => unknown) => { + const modes = transaction(run); + const mode = (call: (...args: never[]) => unknown) => taking((...args: unknown[]) => Reflect.apply(call, modes, args)); + return Object.assign(mode(modes), { deferred: mode(modes.deferred), immediate: mode(modes.immediate), exclusive: mode(modes.exclusive) }); + }) as Database["transaction"]; + // An autocommit write is a transaction of its own: it takes its turn too. + for (const name of ["run", "exec"] as const) { + const direct = (db[name] as (...args: unknown[]) => unknown).bind(db); + Object.defineProperty(db, name, { configurable: true, writable: true, value: taking(direct) }); + } + const statements = new WeakSet(); + for (const name of ["query", "prepare"] as const) { + const make = (db[name] as unknown as (sql: string, ...rest: unknown[]) => Record).bind(db); + Object.defineProperty(db, name, { configurable: true, writable: true, value: (sql: string, ...rest: unknown[]) => { + const statement = make(sql, ...rest); + if (statements.has(statement)) return statement; + statements.add(statement); + // Reads need no turn; a RETURNING write runs through the readers too. + for (const step of /\bRETURNING\b/i.test(sql) ? ["run", "get", "all", "values"] : ["run"]) { + const call = (statement[step] as (...args: unknown[]) => unknown).bind(statement); + Object.defineProperty(statement, step, { configurable: true, writable: true, value: taking(call) }); + } + return statement; + } }); + } +} + +function inlineWriter(db: Database): SqliteWriter { + let tail: Promise = Promise.resolve(); + let closed = false; + return { + offloop: false, + channel(module, options) { + let procedures: WriteProcedures | undefined, failure: unknown; + try { procedures = module.setup(db, options); } catch (error) { failure = error; } + const ready = failure === undefined ? Promise.resolve() : Promise.reject(failure); + ready.catch(() => {}); + return { + ready, + call(name: string, ...args: unknown[]): Promise { + // Run at submission, exactly as a synchronous write did; errors keep + // their class because nothing crosses a thread. + let result: Promise; + try { + if (closed) throw new SqliteWriterClosed(); + if (!procedures) throw failure; + const procedure = procedures[name]; + if (typeof procedure !== "function" || name === "close") throw new Error(`Unknown write procedure ${name}.`); + result = Promise.resolve((procedure as (...values: unknown[]) => T)(...args)); + } catch (error) { result = Promise.reject(error); } + tail = result.catch(() => {}); + return result; + }, + async close() { procedures?.close?.(); procedures = undefined; failure ??= new SqliteWriterClosed(); }, + }; + }, + async idle() { await tail; }, + async close() { closed = true; await tail; }, + }; +} + +type Pending = { resolve(value: unknown): void; reject(error: Error): void; revive?: (error: WireError) => Error | undefined }; + +function workerWriter(path: string, attach: readonly AttachedDatabase[], turn: Int32Array): SqliteWriter { + const worker = new Worker(THREAD) as Worker & { ref(): void; unref(): void }; + const pending = new Map(); + let next = 0, channels = 0, closed = false, stopped: Error | undefined; + // An idle writer never keeps the process alive; a pending commit does. + const reference = () => { if (pending.size) worker.ref(); else worker.unref(); }; + const stop = (error: Error) => { + stopped ??= error; + for (const [id, waiter] of pending) { pending.delete(id); waiter.reject(stopped); } + reference(); + }; + worker.onmessage = (event: MessageEvent<{ id: number; ok: boolean; value?: unknown; error?: WireError }>) => { + const waiter = pending.get(event.data.id); + if (!waiter) return; + pending.delete(event.data.id); + if (event.data.ok) waiter.resolve(event.data.value); + else waiter.reject(decodeError(event.data.error!, waiter.revive)); + reference(); + }; + worker.onerror = event => { event.preventDefault?.(); stop(new SqliteWriterClosed(`The SQLite write worker failed: ${event.message}`)); }; + worker.addEventListener("close", () => stop(new SqliteWriterClosed())); + function send(message: Record, revive?: Pending["revive"]): Promise { + if (stopped) return Promise.reject(stopped); + const id = ++next; + const result = new Promise((resolve, reject) => pending.set(id, { resolve: resolve as (value: unknown) => void, reject, revive })); + worker.postMessage({ ...message, id }); + reference(); + return result; + } + const opened = send({ kind: "open", path, attach: [...attach], turn }); + opened.catch(() => {}); + return { + offloop: true, + channel(module, options, revive) { + const channel = ++channels; + let open = true; + const ready = send({ kind: "register", channel, module: module.url, options }, revive); + ready.catch(() => {}); + return { + ready, + call(name: string, ...args: unknown[]) { + if (!open) return Promise.reject(new SqliteWriterClosed()); + return send({ kind: "call", channel, name, args }, revive); + }, + async close() { + if (!open) return; + open = false; + await send({ kind: "release", channel }).catch(() => {}); + }, + }; + }, + async idle() { await send({ kind: "idle" }).catch(() => {}); }, + async close() { + if (closed) return; + closed = true; + await send({ kind: "close" }).catch(() => {}); + stop(new SqliteWriterClosed()); + worker.terminate(); + }, + }; +} diff --git a/packages/ui-server/src/inbox/autonomous-turn.ts b/packages/ui-server/src/inbox/autonomous-turn.ts index 58da9ddc49..43a64a8022 100644 --- a/packages/ui-server/src/inbox/autonomous-turn.ts +++ b/packages/ui-server/src/inbox/autonomous-turn.ts @@ -411,6 +411,10 @@ async function executeAutonomousTurn( // Flushed after the backend returns, before the terminal record. A failed // update is on each application's record; its paths stay queued. if (turnIndex?.pending()) await turnIndex.barrier().catch(() => {}); + // The run's earlier Activity writes, the flush's index outcome included, + // commit off the event loop (#2814); with none pending, the terminal + // below commits inside this transaction. + await recorder.settled(); // The backend has returned. Release its kernel lock under the operational // write lock, then settle before another connection can claim cleanup. deps.db.transaction(() => { diff --git a/packages/ui-server/src/labels/labeller.ts b/packages/ui-server/src/labels/labeller.ts index 951dc8fe0e..67e147e138 100644 --- a/packages/ui-server/src/labels/labeller.ts +++ b/packages/ui-server/src/labels/labeller.ts @@ -26,7 +26,8 @@ import type { BillingMode } from "@schlessera/brain-ui-sdk/protocol"; import type { Logger } from "@opentelemetry/api-logs"; import { createHash } from "node:crypto"; -import type { ActivityStore, SpanOutcome, SpanUsage } from "../activity/store.js"; +import type { AwaitableActivityStore, SpanOutcome, SpanUsage } from "../activity/store.js"; +import { orderedWrites } from "../activity/store-writes.js"; import { normaliseLabel } from "./normalise.js"; /** What the labeller sends for one label. */ @@ -124,7 +125,7 @@ export interface CreateLabellerDeps { meter?: Meter; /** Where each provider call is recorded as a `pill label` run. Absent records none. */ activity?: { - store: Pick; + store: Pick; /** Called after each write so the live Activity stream can pump. */ onWrite?: () => void; }; @@ -194,26 +195,32 @@ export function createLabeller(deps: CreateLabellerDeps): Labeller { type CallResult = { outcome: LabelOutcome; label: string | null; err?: unknown }; - const guardActivity = (write: () => void) => { - if (!deps.activity) return; + // Recording never costs the pill its label. + const notRecorded = (err: unknown) => { + deps.log?.emit({ + severityText: "WARN", + body: "pill label run not recorded", + attributes: { error: err instanceof Error ? err.message : String(err) }, + }); + }; + // Runs commit in call order, off the event loop for the hosted store (#2814). + const writes = deps.activity ? orderedWrites(deps.activity.store, notRecorded) : undefined; + const guardActivity = (write: (store: Pick) => void) => { + if (!deps.activity || !writes) return; try { - write(); - deps.activity.onWrite?.(); + write(writes.store); + const wrote = deps.activity.onWrite; + if (wrote) writes.after(wrote); } catch (err) { - // Recording never costs the pill its label. - deps.log?.emit({ - severityText: "WARN", - body: "pill label run not recorded", - attributes: { error: err instanceof Error ? err.message : String(err) }, - }); + notRecorded(err); } }; /** Open the call's run as it reaches the provider. */ const startRun = (sessionId: string | undefined): string => { const runId = `label-${crypto.randomUUID()}`; - guardActivity(() => { - deps.activity!.store.startSpan({ + guardActivity(store => { + store.startSpan({ spanId: `${runId}:root`, runId, name: LABEL_RUN_NAME, @@ -233,11 +240,10 @@ export function createLabeller(deps: CreateLabellerDeps): Labeller { * read as free rather than unknown. */ const endRun = (runId: string, outcome: SpanOutcome, completion?: Omit) => { - guardActivity(() => { + guardActivity(store => { const usage = reportedUsage(completion?.usage); const model = typeof completion?.model === "string" && completion.model ? completion.model : undefined; const spanUsage: SpanUsage | undefined = usage ? { ...usage, ...(model ? { model } : {}) } : undefined; - const { store } = deps.activity!; store.endSpan(`${runId}:root`, { outcome, ...(spanUsage ? { usage: spanUsage } : {}), diff --git a/packages/ui-server/src/routes/activity.ts b/packages/ui-server/src/routes/activity.ts index e8fc5940bc..5ba6690369 100644 --- a/packages/ui-server/src/routes/activity.ts +++ b/packages/ui-server/src/routes/activity.ts @@ -9,7 +9,7 @@ import { type ActivityRunSummary, } from "@schlessera/brain-ui-sdk/protocol"; -import { rowToRunRollup, type ActivityStore } from "../activity/store.js"; +import { rowToRunRollup, type AwaitableActivityStore } from "../activity/store.js"; import { toWireEvent, toWireRollup, toWireSpan } from "../activity/stream.js"; import type { ActivityNotifier } from "../activity/notify.js"; import { noticeClientId, type ActionNotifier } from "../inbox/notify.js"; @@ -39,7 +39,7 @@ import { */ export function createActivityRoutes(deps: { db: Database; - store: ActivityStore; + store: AwaitableActivityStore; notifier?: ActivityNotifier; /** Durable Action notices; their digest contribution is per client context. */ actionNotices?: ActionNotifier; diff --git a/packages/ui-server/src/routes/journal.ts b/packages/ui-server/src/routes/journal.ts index 4f6a5bf647..729940c21c 100644 --- a/packages/ui-server/src/routes/journal.ts +++ b/packages/ui-server/src/routes/journal.ts @@ -34,13 +34,13 @@ export function createJournalRoutes({ journal }: { journal: TurnJournal }): Hono return c.json({ inline: true, before, after }); } catch { return c.json({ inline: false, reason: "binary_or_unavailable" }); } }) - .get("/sessions/:sessionId/turns/:turnId/journal/:sequence/:side", c => { + .get("/sessions/:sessionId/turns/:turnId/journal/:sequence/:side", async c => { const sequence = Number(c.req.param("sequence")), side = c.req.param("side"); if (!Number.isSafeInteger(sequence) || sequence < 1 || !["before", "after"].includes(side)) return c.json({ error: "Invalid retained version." }, 400); const version = journal.version(c.req.param("sessionId"), c.req.param("turnId"), sequence, side as "before" | "after"); if (!version) return c.json({ error: "Journal detail expired or unavailable." }, 404); try { - const stream = version.stream(); + const stream = await version.stream(); return new Response(stream.body, { headers: { "Content-Type": "application/octet-stream", "Content-Length": String(stream.bytes), "Content-Disposition": `attachment; filename="journal-${sequence}-${side}"`, "Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff" } }); } catch { return c.json({ error: "Retained version is temporarily unavailable." }, 503); } }); diff --git a/packages/ui-server/src/routes/storage-removal.ts b/packages/ui-server/src/routes/storage-removal.ts index b2d18b3cdd..137385ac60 100644 --- a/packages/ui-server/src/routes/storage-removal.ts +++ b/packages/ui-server/src/routes/storage-removal.ts @@ -40,8 +40,8 @@ export function createStorageRemovalRoutes(removal: StorageRemoval): Hono c.json(removal.cancel(c.get("principal")!.id, c.req.param("id")), 202)); + routes.delete("/storage/removals/:id", async c => c.json(await removal.cancel(c.get("principal")!.id, c.req.param("id")), 202)); return routes; } diff --git a/packages/ui-server/src/routes/turn-revert.ts b/packages/ui-server/src/routes/turn-revert.ts index cc049b918b..84c65a4cac 100644 --- a/packages/ui-server/src/routes/turn-revert.ts +++ b/packages/ui-server/src/routes/turn-revert.ts @@ -36,7 +36,7 @@ export function createTurnRevertRoutes(reverts: TurnReverts): Hono { if(Object.keys(request).some(key=>!["id","path","action","version","otherVersion","expected"].includes(key))||typeof request.id!=="string"||request.path!==undefined&&typeof request.path!=="string"||request.action!==undefined&&!["revert","restore","keep_version","keep_both"].includes(request.action) ||resolving&&(!request.path||!reference(request.version)||!reference(request.expected)||request.action==="keep_both"&&!reference(request.otherVersion)) ||request.otherVersion!==undefined&&!reference(request.otherVersion))return c.json({error:"Invalid history action."},400); - try { return c.json(reverts.start(c.get("principal")!.id,c.req.param("sessionId"),c.req.param("turnId"),request),202); } + try { return c.json(await reverts.start(c.get("principal")!.id,c.req.param("sessionId"),c.req.param("turnId"),request),202); } catch (error) { return c.json({error:error instanceof RevertUnavailable ? error.message : "History storage is temporarily unavailable; reconcile the original operation identity before considering another effect."},error instanceof RevertUnavailable ? 409 : 503); } }); return routes; diff --git a/packages/ui-server/src/storage/removal.ts b/packages/ui-server/src/storage/removal.ts index e672346b82..c6434a0096 100644 --- a/packages/ui-server/src/storage/removal.ts +++ b/packages/ui-server/src/storage/removal.ts @@ -11,6 +11,7 @@ import { BRAIN_LOCK_KEY, type KeyedLock } from "@schlessera/brain-ui-sdk/server" import { GIT_LOCK_KEY } from "@schlessera/brain-ui-sdk/internal"; import { MAINTENANCE_ENTRY, type HostBoundary } from "../brain/host-boundary.js"; import type { TurnJournal } from "../brain/turn-journal.js"; +import { journalCheck, journalStatement, JournalStatementRefused } from "../brain/journal-store.js"; import { SELF, processAlive, type WriterGate } from "../brain/writer-gate.js"; import { StorageUnavailable, type StorageCoordinator } from "./coordinator.js"; import { readRemovalFacts } from "./removal-facts.js"; @@ -71,17 +72,20 @@ export function createStorageRemoval(db: Database, options: { if ([...options.protectedPaths, ...status.protectedPaths.map(value => value.path)].some(root => prefix(path, root))) return "protected_owner_path"; return null; } - const save = (row: Row, result: StorageRemovalReceipt, versions?: PrivateFile[]) => journal.writeOperation(() => { - db.query("UPDATE turn_journal_storage_operations SET state=?,receipt=?,private_versions=? WHERE id=? AND brain=?") - .run(result.state, encode(result), versions ? encode(versions) : row.private_versions, row.id, brain); - row.state = result.state; row.receipt = encode(result); if (versions) row.private_versions = encode(versions); - }); + /** Resolves once the receipt has committed. */ + const save = async (row: Row, result: StorageRemovalReceipt, versions?: PrivateFile[]) => { + const receipt = encode(result), privateVersions = versions ? encode(versions) : row.private_versions; + await journal.writeOperation([journalStatement("UPDATE turn_journal_storage_operations SET state=?,receipt=?,private_versions=? WHERE id=? AND brain=?", + result.state, receipt, privateVersions, row.id, brain)]); + row.state = result.state; row.receipt = receipt; row.private_versions = privateVersions; + }; function recover(row: Row): StorageRemovalReceipt { const result = JSON.parse(row.receipt) as StorageRemovalReceipt; if (["waiting", "running"].includes(row.state) && (!processAlive(row.pid, row.pid_start) || row.pid === SELF.pid && row.pid_start === SELF.start && !runningHere.has(key(row.id)))) { result.state = row.state === "waiting" ? "refused" : "unknown"; result.reason = row.state === "waiting" ? "interrupted_before_effect" : "interrupted_outcome_unknown"; result.completedAt = Date.now(); for (const item of result.items) if (item.state === "applying") { item.state = "unknown"; item.reason = "interrupted_outcome_unknown"; } - try { save(row, result); } catch { /* Durable pending intent still prevents replay. */ } + // Durable pending intent still prevents replay if this repair cannot commit. + void save(row, result).catch(() => {}); } const versions = JSON.parse(row.private_versions) as PrivateFile[]; for (const item of result.items) { @@ -150,11 +154,11 @@ export function createStorageRemoval(db: Database, options: { const result: StorageRemovalReceipt = { review: reviewed, state: "review", requestedAt: now, startedAt: null, completedAt: null, reason: null, items: files.map(file => ({ path: file.path, state: file.eligible ? "not-started" : "protected", reason: file.reason, undoAvailable: false, undoReason: "no_confirmed_removal" })), before: null, after: null, index: "not-requested", commit: "not-requested", sync: "not-requested", retained: { journal: "subject-to-retention", git: "unchanged-by-operation", backups: "not-measured" } }; - if (persist) journal.writeOperation(() => { - if ((db.query("SELECT count(*) AS total FROM turn_journal_storage_operations WHERE brain=?").get(brain) as {total: number}).total >= STORAGE_REMOVAL_LIMITS.receipts) throw new StorageUnavailable(503, "Attachment receipt capacity reached; existing receipts remain readable."); - db.query("INSERT INTO turn_journal_storage_operations (id,brain,principal_id,action,source_id,selection,fingerprint,journal_turn_id,state,pid,pid_start,requested_at,receipt,private_versions) VALUES (?,?,?,?,?,?,?,?,'review',?,?,?,?,?)") - .run(id, brain, principal, action, sourceId, selection, reviewed.fingerprint, `storage_${id}`, SELF.pid, SELF.start, now, encode(result), encode(versions)); - }); + if (persist) await journal.writeOperation([ + journalCheck("SELECT 1 WHERE (SELECT count(*) FROM turn_journal_storage_operations WHERE brain=?) < ?", "Attachment receipt capacity reached; existing receipts remain readable.", brain, STORAGE_REMOVAL_LIMITS.receipts), + journalStatement("INSERT INTO turn_journal_storage_operations (id,brain,principal_id,action,source_id,selection,fingerprint,journal_turn_id,state,pid,pid_start,requested_at,receipt,private_versions) VALUES (?,?,?,?,?,?,?,?,'review',?,?,?,?,?)", + id, brain, principal, action, sourceId, selection, reviewed.fingerprint, `storage_${id}`, SELF.pid, SELF.start, now, encode(result), encode(versions)), + ]).catch(error => { throw error instanceof JournalStatementRefused ? new StorageUnavailable(503, error.message) : error; }); return result; }); } finally { clearTimeout(timeout); reviews.delete(controller); pendingReviews.delete(done); finish(); } } @@ -175,13 +179,13 @@ export function createStorageRemoval(db: Database, options: { const covered = await journal.removalCoverage(eligible, controller.signal, row.action === "undo"); check(); await journal.begin(row.journal_turn_id, { sessionId: null, actor: "user", model: null, profile: "strict", policyHash: null }, () => controller.abort(), controller.signal); journalStarted = true; - result.state = "running"; result.startedAt = Date.now(); save(row, result); + result.state = "running"; result.startedAt = Date.now(); await save(row, result); for (const item of result.items) { if (item.state !== "not-started") continue; check(); const privateFile = versions.find(file => file.path === item.path)!; const reason = protectedReason(item.path); - if (reason) { item.state = "protected"; item.reason = reason; save(row, result); continue; } - if (privateFile.consequences !== factIdentity(facts, item.path)) { item.state = "changed"; item.reason = "consequences_changed"; save(row, result); continue; } + if (reason) { item.state = "protected"; item.reason = reason; await save(row, result); continue; } + if (privateFile.consequences !== factIdentity(facts, item.path)) { item.state = "changed"; item.reason = "consequences_changed"; await save(row, result); continue; } try { if (!covered[eligible.indexOf(item.path)]) throw new StorageAttachmentRefusal("protected"); if (row.action === "remove") { @@ -198,20 +202,20 @@ export function createStorageRemoval(db: Database, options: { if (!storageAttachmentVacant(brain, item.path, privateFile.version!)) throw new StorageAttachmentRefusal("occupied"); } check(); if (protectedReason(item.path)) throw new StorageAttachmentRefusal("protected"); - item.state = "applying"; save(row, result, versions); // Durable retained identity and item intent precede the first file effect. + item.state = "applying"; await save(row, result, versions); // Durable retained identity and item intent precede the first file effect. if (row.action === "remove") await removeStorageAttachment(brain, item.path, privateFile.version!, controller.signal, check); else { - try { journal.withRetainedRemoval(select(row.source_id!)!.journal_turn_id, privateFile.retained!, copy => restoreStorageAttachment(brain, item.path, privateFile.version!, copy, check)); record("brain", item.path, "success"); } + try { await journal.withRetainedRemoval(select(row.source_id!)!.journal_turn_id, privateFile.retained!, copy => restoreStorageAttachment(brain, item.path, privateFile.version!, copy, check)); record("brain", item.path, "success"); } catch (error) { record("brain", item.path, "no-space", error); throw error; } } effects = true; item.state = row.action === "remove" ? "removed" : "restored"; item.reason = null; - save(row, result, versions); // An acknowledgement failure cannot authorize another file. + await save(row, result, versions); // An acknowledgement failure cannot authorize another file. await journal.observe(); } catch (error) { if (item.state === "applying" || item.state === "removed" || item.state === "restored") { unresolved = true; throw error; } item.state = error instanceof StorageAttachmentRefusal && ["changed", "occupied"].includes(error.reason) ? "changed" : error instanceof StorageAttachmentRefusal && error.reason === "protected" ? "protected" : "failed"; item.reason = error instanceof StorageAttachmentRefusal ? error.reason : (error as {reason?: string}).reason === "journal_capacity" ? "journal_capacity" : ["ENOSPC", "EDQUOT"].includes((error as {code?: string}).code ?? "") ? "recovery_no_space" : "recovery_or_file_unavailable"; - save(row, result, versions); + await save(row, result, versions); } } await journal.finish(row.journal_turn_id); journalStarted = false; @@ -237,7 +241,7 @@ export function createStorageRemoval(db: Database, options: { } for (const item of result.items) if (item.state === "applying") { item.state = "unknown"; item.reason = "outcome_requires_reconciliation"; } result.completedAt = Date.now(); - try { save(row, result, versions); } catch { /* Original per-item intent survives; a later read returns unknown without replay. */ } + try { await save(row, result, versions); } catch { /* Original per-item intent survives; a later read returns unknown without replay. */ } } } return { @@ -250,25 +254,26 @@ export function createStorageRemoval(db: Database, options: { encode(result); // Same response ceiling; never persist or expose a confirmable identity. return result; }, - start(principal: string, id: string, fingerprint: string) { + async start(principal: string, id: string, fingerprint: string): Promise { authorize(principal); const row = select(id); if (!row || row.principal_id !== principal) throw new StorageUnavailable(404, "Attachment review unavailable."); if (fingerprint !== row.fingerprint) throw new StorageUnavailable(409, "Review identity changed."); const result = recover(row); if (row.state !== "review") return result; if (!capability(principal).available || Date.now() >= result.review.expiresAt || !result.review.files.some(file => file.eligible)) throw new StorageUnavailable(409, "Refresh the attachment review before removal."); - journal.writeOperation(() => { - if (db.query("SELECT 1 FROM turn_journal_storage_operations WHERE brain=? AND state IN ('waiting','running')").get(brain)) throw new StorageUnavailable(409, "Another attachment operation is active."); - result.state = "waiting"; - const changed = db.query("UPDATE turn_journal_storage_operations SET state='waiting',pid=?,pid_start=?,receipt=? WHERE id=? AND state='review'").run(SELF.pid, SELF.start, encode(result), id); - if (changed.changes !== 1) throw new StorageUnavailable(409, "Attachment operation already admitted."); - }); + const waiting = { ...result, state: "waiting" as const }; + const [, changed] = await journal.writeOperation([ + journalCheck("SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM turn_journal_storage_operations WHERE brain=? AND state IN ('waiting','running'))", "Another attachment operation is active.", brain), + journalStatement("UPDATE turn_journal_storage_operations SET state='waiting',pid=?,pid_start=?,receipt=? WHERE id=? AND state='review'", SELF.pid, SELF.start, encode(waiting), id), + ]).catch(error => { throw error instanceof JournalStatementRefused ? new StorageUnavailable(409, error.message) : error; }); + if (changed !== 1) throw new StorageUnavailable(409, "Attachment operation already admitted."); + result.state = "waiting"; const controller = new AbortController(); runningHere.add(key(id)); const pending = Promise.resolve().then(() => execute(select(id)!, controller)).finally(() => { active.delete(id); runningHere.delete(key(id)); }); active.set(id, { controller, promise: pending }); return result; }, - cancel(principal: string, id: string) { const result = receipt(principal, id); if (["waiting", "running"].includes(result.state)) journal.writeOperation(() => db.query("UPDATE turn_journal_storage_operations SET cancel_requested=1 WHERE id=?").run(id)); active.get(id)?.controller.abort(); return result; }, + async cancel(principal: string, id: string): Promise { const result = receipt(principal, id); if (["waiting", "running"].includes(result.state)) await journal.writeOperation([journalStatement("UPDATE turn_journal_storage_operations SET cancel_requested=1 WHERE id=?", id)]); active.get(id)?.controller.abort(); return result; }, async idle() { await Promise.allSettled([...active.values()].map(value => value.promise)); }, async close() { closed = true; for (const controller of reviews) controller.abort(); for (const value of active.values()) value.controller.abort(); await Promise.allSettled([...pendingReviews, ...[...active.values()].map(value => value.promise)]); }, }; diff --git a/packages/ui-server/src/ws/bridge.ts b/packages/ui-server/src/ws/bridge.ts index 6085b32092..4ce1c045cf 100644 --- a/packages/ui-server/src/ws/bridge.ts +++ b/packages/ui-server/src/ws/bridge.ts @@ -266,15 +266,16 @@ export function makeBridge( } } } - // Persist-then-emit: the span write commits before the frame goes out, - // so a subscriber's snapshot can never be behind what it just saw live. - // Late frames through a previous turn's bridge stay un-recorded — the - // recorder belongs to ONE turn identity. + // The span write is submitted before the frame goes out, in frame order; + // it commits off the event loop (#2814), and the live Activity stream + // pumps it once committed. Late frames through a previous turn's bridge + // stay un-recorded — the recorder belongs to ONE turn identity. if (turn.turnId === turnId) recorder?.observeFrame(msg); const scoped = withTurnScope(msg, turn, turnId); host.sendToClients(scoped); // Identity must reach the client before its first scoped journal receipt. - if (msg.type === "session_info" && turn.turnId === turnId) { host.budgets?.bind(turnId, msg.sessionId); host.journal?.bindSession(turnId, msg.sessionId); host.scope?.bindSession(turnId, msg.sessionId); } + // The journal publishes its receipt once the binding commits, after this frame. + if (msg.type === "session_info" && turn.turnId === turnId) { host.budgets?.bind(turnId, msg.sessionId); void host.journal?.bindSession(turnId, msg.sessionId).catch(() => { /* The turn's record stays unbound; its observations still commit. */ }); host.scope?.bindSession(turnId, msg.sessionId); } work?.observe(scoped); if (msg.type === "result") { // Only the live turn's own result may set its disposition — a late diff --git a/packages/ui-server/src/ws/host.ts b/packages/ui-server/src/ws/host.ts index cf415208d1..8d1e00b076 100644 --- a/packages/ui-server/src/ws/host.ts +++ b/packages/ui-server/src/ws/host.ts @@ -9,7 +9,7 @@ import type { SessionCatalog } from "./session-catalog.js"; import type { BackendRegistry } from "../agent/backend.js"; import { createSilentObservability, type Observability } from "../observability/index.js"; import { FrameRateLimiter } from "./rate-limit.js"; -import type { ActivityStore } from "../activity/store.js"; +import type { AwaitableActivityStore } from "../activity/store.js"; import type { ActivityStream } from "../activity/stream.js"; import type { PushSender } from "../activity/push-sender.js"; import type { RuntimeStatus } from "../activity/runtime-status.js"; @@ -31,7 +31,7 @@ import type { ApplicationPlanner } from "../brain/application.js"; /** The activity record and its live stream, when the host records activity. */ export interface ActivityRuntime { - store: ActivityStore; + store: AwaitableActivityStore; stream: ActivityStream; /** Push bindings invalidated through the same principal-revocation boundary. */ pushSender?: Pick; diff --git a/packages/ui-server/src/ws/run-session.ts b/packages/ui-server/src/ws/run-session.ts index 464988c298..611b8ec6e7 100644 --- a/packages/ui-server/src/ws/run-session.ts +++ b/packages/ui-server/src/ws/run-session.ts @@ -918,6 +918,9 @@ async function runRetainedSession( turn.work = undefined; releaseActiveAuthorization(); releaseActiveAuthorization = undefined; + // The recorder's writes, terminal included, commit off the event loop + // (#2814); the slot waits for them before the next turn or its end. + await recorder?.settled(); } // Subsequent (queued) turns resume the now-known session and keep its diff --git a/packages/ui-server/src/ws/session-catalog.ts b/packages/ui-server/src/ws/session-catalog.ts index 6e5ef09962..e24b818f52 100644 --- a/packages/ui-server/src/ws/session-catalog.ts +++ b/packages/ui-server/src/ws/session-catalog.ts @@ -1,4 +1,4 @@ -import { createActivityStore, priceUsage, type ActivityStore, type RollupPricing } from "../activity/store.js"; +import { createActivityStore, priceUsage, type AwaitableActivityStore, type RollupPricing } from "../activity/store.js"; import { createModelPricing } from "../pricing/model-pricing.js"; import { resolveStandalonePricingConfig } from "../config/env.js"; import { allocateClaudeTitleUsage, reconcileClaudeUsage, type ClaudeUsageEvidence, type ClaudeUsageState } from "./claude-usage-accounting.js"; @@ -162,7 +162,7 @@ const UPSERT_SESSION_SQL = `INSERT INTO sessions (id, title, created_at, last_ac /** SQLite-backed catalog over the app's own database (injected by createApp). */ export function createSessionCatalog(db: () => Database, log?: Logger, scratchPins?: { record(sessionId: string, backendId: string, messages: SessionHistoryMessage[]): void }, - accounting?: { pricing: RollupPricing; store: ActivityStore }): SessionCatalog { + accounting?: { pricing: RollupPricing; store: AwaitableActivityStore }): SessionCatalog { let accountingDeps = accounting; const getAccounting = () => accountingDeps ??= (() => { const pricing = createModelPricing(resolveStandalonePricingConfig()); diff --git a/packages/ui-server/tests/activity-offloop.test.ts b/packages/ui-server/tests/activity-offloop.test.ts new file mode 100644 index 0000000000..f105c429a2 --- /dev/null +++ b/packages/ui-server/tests/activity-offloop.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createUiDb } from "../src/db/client.js"; +import { openSqliteWriter } from "../src/db/write-worker.js"; +import { createActivityStore } from "../src/activity/store.js"; +import { createOffloopActivityStore } from "../src/activity/store-writes.js"; +import { createModelPricing, pricingCachePath } from "../src/pricing/model-pricing.js"; +import * as probe from "./fixtures/write-worker-probe.js"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "activity-offloop-odysseus-")), brain = join(root, "brain"), path = join(root, "ui.db"); + mkdirSync(brain); + const db = createUiDb(path), writer = openSqliteWriter(db); + const store = createOffloopActivityStore(db, writer, { writer: "odysseus-server", pricingConfig: { brainPath: brain, enabled: true } }); + return { root, brain, db, writer, store, async close() { await writer.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; +} +const root = (runId: string, model = "claude-sonnet-5-5") => ({ spanId: `${runId}:root`, runId, name: "invoke_agent", kind: "turn" as const, origin: "session" as const, sessionId: "ithaca", + attrs: { "brain.billing_mode": "api", "gen_ai.usage.per_model": { [model]: { inputTokens: 1000, outputTokens: 100 } } } }); + +test("standalone writes commit on the worker in call order, unawaited", async () => { + const f = fixture(); + try { + expect(f.writer.offloop).toBe(true); + void f.store.startSpan(root("voyage")); + for (let index = 0; index < 30; index++) void f.store.appendEvent("voyage:root", "log", { index, port: index % 2 ? "Ithaca" : "Troy" }); + void f.store.endSpan("voyage:root", { outcome: "success" }); + await f.store.rollupRun("voyage"); + const snapshot = f.store.snapshotRun("voyage")!; + expect(snapshot.events.map(event => [event.eventIndex, (event.payload as { index: number }).index]), "ACTIVITY_OFFLOOP_WRITE_ORDER") + .toEqual(Array.from({ length: 30 }, (_, index) => [index, index])); + expect(snapshot.spans[0]?.outcome, "ACTIVITY_OFFLOOP_TERMINAL_AFTER_EVENTS").toBe("success"); + expect(f.store.runRollup?.("voyage")?.runId).toBe("voyage"); + } finally { await f.close(); } +}); + +test("a write inside the caller's transaction stays atomic with it", async () => { + const f = fixture(); + try { + const span = f.db.transaction(() => f.store.startSpan(root("kept")))(); + expect((span as { spanId?: string }).spanId, "ACTIVITY_IN_TRANSACTION_WRITE_IS_SYNCHRONOUS").toBe("kept:root"); + expect(() => f.db.transaction(() => { f.store.startSpan(root("rolled-back")); throw new Error("Scylla"); })()).toThrow("Scylla"); + await f.writer.idle(); + expect(f.store.getSpan("rolled-back:root"), "ACTIVITY_IN_TRANSACTION_WRITE_ROLLS_BACK_WITH_CALLER").toBeNull(); + expect(f.store.getSpan("kept:root")?.spanId).toBe("kept:root"); + } finally { await f.close(); } +}); + +test("clock defaults are taken at submission, not when the worker reaches the write", async () => { + const f = fixture(); + try { + const held = f.writer.channel(probe, { table: "odysseus_log" }); await held.ready; + const slow = held.call("slow", "Calypso", 300); + const submitted = Date.now(); + const started = f.store.startSpan({ spanId: "late:root", runId: "late", name: "invoke_agent", kind: "turn", origin: "session" }); + await slow; await started; + expect(f.store.getSpan("late:root")!.startedAt - submitted, "ACTIVITY_DEFAULT_CLOCK_AT_SUBMISSION").toBeLessThan(150); + } finally { await f.close(); } +}); + +test("worker rollups price like the app's store and follow a republished table", async () => { + const f = fixture(); + const local = createUiDb(join(f.root, "local.db")); + try { + const sync = createActivityStore(local, { writer: "odysseus-local", pricing: createModelPricing({ brainPath: f.brain, enabled: true }) }); + for (const store of [sync, f.store]) { + await store.startSpan(root("priced")); + await store.endSpan("priced:root", { outcome: "success" }); + await store.rollupRun("priced"); + } + const worker = f.store.runRollup?.("priced"), app = sync.runRollup?.("priced"); + expect(app?.effectiveCostUsd, "ACTIVITY_SNAPSHOT_PRICES_RUN").toBeGreaterThan(0); + expect(worker?.effectiveCostUsd, "ACTIVITY_WORKER_PRICES_LIKE_APP").toBe(app!.effectiveCostUsd); + // The app's refresh republishes the shared on-disk table. + const cache = pricingCachePath(f.brain); mkdirSync(dirname(cache), { recursive: true }); + const rates = { "claude-sonnet-5-5": { input: 1e-3, output: 1e-3, cacheRead: null, cacheWrite: null } }; + writeFileSync(cache, JSON.stringify({ version: 1, fetchedAt: Date.now(), litellm: { fetchedAt: Date.now(), rates }, openrouter: { fetchedAt: Date.now(), rates } })); + await f.store.startSpan(root("republished")); + await f.store.endSpan("republished:root", { outcome: "success" }); + await f.store.rollupRun("republished"); + expect(f.store.runRollup?.("republished")?.effectiveCostUsd, "ACTIVITY_WORKER_RELOADS_PUBLISHED_PRICING").toBeCloseTo(1.1, 6); + } finally { local.close(); await f.close(); } +}); diff --git a/packages/ui-server/tests/app-native-cleanup.test.ts b/packages/ui-server/tests/app-native-cleanup.test.ts index 73a1faa979..881a3c41bf 100644 --- a/packages/ui-server/tests/app-native-cleanup.test.ts +++ b/packages/ui-server/tests/app-native-cleanup.test.ts @@ -1,4 +1,5 @@ import { expect, spyOn, test } from "bun:test"; +import { expectRejection, rejection } from "./helpers/expect-rejection.js"; import { Database } from "bun:sqlite"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; @@ -28,10 +29,10 @@ test("app shutdown closes operational state even after journal cleanup rejects, const journal = app.wsHost.journal!, close = journal.close.bind(journal), failure = Error("Fixture journal cleanup failed after settling roots"); journal.close = async () => { await close(); throw failure; }; try { - await expect(app.close()).rejects.toBe(failure); + expect(await rejection(app.close())).toBe(failure); expect(() => app.db.query("SELECT 1").get(), "APP_CLEANUP_FAILURE_STILL_CLOSES_UI_DATABASE").toThrow(); expect(app.wsHost.closing).toBe(true); - await expect(app.close()).rejects.toBe(failure); + expect(await rejection(app.close())).toBe(failure); } finally { await f.close(); } }); @@ -42,7 +43,7 @@ test("unknown native deletion stays durable while app shutdown still closes UI s try { await app.wsHost.journal!.begin("cleanup-turn", { actor: "interactive", sessionId: "ithaca", model: null, profile: "governed", policyHash: "fixture" }, () => {}); f.native.client.release = async () => { throw Error("Fixture unknown native deletion receipt"); }; - await expect(app.close()).rejects.toBeInstanceOf(AggregateError); + await expectRejection(app.close(), AggregateError); expect(() => app.db.query("SELECT 1").get(), "APP_UNKNOWN_NATIVE_RELEASE_STILL_CLOSES_UI_DATABASE").toThrow(); ledger = new Database(f.leasePath); const rows = ledger.query("SELECT id,state FROM native_root_leases").all() as { id: string; state: string }[]; @@ -63,7 +64,7 @@ test("late app boot failure drains acquired runtimes and closes its UI database const observability = createRecordingObservability(), meter = observability.meter.bind(observability), failure = Error("Fixture late boot auth meter refusal"); observability.meter = name => { if (name === "auth") throw failure; return meter(name); }; try { - await expect(createApp({ config: f.config, registry: f.registry, observability })).rejects.toBe(failure); + expect(await rejection(createApp({ config: f.config, registry: f.registry, observability }))).toBe(failure); expect(handles.length, "APP_LATE_BOOT_FAILURE_EXERCISES_OPEN_UI_STATE").toBeGreaterThan(0); for (const db of handles) expect(() => db.query("SELECT 1").get(), "APP_LATE_BOOT_FAILURE_CLOSES_EVERY_UI_HANDLE").toThrow(); } finally { capture.mockRestore(); await f.close(); } diff --git a/packages/ui-server/tests/autonomous-turn.test.ts b/packages/ui-server/tests/autonomous-turn.test.ts index 2aab98d1bb..e3889c731a 100644 --- a/packages/ui-server/tests/autonomous-turn.test.ts +++ b/packages/ui-server/tests/autonomous-turn.test.ts @@ -9,7 +9,7 @@ import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync } import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AgentBackend, BackendBridge, StartTurnRequest } from "@schlessera/brain-ui-sdk/server"; -import { assertTurnPosture, BackendRequestError, requestToolPermission } from "@schlessera/brain-ui-sdk/server"; +import { assertTurnPosture, BackendRequestError, createKeyedLock, requestToolPermission } from "@schlessera/brain-ui-sdk/server"; import { createUiDb } from "../src/db/client.js"; import { createPrincipal, revokePrincipal } from "../src/db/principals.js"; import { createActivityStore } from "../src/activity/store.js"; @@ -17,6 +17,9 @@ import { createActivityRoutes } from "../src/routes/activity.js"; import { createInboxBudget } from "../src/inbox/budget.js"; import { createInboxStore, InboxStore } from "../src/inbox/store.js"; import { runAutonomousTurn } from "../src/inbox/autonomous-turn.js"; +import { ACTIVITY_WRITE_METHODS } from "../src/activity/store-writes.js"; +import { boundaryPlanner, type ApplicationPlanner } from "../src/brain/application.js"; +import { createFixtureHostBoundary } from "./helpers/native-host-fixture.js"; import { WsHost } from "../src/ws/host.js"; import { createStaticBackendRegistry } from "../src/agent/backend.js"; import { createSessionCatalog } from "../src/ws/session-catalog.js"; @@ -506,3 +509,43 @@ for (const adapter of ["claude", "pi"] as const) test(`${adapter} autonomous inb expect(reached).toBe(true); } finally { f.db.close(); } }); + +test("an autonomous turn's index outcome commits before the transaction that writes its terminal (#2814, #2835)", async () => { + const proposal = { operation: "write" as const, path: "notes/ithaca.md", content: "---\ntitle: Ithaca\ntype: note\n---\nOdysseus reaches Ithaca.\n", expectedBaseHash: null }; + let applied: unknown; + const f = setup(async req => { + applied = await req.bridge.applyBrain!(proposal); + req.bridge.emit({ type: "result", sessionId: "runtime", outcome: "success", isError: false, durationMs: 1, numTurns: 1 }); + }); + // Activity writes resolve only once committed, as the write worker's do; + // the index flush holds them, as a commit waiting on fsync would. + let holding = false; + const queued: (() => void)[] = []; + const writes = Object.create(f.store) as typeof f.store; + for (const name of ACTIVITY_WRITE_METHODS) Object.defineProperty(writes, name, { value: (...args: unknown[]) => { + const commit = () => Reflect.apply(f.store[name] as (...values: unknown[]) => unknown, f.store, args); + if (!holding && !queued.length) return Promise.resolve(commit()); + return new Promise(resolve => queued.push(() => resolve(commit()))); + } }); + const real = boundaryPlanner(createFixtureHostBoundary({ brainPath: f.brainPath })); + const applicationPlanner: ApplicationPlanner = { ...real, async index() { holding = true; throw new Error("maintenance worker timed out"); } }; + f.backend.id = "pi"; + f.backend.brainApplicationPolicy = () => ({ autoAllowed: [], available: ["write"], lock: createKeyedLock() }); + try { + let done = false; + const running = runAutonomousTurn({ ...f, store: writes, applicationPlanner, checkpoint() {} }, + { ...f.input, allowedTools: ["write_file"], application: { approved: { toolName: "write_file", input: proposal, targetPath: proposal.path } } }).finally(() => { done = true; }); + for (let tries = 0; tries < 400 && !holding; tries++) await Bun.sleep(5); + expect(holding).toBe(true); + await Bun.sleep(100); + expect(done, "AUTONOMOUS_TERMINAL_WAITS_FOR_INDEX_OUTCOME").toBe(false); + expect(f.store.getSpan("run:turn")?.outcome ?? null).toBeNull(); + holding = false; + for (const commit of queued.splice(0)) commit(); + await running; + expect(applied).toMatchObject({ ok: true, indexStatus: "pending" }); + const events = f.store.snapshotRun("run")!.events.map(event => event.eventType); + expect(events.indexOf("brain_application_index")).toBeGreaterThan(events.indexOf("brain_application")); + expect(f.store.getSpan("run:turn")!.outcome).not.toBeNull(); + } finally { f.db.close(); } +}); diff --git a/packages/ui-server/tests/claude-memory-journal.test.ts b/packages/ui-server/tests/claude-memory-journal.test.ts index 030861dd00..2b4beb72b8 100644 --- a/packages/ui-server/tests/claude-memory-journal.test.ts +++ b/packages/ui-server/tests/claude-memory-journal.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -8,6 +9,7 @@ import { createStateLedger } from "../../ui-backend-claude/src/worker-state-budg import { createNativeJournalFixture } from "./helpers/native-journal-fixture.js"; import { createUiDb } from "../src/db/client.js"; import { createTurnJournal } from "../src/brain/turn-journal.js"; +import { journalStatement } from "../src/brain/journal-store.js"; import { createTurnReverts } from "../src/brain/turn-revert.js"; import { createWriterGate } from "../src/brain/writer-gate.js"; import { recordGovernanceEvent, queryGovernanceEvents } from "../src/governance/events.js"; @@ -25,12 +27,12 @@ function fixture(capBytes?: number) { function worker(id: string) { const state = createClaudeWorkerState(brain, { CLAUDE_CONFIG_DIR: saved }, { ledger: createStateLedger() }); states.push(state); const temporary = join(root, id); mkdirSync(temporary); state.populate(temporary); - return { state, temporary, notes: join(temporary, project, "memory"), persist() { state.persist(temporary, undefined, (change, save) => journal.persistClaudeMemory(id, change, save)); } }; + return { state, temporary, notes: join(temporary, project, "memory"), persist() { return state.persist(temporary, undefined, (change, save) => journal.persistClaudeMemory(id, change, save)); } }; } const begin = (id: string) => journal.begin(id, { sessionId: "ithaca", actor: "interactive", model: "odysseus-model", profile: "governed", policyHash: "recorded-policy" }, () => {}); const reverts = () => createTurnReverts(db, { brainPath: brain, journal, gate: createWriterGate(db, brain, { pollMs: 1 }), lock: createKeyedLock(), isAuthorized: id => id === "odysseus", refresh() {} }); return { root, brain, saved, memory, db, get journal() { return journal; }, worker, begin, reverts, - async reopen() { await journal.close(); journal = createTurnJournal(db, { nativeManager: native.manager, brainPath: brain, dbPath: path, objectsRoot: join(root, "objects"), capBytes }); }, + async reopen() { await journal.close(); journal = createTurnJournal(db, { nativeManager: native.manager, brainPath: brain, dbPath: path, objectsRoot: join(root, "objects"), capBytes }); await journal.ready; }, async close() { for (const state of states) state.release(); await journal.close(); await native.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; } @@ -41,7 +43,7 @@ test("real populate/persist retains created and replaced external notes with tai const w = f.worker("return"), after = Buffer.from([66, 10, 0, 255]); writeFileSync(join(w.notes, "voyage.md"), after); writeFileSync(join(w.notes, "athena.md"), "Counsel\r\n"); writeFileSync(join(f.brain, "content.md"), "Unrelated content\n"); - w.persist(); await f.journal.finish("return"); + await w.persist(); await f.journal.finish("return"); await f.reopen(); const details = f.journal.details("ithaca", "return")!; const notes = details.changes.filter(c => c.scope === "claude_memory"); @@ -56,18 +58,18 @@ test("real populate/persist retains created and replaced external notes with tai expect(JSON.stringify(details)).not.toContain(f.saved); expect(f.journal.details("other", "return")).toBeNull(); const r = f.reverts(); - expect(() => r.start("odysseus", "ithaca", "return", { id: "memory_forged_0001", path: join(f.memory, "voyage.md") })).toThrow("No retained eligible paths"); - expect(() => r.start("agent", "ithaca", "return", { id: "memory_agent_0001" })).toThrow(); - const receipt = r.start("odysseus", "ithaca", "return", { id: "memory_revert_0001", path: replaced.path }); await r.idle(receipt.id); + await expectRejection(r.start("odysseus", "ithaca", "return", { id: "memory_forged_0001", path: join(f.memory, "voyage.md") }), "No retained eligible paths"); + await expectRejection(r.start("agent", "ithaca", "return", { id: "memory_agent_0001" })); + const receipt = await r.start("odysseus", "ithaca", "return", { id: "memory_revert_0001", path: replaced.path }); await r.idle(receipt.id); expect(r.receipt(receipt.id, "odysseus", "ithaca", "return")?.state, "MEMORY_REVERT_SUCCEEDS_FROM_PRIVATE_RETAINED_MAPPING").toBe("complete"); expect(readFileSync(join(f.memory, "voyage.md"))).toEqual(Buffer.from([65, 13, 10, 255])); expect(readFileSync(join(f.brain, "content.md"), "utf8")).toBe("Unrelated content\n"); - const undo = r.start("odysseus", "ithaca", receipt.journalTurnId, { id: "memory_undo_00001" }); await r.idle(undo.id); + const undo = await r.start("odysseus", "ithaca", receipt.journalTurnId, { id: "memory_undo_00001" }); await r.idle(undo.id); expect(readFileSync(join(f.memory, "voyage.md"))).toEqual(after); - const created = r.start("odysseus", "ithaca", "return", { id: "memory_create_revert", path: "claude-memory://athena.md" }); await r.idle(created.id); + const created = await r.start("odysseus", "ithaca", "return", { id: "memory_create_revert", path: "claude-memory://athena.md" }); await r.idle(created.id); expect(existsSync(join(f.memory, "athena.md"))).toBe(false); - const undoCreated = r.start("odysseus", "ithaca", created.journalTurnId, { id: "memory_creation_undo" }); await r.idle(undoCreated.id); - const whole = r.start("odysseus", "ithaca", "return", { id: "memory_whole_turn" }); await r.idle(whole.id); + const undoCreated = await r.start("odysseus", "ithaca", created.journalTurnId, { id: "memory_creation_undo" }); await r.idle(undoCreated.id); + const whole = await r.start("odysseus", "ithaca", "return", { id: "memory_whole_turn" }); await r.idle(whole.id); expect(r.receipt(whole.id, "odysseus", "ithaca", "return")?.state, "MEMORY_WHOLE_TURN_RECOVERS_CONTENT_AND_EXTERNAL_NOTES").toBe("complete"); expect(readFileSync(join(f.memory, "voyage.md"))).toEqual(Buffer.from([65,13,10,255])); expect(existsSync(join(f.memory,"athena.md"))).toBe(false); @@ -80,9 +82,9 @@ test("peer persistence refuses only the stale note, retains successful notes and const f = fixture(); try { await f.begin("return"); const stale = f.worker("return"); await f.begin("athena"); const peer = f.worker("athena"); - writeFileSync(join(peer.notes, "voyage.md"), "Peer counsel\n"); peer.persist(); await f.journal.finish("athena"); + writeFileSync(join(peer.notes, "voyage.md"), "Peer counsel\n"); await peer.persist(); await f.journal.finish("athena"); writeFileSync(join(stale.notes, "voyage.md"), "Stale counsel\n"); writeFileSync(join(stale.notes, "telemachus.md"), "Independent\n"); - expect(() => stale.persist()).toThrow("changed concurrently"); await f.journal.finish("return"); + await expectRejection(stale.persist(), "changed concurrently"); await f.journal.finish("return"); expect(readFileSync(join(f.memory, "voyage.md"), "utf8"), "MEMORY_PEER_PERSIST_IS_NEVER_OVERWRITTEN").toBe("Peer counsel\n"); expect(readFileSync(join(f.memory, "telemachus.md"), "utf8")).toBe("Independent\n"); expect(f.journal.details("ithaca", "return")?.summary.state).toBe("partial"); @@ -91,7 +93,7 @@ test("peer persistence refuses only the stale note, retains successful notes and expect(partialReverts.plan("ithaca", "return").paths.find(item => item.path === "claude-memory://telemachus.md")?.state, "MEMORY_PARTIAL_SAVE_RETAINS_KNOWN_SUCCESSFUL_RECOVERY").toBe("eligible"); await partialReverts.close(); writeFileSync(join(f.memory, "voyage.md"), "Later peer\n"); - const r = f.reverts(), receipt = r.start("odysseus", "ithaca", "athena", { id: "memory_stale_0001", path: "claude-memory://voyage.md" }); await r.idle(receipt.id); + const r = f.reverts(), receipt = await r.start("odysseus", "ithaca", "athena", { id: "memory_stale_0001", path: "claude-memory://voyage.md" }); await r.idle(receipt.id); expect(r.receipt(receipt.id, "odysseus", "ithaca", "athena")?.paths[0]?.state, "MEMORY_STALE_REVERT_PRESERVES_LATER_SAVE").toBe("conflict"); expect(readFileSync(join(f.memory, "voyage.md"), "utf8")).toBe("Later peer\n"); await r.close(); } finally { await f.close(); } @@ -100,11 +102,11 @@ test("peer persistence refuses only the stale note, retains successful notes and test("interruption after memory commit retains staged versions and reports unconfirmed persistence after reopen", async () => { const f = fixture(); try { await f.begin("return"); const w = f.worker("return"); writeFileSync(join(w.notes, "voyage.md"), "Saved before interruption\n"); - expect(() => w.state.persist(w.temporary, undefined, (change, save) => f.journal.persistClaudeMemory("return", change, () => { save(); throw Error("interrupted finalization"); }))).toThrow("interrupted finalization"); + await expectRejection(w.state.persist(w.temporary, undefined, (change, save) => f.journal.persistClaudeMemory("return", change, () => { save(); throw Error("interrupted finalization"); })), "interrupted finalization"); // Model abrupt owner loss before the exception handler could finalize the // turn. The staged receipt and raw file are the already-durable facts; // successor initialization must not infer whether the save committed. - f.journal.writeOperation(() => f.db.query("UPDATE turn_journal_turns SET state='recording',missing='[]',failure='',pid=2147483647,pid_start='',ended_at=NULL WHERE turn_id='return'").run()); + await f.journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET state='recording',missing='[]',failure='',pid=2147483647,pid_start='',ended_at=NULL WHERE turn_id='return'")]); await f.reopen(); const details = f.journal.details("ithaca", "return")!; expect(details.summary.state).toBe("partial"); @@ -121,7 +123,7 @@ test("memory staging overflow refuses before publication and cannot promise a re const f = fixture(1024 * 1024); try { await f.begin("return"); const w = f.worker("return"); writeFileSync(join(w.notes, "voyage.md"), Buffer.alloc(2 * 1024 * 1024, 65)); - expect(() => w.persist()).toThrow(); await f.journal.finish("return"); + await expectRejection(w.persist()); await f.journal.finish("return"); expect(readFileSync(join(f.memory, "voyage.md")),"MEMORY_CAPACITY_REFUSES_BEFORE_SAVED_BYTES_CHANGE").toEqual(Buffer.from([65,13,10,255])); expect(f.journal.details("ithaca","return")?.changes).toEqual([]); expect(f.journal.details("ithaca","return")?.summary.state).toBe("partial"); @@ -131,9 +133,9 @@ test("memory staging overflow refuses before publication and cannot promise a re test("changed saved-store identity refuses user memory recovery while keeping its replacement untouched", async () => { const { renameSync } = await import("node:fs"); const f = fixture(); try { - await f.begin("return"); const w=f.worker("return");writeFileSync(join(w.notes,"voyage.md"),"Saved by turn\n");w.persist();await f.journal.finish("return"); + await f.begin("return"); const w=f.worker("return");writeFileSync(join(w.notes,"voyage.md"),"Saved by turn\n");await w.persist();await f.journal.finish("return"); renameSync(f.memory,`${f.memory}-held`);mkdirSync(f.memory);writeFileSync(join(f.memory,"voyage.md"),"Replacement store\n"); - const r=f.reverts(),receipt=r.start("odysseus","ithaca","return",{id:"memory_store_0001",path:"claude-memory://voyage.md"});await r.idle(receipt.id); + const r=f.reverts(),receipt=await r.start("odysseus","ithaca","return",{id:"memory_store_0001",path:"claude-memory://voyage.md"});await r.idle(receipt.id); expect(r.receipt(receipt.id,"odysseus","ithaca","return")?.paths[0]?.state,"MEMORY_STORE_IDENTITY_FENCES_REPLACEMENT").toBe("conflict"); expect(readFileSync(join(f.memory,"voyage.md"),"utf8")).toBe("Replacement store\n");await r.close(); } finally { await f.close(); } @@ -145,7 +147,7 @@ test("uninspectable native copy-out marks external memory unavailable before any const f=fixture();try { await f.begin("return");const w=f.worker("return"); symlinkSync(join(f.memory,"voyage.md"),join(w.notes,"untrusted.md")); - expect(()=>w.persist()).toThrow();await f.journal.finish("return"); + await expectRejection(w.persist());await f.journal.finish("return"); const details=f.journal.details("ithaca","return")!; expect(details.summary.state,"MEMORY_PRECALLBACK_FAILURE_IS_PARTIAL").toBe("partial"); expect(details.summary.changedFiles).toBeNull(); diff --git a/packages/ui-server/tests/fixtures/storage-removal-worker.ts b/packages/ui-server/tests/fixtures/storage-removal-worker.ts index 863ea36271..cf053dbf29 100644 --- a/packages/ui-server/tests/fixtures/storage-removal-worker.ts +++ b/packages/ui-server/tests/fixtures/storage-removal-worker.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { storageRemovalFixture } from "../helpers/storage-removal-fixture.js"; import { createUiDb } from "../../src/db/client.js"; import { createWriterGate } from "../../src/brain/writer-gate.js"; +import type { JournalStatement } from "../../src/brain/journal-store.js"; const [root,phase,id,marker]=process.argv.slice(2) as [string,string,string,string]; if(phase==="writer") { const db=createUiDb(join(root,"ui.db")),gate=createWriterGate(db,join(root,"brain"),{pollMs:1}),hold=await gate.acquire("shared","turn:athena",{turn:true}); @@ -13,14 +14,14 @@ if(phase==="writer") { const pause=()=>{writeFileSync(marker,phase);Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);}; const retain=f.journal.retainBefore.bind(f.journal),write=f.journal.writeOperation.bind(f.journal); if(phase==="after-retain")f.journal.retainBefore=async(...args)=>{const result=await retain(...args);pause();return result;}; - f.journal.writeOperation=(apply:()=>T):T=>{ + f.journal.writeOperation=async(statements:readonly JournalStatement[])=>{ const state=()=>{const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(id) as {receipt:string}|null;return row?JSON.parse(row.receipt).items[0].state:null;}; if(phase==="after-unlink"&&state()==="applying"&&!existsSync(target))pause(); - const result=write(apply); + const result=await write(statements); if(phase==="after-intent"&&state()==="applying")pause(); if(phase==="after-ack"&&state()==="removed")pause(); return result; }; - try{const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();throw Error("Crash boundary was not reached");} + try{const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);await f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();throw Error("Crash boundary was not reached");} finally{await f.close();} } diff --git a/packages/ui-server/tests/fixtures/write-worker-probe.ts b/packages/ui-server/tests/fixtures/write-worker-probe.ts new file mode 100644 index 0000000000..42d05a1c90 --- /dev/null +++ b/packages/ui-server/tests/fixtures/write-worker-probe.ts @@ -0,0 +1,27 @@ +/** A write-worker procedure module for tests (#2814): Odysseus's log, one row per call. */ +import type { Database } from "bun:sqlite"; + +export const url = import.meta.url; + +export function setup(db: Database, options: { table: string }) { + db.exec(`CREATE TABLE IF NOT EXISTS ${options.table} (n INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT NOT NULL)`); + const insert = (label: string) => db.query(`INSERT INTO ${options.table} (label) VALUES (?)`).run(label).changes; + return { + log(label: string) { return db.transaction(() => insert(label)).immediate(); }, + /** Holds the commit for `ms`, as a slow fsync would. */ + slow(label: string, ms: number) { + return db.transaction(() => { insert(label); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); return 1; }).immediate(); + }, + lowered(label: string) { db.exec("PRAGMA synchronous = NORMAL"); return db.transaction(() => insert(label)).immediate(); }, + loweredAutocommit(label: string) { db.exec("PRAGMA synchronous = NORMAL"); return insert(label); }, + /** Counts the log twice across a held gap, outside any transaction. */ + gap(ms: number) { + const count = () => (db.query(`SELECT count(*) AS n FROM ${options.table}`).get() as { n: number }).n; + const before = count(); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); return [before, count()]; + }, + level() { return (db.query("PRAGMA synchronous").get() as { synchronous: number }).synchronous; }, + levelOf(schema: string) { return (db.query(`PRAGMA "${schema}".synchronous`).get() as { synchronous: number }).synchronous; }, + loweredIn(schema: string, label: string) { db.exec(`PRAGMA "${schema}".synchronous = NORMAL`); return db.transaction(() => insert(label)).immediate(); }, + refuse(message: string) { throw Object.assign(new Error(message), { reason: "odysseus_refused" }); }, + }; +} diff --git a/packages/ui-server/tests/helpers/expect-rejection.ts b/packages/ui-server/tests/helpers/expect-rejection.ts new file mode 100644 index 0000000000..951772d460 --- /dev/null +++ b/packages/ui-server/tests/helpers/expect-rejection.ts @@ -0,0 +1,24 @@ +import { expect } from "bun:test"; + +/** + * `await expect(promise).rejects.toThrow(expected)` for a promise that waits + * on the SQLite write worker (#2814). Bun 1.4.2's `expect().rejects` and + * `.resolves` wait in a nested event loop that does not deliver the worker's + * reply, so such an assertion never settles. This awaits normally, then applies + * the same `toThrow` matcher. + */ +export async function expectRejection(promise: PromiseLike | unknown, expected?: string | RegExp | Error | (abstract new (...args: never[]) => Error), label?: string): Promise { + const outcome = await Promise.resolve(promise).then(value => ({ resolved: true as const, value }), (error: unknown) => ({ resolved: false as const, error })); + if (outcome.resolved) throw new Error(`${label ? `${label}: ` : ""}expected a rejection, but the promise resolved`); + const thrower = () => { throw outcome.error; }; + if (expected === undefined) expect(thrower, label).toThrow(); + else expect(thrower, label).toThrow(expected as never); + return outcome.error; +} + +/** The rejection reason, for `expect(promise).rejects.` assertions other than `toThrow`. */ +export async function rejection(promise: PromiseLike | unknown, label?: string): Promise { + const outcome = await Promise.resolve(promise).then(value => ({ resolved: true as const, value }), (error: unknown) => ({ resolved: false as const, error })); + if (outcome.resolved) throw new Error(`${label ? `${label}: ` : ""}expected a rejection, but the promise resolved`); + return outcome.error; +} diff --git a/packages/ui-server/tests/journal-database.test.ts b/packages/ui-server/tests/journal-database.test.ts index 806ea97a26..5ebe7fe8e0 100644 --- a/packages/ui-server/tests/journal-database.test.ts +++ b/packages/ui-server/tests/journal-database.test.ts @@ -159,6 +159,7 @@ test("the turn journal replays a missing steering event when it starts", async ( const native = createNativeJournalFixture(brain, f.root); const journal = createTurnJournal(db, { nativeManager: native.manager, brainPath: brain, dbPath: f.path, objectsRoot: join(f.root, "objects") }); try { + await journal.ready; expect(db.query("SELECT dedupe_key FROM governance_events WHERE kind='steering_change'").all(), "JOURNAL_STARTUP_REPLAYS_STEERING").toEqual([{ dedupe_key: "voyage:1" }]); } finally { await journal.close(); await native.close(); f.close(db); } }); diff --git a/packages/ui-server/tests/journal-write-worker.test.ts b/packages/ui-server/tests/journal-write-worker.test.ts new file mode 100644 index 0000000000..e1b73a32f6 --- /dev/null +++ b/packages/ui-server/tests/journal-write-worker.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createNativeJournalFixture } from "./helpers/native-journal-fixture.js"; +import { createUiDb } from "../src/db/client.js"; +import { JournalCapacityError } from "../src/brain/journal-objects.js"; +import { journalStatement, JournalStatementRefused } from "../src/brain/journal-store.js"; +import { createTurnJournal } from "../src/brain/turn-journal.js"; + +function fixture(capBytes?: number) { + const root = mkdtempSync(join(tmpdir(), "journal-worker-odysseus-")), brain = join(root, "brain"), path = join(root, "ui.db"); + mkdirSync(join(brain, "notes"), { recursive: true }); + writeFileSync(join(brain, "notes", "odysseus.md"), "# Ithaca\n"); + const native = createNativeJournalFixture(brain, root), db = createUiDb(path); + const open = () => createTurnJournal(db, { nativeManager: native.manager, brainPath: brain, dbPath: path, objectsRoot: join(root, "objects"), capBytes }); + const turn = (id: string) => db.query("SELECT state,missing,session_id FROM turn_journal_turns WHERE turn_id=?").get(id) as { state: string; missing: string; session_id: string | null } | null; + return { root, db, native, open, turn, async close() { await native.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; +} + +test("the app connection refuses journal rows while the worker connection commits them", async () => { + const f = fixture(), journal = f.open(); + try { + await journal.ready; + expect(() => f.db.query("INSERT INTO turn_journal_turns (turn_id,actor,profile,started_at,state) VALUES ('suitor','interactive','strict',1,'recording')").run(), + "JOURNAL_APP_CONNECTION_REFUSES_JOURNAL_WRITES").toThrow("outside capacity accounting"); + await journal.writeOperation([journalStatement("INSERT INTO turn_journal_turns (turn_id,actor,profile,started_at,state) VALUES ('odysseus','interactive','strict',1,'recording')")]); + expect(f.turn("odysseus")?.state, "JOURNAL_WORKER_COMMITS_ACCOUNTED_WRITE").toBe("recording"); + } finally { await journal.close(); await f.close(); } +}); + +test("startup recovery commits before a write submitted during construction, and writes keep their order", async () => { + const f = fixture(); + const first = f.open(); await first.ready; + // A turn whose owner died, as a restart finds it. + await first.writeOperation([journalStatement("INSERT INTO turn_journal_turns (turn_id,actor,profile,pid,pid_start,started_at,state) VALUES ('interrupted','interactive','strict',2147483647,'',1,'recording')")]); + await first.close(); + const journal = f.open(); + try { + // Submitted before `ready` settles; recovery must still land first. + const writes = [ + journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET missing=? WHERE turn_id='interrupted'", JSON.stringify([{ path: ".", reason: "after recovery" }]))]), + journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET session_id='penelope' WHERE turn_id='interrupted'")]), + journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET session_id='ithaca' WHERE turn_id='interrupted'")]), + ]; + await Promise.all(writes); + expect(f.turn("interrupted"), "JOURNAL_STARTUP_RECOVERY_PRECEDES_LATER_WRITES").toEqual({ state: "partial", missing: JSON.stringify([{ path: ".", reason: "after recovery" }]), session_id: "ithaca" }); + } finally { await journal.close(); await f.close(); } +}); + +test("journal refusals keep their class across the worker and roll the whole write back", async () => { + const f = fixture(256 * 1024), journal = f.open(); + try { + await journal.ready; + const insert = journalStatement("INSERT INTO turn_journal_turns (turn_id,actor,profile,started_at,state) VALUES ('telemachus','interactive','strict',1,'recording')"); + const refused = await expectRejection(journal.writeOperation([insert, { check: "SELECT 1 WHERE 0", refusal: "Penelope refuses" }]), JournalStatementRefused, "JOURNAL_CHECK_REFUSAL_CLASS"); + expect((refused as Error).message).toBe("Penelope refuses"); + expect(f.turn("telemachus"), "JOURNAL_REFUSED_WRITE_ROLLED_BACK").toBeNull(); + const capacity = await expectRejection(journal.writeOperation([journalStatement("INSERT INTO turn_journal_turns (turn_id,actor,profile,started_at,state,summary) VALUES ('suitors','interactive','strict',1,'recording',?)", "Odysseus ".repeat(64 * 1024))]), + JournalCapacityError, "JOURNAL_CAPACITY_CLASS_CROSSES_WORKER") as JournalCapacityError; + expect(capacity.reason).toBe("journal_capacity"); + expect(f.turn("suitors")).toBeNull(); + } finally { await journal.close(); await f.close(); } +}); diff --git a/packages/ui-server/tests/native-brain-leases.test.ts b/packages/ui-server/tests/native-brain-leases.test.ts index ad0a83de5a..0b28ac481d 100644 --- a/packages/ui-server/tests/native-brain-leases.test.ts +++ b/packages/ui-server/tests/native-brain-leases.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; import { Database } from "bun:sqlite"; import { chmodSync, mkdtempSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -27,8 +28,8 @@ test("native UUID ownership is durable before RPC and a second live generation c try { await f.owner.client.checkpoint(ID); expect(beforeRpc, "NATIVE_LEASE_DURABLE_BEFORE_CREATION_RPC").toBe(true); - await expect(second.client.release(ID)).rejects.toThrow("another live"); - await expect(second.client.checkpoint(ID)).rejects.toThrow("another live"); + await expectRejection(second.client.release(ID), "another live"); + await expectRejection(second.client.checkpoint(ID), "another live"); expect(await second.recoverDeadOwners()).toMatchObject({ released: [], retainedLive: 1 }); await second.close(); expect(f.state.query("SELECT count(*) AS n FROM native_root_leases").get()).toEqual({ n: 1 }); @@ -61,12 +62,12 @@ test("lost creation and deletion replies retain ownership; durable deleting can f.native.client.checkpoint = async (...args) => { const value = await checkpoint(...args); if (loseCreate) { loseCreate = false; throw Error("Lost creation receipt"); } return value; }; f.native.client.release = async (...args) => { const value = await release(...args); if (loseDelete) { loseDelete = false; throw Error("Lost deletion receipt"); } return value; }; try { - await expect(f.owner.client.checkpoint(ID)).rejects.toThrow("Lost creation receipt"); + await expectRejection(f.owner.client.checkpoint(ID), "Lost creation receipt"); expect(f.state.query("SELECT state FROM native_root_leases WHERE id=?").get(ID)).toEqual({ state: "creating" }); const second = await createNativeBrainLeaseManager(f.native.client, f.options); try { expect(await second.recoverDeadOwners()).toMatchObject({ retainedLive: 1, released: [] }); - await expect(f.owner.client.release(ID)).rejects.toThrow("Lost deletion receipt"); + await expectRejection(f.owner.client.release(ID), "Lost deletion receipt"); expect(f.state.query("SELECT state FROM native_root_leases WHERE id=?").get(ID)).toEqual({ state: "deleting" }); expect(await second.recoverDeadOwners()).toMatchObject({ released: [ID], unknown: [] }); expect(f.state.query("SELECT count(*) AS n FROM native_root_leases").get()).toEqual({ n: 0 }); @@ -78,8 +79,8 @@ test("lease state never opens the UI database and mismatched store identity cann const f = await fixture(); try { expect(f.state.query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all()).toEqual([{ name: "native_lease_store" }, { name: "native_pool_leases" }, { name: "native_root_leases" }]); - await expect(createNativeBrainLeaseManager(f.native.client, { ...f.options, fsid: "another-store" })).rejects.toThrow("another managed brain"); - await expect(f.owner.client.release(ID)).rejects.toThrow("no recorded checkpoint lease"); + await expectRejection(createNativeBrainLeaseManager(f.native.client, { ...f.options, fsid: "another-store" }), "another managed brain"); + await expectRejection(f.owner.client.release(ID), "no recorded checkpoint lease"); } finally { await f.close(); } }); @@ -96,11 +97,11 @@ test("output stages record ownership before creation and close releases stages b path: join(f.root, "test-stage"), namespacePid: process.pid, namespaceStart: "fixture", appUid: process.geteuid!(), appGid: process.getegid!(), rootDevice: "1", rootInode: "2" }; }; try { - await expect(f.owner.client.prepareStage(OTHER, ID, 4096)).rejects.toThrow("retained sealed"); + await expectRejection(f.owner.client.prepareStage(OTHER, ID, 4096), "retained sealed"); await f.owner.client.checkpoint(ID); await f.owner.client.prepareStage(OTHER, ID, 4096); - await expect(f.owner.client.prepareStage(OTHER, ID, 8192)).rejects.toThrow("another owner or deletion"); - await expect(f.owner.client.release(OTHER)).rejects.toThrow("no recorded checkpoint lease"); - await expect(f.owner.client.release(ID)).rejects.toThrow("stage still depends"); + await expectRejection(f.owner.client.prepareStage(OTHER, ID, 8192), "another owner or deletion"); + await expectRejection(f.owner.client.release(OTHER), "no recorded checkpoint lease"); + await expectRejection(f.owner.client.release(ID), "stage still depends"); expect(releases, "NATIVE_PENDING_STAGE_PINS_LOWER_CHECKPOINT").toEqual([]); await f.owner.close(); expect(releases, "NATIVE_STAGE_CLOSE_RELEASES_LOWER_LAST").toEqual([`stage:${OTHER}`, `checkpoint:${ID}`]); @@ -112,7 +113,7 @@ test("checkpoint source changes refuse admission while retaining the same durabl const f = await fixture(), checkpoint = f.native.client.checkpoint; f.native.client.checkpoint = async (...args) => ({ ...await checkpoint(...args), sourceUuid: "22".repeat(16) }); try { - await expect(f.owner.client.checkpoint(ID)).rejects.toThrow("durable lease identity"); + await expectRejection(f.owner.client.checkpoint(ID), "durable lease identity"); expect(f.state.query("SELECT id,state FROM native_root_leases").all()).toEqual([{ id: ID, state: "creating" }]); await f.owner.client.release(ID); expect(f.state.query("SELECT count(*) AS n FROM native_root_leases").get()).toEqual({ n: 0 }); @@ -148,7 +149,7 @@ test("close drains cancelled pending creation before releasing its recorded UUID await closing; expect(events, "NATIVE_LATE_CREATION_RELEASED_ONLY_AFTER_SETTLEMENT").toEqual(["creating", "created", "released"]); expect(f.state.query("SELECT count(*) AS n FROM native_root_leases").get()).toEqual({ n: 0 }); - await expect(f.owner.client.checkpoint(OTHER)).rejects.toThrow("closed"); + await expectRejection(f.owner.client.checkpoint(OTHER), "closed"); } finally { finish(); await f.close(); } }); @@ -161,12 +162,12 @@ test("unknown stage creation pins its lower root until confirmed stage deletion const second = await createNativeBrainLeaseManager(f.native.client, f.options); try { await f.owner.client.checkpoint(ID); - await expect(f.owner.client.prepareStage(OTHER, ID, 4096)).rejects.toThrow("Lost stage creation receipt"); + await expectRejection(f.owner.client.prepareStage(OTHER, ID, 4096), "Lost stage creation receipt"); expect(f.state.query("SELECT state FROM native_root_leases WHERE id=?").get(OTHER)).toEqual({ state: "creating" }); expect(await second.recoverDeadOwners()).toMatchObject({ retainedLive: 2, released: [] }); - await expect(f.owner.client.release(ID)).rejects.toThrow("stage still depends"); + await expectRejection(f.owner.client.release(ID), "stage still depends"); expect(releases, "NATIVE_UNKNOWN_STAGE_CREATION_PINS_LOWER").toEqual([]); - await expect(f.owner.client.releaseStage(OTHER)).rejects.toThrow("Lost stage deletion receipt"); + await expectRejection(f.owner.client.releaseStage(OTHER), "Lost stage deletion receipt"); expect(f.state.query("SELECT state FROM native_root_leases WHERE id=?").get(OTHER)).toEqual({ state: "deleting" }); expect(await second.recoverDeadOwners()).toMatchObject({ released: [OTHER, ID], unknown: [] }); expect(releases, "NATIVE_RECOVERY_DELETES_UNKNOWN_STAGE_BEFORE_LOWER").toEqual([`stage:${OTHER}`, `stage:${OTHER}`, `checkpoint:${ID}`]); @@ -192,14 +193,14 @@ test("reference pool roots are durable app leases that dead-owner recovery and c const second = await createNativeBrainLeaseManager(f.native.client, f.options); try { await f.owner.client.poolCreate(ID, "source", null, 4096); - await expect(f.owner.client.poolCreate(OTHER, "worktree", "33333333-3333-4333-8333-333333332428", 4096), "REFERENCE_POOL_SNAPSHOT_NEEDS_HELD_SOURCE").rejects.toThrow("held source"); - await expect(f.owner.client.poolCreate(ID, "source", null, 8192), "REFERENCE_POOL_LEASE_IMMUTABLE").rejects.toThrow("another purpose"); + await expectRejection(f.owner.client.poolCreate(OTHER, "worktree", "33333333-3333-4333-8333-333333332428", 4096), "held source", "REFERENCE_POOL_SNAPSHOT_NEEDS_HELD_SOURCE"); + await expectRejection(f.owner.client.poolCreate(ID, "source", null, 8192), "another purpose", "REFERENCE_POOL_LEASE_IMMUTABLE"); expect(await second.recoverDeadOwners(), "REFERENCE_POOL_NOT_AUTOMATICALLY_DELETED").toMatchObject({ released: [], retainedPool: 1 }); await f.owner.close(); expect(calls.filter(call => call.startsWith("release")), "REFERENCE_POOL_CLOSE_RETAINS_ROOTS").toEqual([]); // Pool ownership is app state, not one process generation: a later owner can seal and release it. await second.client.poolSeal(ID); - await expect(second.client.poolRelease(OTHER), "REFERENCE_POOL_UNRECORDED_NOT_DELETED").rejects.toThrow("no recorded app lease"); + await expectRejection(second.client.poolRelease(OTHER), "no recorded app lease", "REFERENCE_POOL_UNRECORDED_NOT_DELETED"); expect((await second.client.poolRelease(ID)).state).toBe("reclaimed"); expect(f.state.query("SELECT COUNT(*) AS count FROM native_pool_leases").get(), "REFERENCE_POOL_LEASE_REMOVED_AFTER_RECEIPT").toEqual({ count: 0 }); expect(calls).toEqual([`create:${ID}`, `seal:${ID}`, `release:${ID}`]); @@ -225,11 +226,11 @@ test("worktree checkpoints, deltas, reads and access moves need this app's held f.native.client.poolPark = async id => { calls.push(`park:${id}`); return record(id, "worktree", SOURCE, "writable"); }; try { await f.owner.client.poolCreate(SOURCE, "source", null, 4096); - await expect(f.owner.client.poolCheckpoint(CHECKPOINT, SOURCE), "REFERENCE_CHECKPOINT_ONLY_OF_A_HELD_WORKTREE").rejects.toThrow("held worktree"); - await expect(f.owner.client.poolAccess(SOURCE, ACCESS), "REFERENCE_ACCESS_ONLY_FOR_A_WORKTREE").rejects.toThrow("held worktree"); + await expectRejection(f.owner.client.poolCheckpoint(CHECKPOINT, SOURCE), "held worktree", "REFERENCE_CHECKPOINT_ONLY_OF_A_HELD_WORKTREE"); + await expectRejection(f.owner.client.poolAccess(SOURCE, ACCESS), "held worktree", "REFERENCE_ACCESS_ONLY_FOR_A_WORKTREE"); await f.owner.client.poolCreate(WORKTREE, "worktree", SOURCE, 4096); - await expect(f.owner.client.poolDelta(CHECKPOINT, CHECKPOINT), "REFERENCE_DELTA_NEEDS_HELD_CHECKPOINTS").rejects.toThrow("held checkpoints"); - await expect(f.owner.client.poolStatCheckpoint(CHECKPOINT, "", 1000), "REFERENCE_CHECKPOINT_READ_NEEDS_HELD_LEASE").rejects.toThrow("held checkpoint"); + await expectRejection(f.owner.client.poolDelta(CHECKPOINT, CHECKPOINT), "held checkpoints", "REFERENCE_DELTA_NEEDS_HELD_CHECKPOINTS"); + await expectRejection(f.owner.client.poolStatCheckpoint(CHECKPOINT, "", 1000), "held checkpoint", "REFERENCE_CHECKPOINT_READ_NEEDS_HELD_LEASE"); await f.owner.client.poolCheckpoint(CHECKPOINT, WORKTREE); await f.owner.client.poolDelta(CHECKPOINT, CHECKPOINT); await f.owner.client.poolStatCheckpoint(CHECKPOINT, "", 1000); @@ -265,11 +266,11 @@ test("a release the pool never committed holds its lease again (#2431)", async ( await f.owner.client.poolCreate(WORKTREE, "worktree", SOURCE, 4096); await f.owner.client.poolCheckpoint(CHECKPOINT, WORKTREE); // The worktree's release fails before the pool committed anything: it is held, and assessable, again. - await expect(f.owner.client.poolRelease(WORKTREE)).rejects.toThrow("dropped"); + await expectRejection(f.owner.client.poolRelease(WORKTREE), "dropped"); expect(lease(WORKTREE), "REFERENCE_POOL_UNCOMMITTED_RELEASE_HELD_AGAIN").toEqual({ state: "held" }); // A release intent a restart left behind settles the same way at the next checkpoint. f.state.query("UPDATE native_pool_leases SET state='releasing' WHERE id=?").run(WORKTREE); - await expect(f.owner.client.poolCheckpoint(NEXT, WORKTREE), "REFERENCE_POOL_INTERRUPTED_RELEASE_ASSESSABLE").resolves.toMatchObject({ id: NEXT }); + expect(await f.owner.client.poolCheckpoint(NEXT, WORKTREE), "REFERENCE_POOL_INTERRUPTED_RELEASE_ASSESSABLE").toMatchObject({ id: NEXT }); expect(lease(WORKTREE)).toEqual({ state: "held" }); } finally { await f.close(); } }); diff --git a/packages/ui-server/tests/storage-removal-journal.test.ts b/packages/ui-server/tests/storage-removal-journal.test.ts index 41f4d1b063..3bfd9666ca 100644 --- a/packages/ui-server/tests/storage-removal-journal.test.ts +++ b/packages/ui-server/tests/storage-removal-journal.test.ts @@ -19,7 +19,7 @@ test("the charged journal retains and verifies selected original bytes before re expect(readFileSync(join(brain, "attachments/chart.pdf"))).toEqual(original); expect(journal.hasRetainedVersion("storage-odysseus", before!)).toBe(true); const fd = openSync(join(root, "independent-copy"), "wx"); - try { journal.withRetainedVersion(before, copy => copy(fd)); } finally { closeSync(fd); } + try { await journal.withRetainedVersion(before, copy => copy(fd)); } finally { closeSync(fd); } expect(readFileSync(join(root, "independent-copy")), "STORAGE_RETAINED_ORIGINAL_CONTROL").toEqual(original); expect(await journal.retainBefore("storage-odysseus", "attachments/ignored.pdf"), "STORAGE_NO_IGNORE_EXPANSION").toBeNull(); rmSync(join(brain, "attachments/chart.pdf")); await journal.finish("storage-odysseus"); diff --git a/packages/ui-server/tests/storage-removal-recovery.test.ts b/packages/ui-server/tests/storage-removal-recovery.test.ts index 41c35f6779..392489bb3f 100644 --- a/packages/ui-server/tests/storage-removal-recovery.test.ts +++ b/packages/ui-server/tests/storage-removal-recovery.test.ts @@ -9,6 +9,7 @@ async function reached(marker:string,child:ReturnType) { } for(const phase of ["after-retain","after-intent","after-unlink","after-ack"])test(`SIGKILL ${phase} preserves the original identity and never replays a removal`,async()=>{ const f=storageRemovalFixture(),id=`odysseus-crash-${phase}`,marker=join(f.root,`reached-${phase}`),target=join(f.brain,"attachments/chart.pdf"); + await f.journal.ready; const child=Bun.spawn([process.execPath,entry,f.root,phase,id,marker],{stdout:"pipe",stderr:"pipe"}); try { await reached(marker,child);child.kill("SIGKILL");await child.exited; @@ -22,11 +23,11 @@ for(const phase of ["after-retain","after-intent","after-unlink","after-ack"])te expect(result.items[0]?.undoAvailable).toBe(phase==="after-ack"); if(phase==="after-ack"){ const undo=await recovered.removal.review("odysseus","odysseus-crash-undo",["attachments/chart.pdf"],id); - recovered.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await recovered.removal.idle(); + await recovered.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await recovered.removal.idle(); expect(readFileSync(target),"STORAGE_CRASH_RETAINED_ORIGINAL_UNDO").toEqual(f.original); } writeFileSync(target,"Penelope's later replacement"); - expect(recovered.removal.start("odysseus",id,result.review.fingerprint).state).toBe("unknown");await recovered.removal.idle(); + expect((await recovered.removal.start("odysseus",id,result.review.fingerprint)).state).toBe("unknown");await recovered.removal.idle(); expect(readFileSync(target,"utf8"),"STORAGE_CRASH_NO_REPLAY_ON_REPLACEMENT").toBe("Penelope's later replacement"); } finally{await recovered.close();} }finally{child.kill("SIGKILL");await child.exited;await f.close();} @@ -37,7 +38,7 @@ test("an actual other-process writer finishes before removal validation while ne const reviewed=await f.removal.review("odysseus",id,["attachments/chart.pdf"]); const child=Bun.spawn([process.execPath,entry,f.root,"writer",id,marker],{stdout:"pipe",stderr:"pipe"}); try { - await reached(marker,child);f.removal.start("odysseus",id,reviewed.review.fingerprint); + await reached(marker,child);await f.removal.start("odysseus",id,reviewed.review.fingerprint); for(let i=0;i<500&&!(f.db.query("SELECT count(*) AS n FROM writer_gate_turn_waiters").get() as {n:number}).n;i++)await Bun.sleep(5); expect((f.db.query("SELECT count(*) AS n FROM writer_gate_turn_waiters").get() as {n:number}).n,"STORAGE_REMOVAL_QUEUED_EXCLUSIVE").toBe(1); expect(readFileSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_REMOVAL_WAITS_FOR_LIVE_WRITER").toEqual(f.original); diff --git a/packages/ui-server/tests/storage-removal-space.test.ts b/packages/ui-server/tests/storage-removal-space.test.ts index a32a6836a0..752df3f611 100644 --- a/packages/ui-server/tests/storage-removal-space.test.ts +++ b/packages/ui-server/tests/storage-removal-space.test.ts @@ -14,13 +14,13 @@ const f=storageRemovalFixture(root); const full=()=>{const fd=openSync(root+"/"+volume+"/occupied","wx");let refused=false;try{for(let i=0;i<4096;i++)writeSync(fd,Buffer.alloc(4096));}catch(error){if(error.code!=="ENOSPC")throw error;refused=true;}finally{closeSync(fd);}if(!refused||statfsSync(root+"/"+volume).bavail!==0)throw Error("STORAGE_SPACE_CONTROL_NOT_FULL");}; try{ if(volume==="objects")full(); -const review=await f.removal.review("odysseus","odysseus-space-0001",["attachments/chart.pdf"]);f.removal.start("odysseus",review.review.id,review.review.fingerprint);await f.removal.idle(); +const review=await f.removal.review("odysseus","odysseus-space-0001",["attachments/chart.pdf"]);await f.removal.start("odysseus",review.review.id,review.review.fingerprint);await f.removal.idle(); if(volume==="objects"){ if(!readFileSync(target).equals(original))throw Error("STORAGE_ENOSPC_ORIGINAL_MUST_SURVIVE"); if(f.removal.receipt("odysseus",review.review.id).items[0].state!=="failed")throw Error("STORAGE_ENOSPC_MUST_REFUSE"); }else{ if(existsSync(target))throw Error("STORAGE_UNDO_SPACE_CONTROL_NOT_REMOVED");full(); -const undo=await f.removal.review("odysseus","odysseus-space-undo",["attachments/chart.pdf"],review.review.id);f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); +const undo=await f.removal.review("odysseus","odysseus-space-undo",["attachments/chart.pdf"],review.review.id);await f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); if(existsSync(target))throw Error("STORAGE_UNDO_ENOSPC_PARTIAL_MUST_NOT_SURVIVE"); if(!f.removal.receipt("odysseus",review.review.id).items[0].undoAvailable)throw Error("STORAGE_UNDO_ENOSPC_RETAINED_ORIGINAL_MUST_SURVIVE"); } diff --git a/packages/ui-server/tests/storage-removal.test.ts b/packages/ui-server/tests/storage-removal.test.ts index 1f9939ba79..97ced25ea4 100644 --- a/packages/ui-server/tests/storage-removal.test.ts +++ b/packages/ui-server/tests/storage-removal.test.ts @@ -1,6 +1,8 @@ import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; import { readFileSync, writeFileSync, existsSync, linkSync, symlinkSync, chmodSync } from "node:fs"; import { join } from "node:path"; +import { journalStatement, type JournalStatement } from "../src/brain/journal-store.js"; import { createTurnJournal } from "../src/brain/turn-journal.js"; import { createWriterGate } from "../src/brain/writer-gate.js"; import { createUiDb } from "../src/db/client.js"; @@ -13,7 +15,7 @@ test("exact removal retains original bytes, keeps references and history facts s const review = await f.removal.review("odysseus", id, ["attachments/chart.pdf"]); expect(review.review.files[0], "STORAGE_REMOVAL_REVIEW_COVERAGE").toMatchObject({ eligible: true, recovery: "requires-retention", references: [{source:"voyage.md",certainty:"resolved"}] }); expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); - expect(f.removal.start("odysseus", id, review.review.fingerprint).state).toBe("waiting"); await f.removal.idle(); + expect((await f.removal.start("odysseus", id, review.review.fingerprint)).state).toBe("waiting"); await f.removal.idle(); const result = f.removal.receipt("odysseus", id); expect(result, "STORAGE_REMOVAL_DURABLE_RESULT").toMatchObject({state:"complete",items:[{state:"removed",undoAvailable:true}],commit:"not-requested",sync:"not-requested",index:"complete"}); expect(existsSync(join(f.brain,"attachments/chart.pdf"))).toBe(false); @@ -22,10 +24,10 @@ test("exact removal retains original bytes, keeps references and history facts s expect(f.calls.filter(call=>call.purpose==="storage-removal-review").every(call=>call.publish===false && call.credentials===false)).toBe(true); const db = createUiDb(f.dbPath), journal = createTurnJournal(db, {brainPath:f.brain,dbPath:f.dbPath,objectsRoot:join(f.root,"objects"),nativeManager:f.native.manager}), reopened = createStorageRemoval(db, {...f.options,journal,gate:createWriterGate(db,f.brain,{pollMs:1})}); try { - expect(reopened.start("odysseus", id, review.review.fingerprint).state, "STORAGE_REMOVAL_NO_REPLAY").toBe("complete"); + expect((await reopened.start("odysseus", id, review.review.fingerprint)).state, "STORAGE_REMOVAL_NO_REPLAY").toBe("complete"); const undo = await reopened.review("odysseus", "odysseus-undo-0001", ["attachments/chart.pdf"], id); expect(undo.review.files[0], "STORAGE_UNDO_CURRENT_REVIEW").toMatchObject({eligible:true,recovery:"retained"}); - reopened.start("odysseus",undo.review.id,undo.review.fingerprint); await reopened.idle(); + await reopened.start("odysseus",undo.review.id,undo.review.fingerprint); await reopened.idle(); expect(reopened.receipt("odysseus",undo.review.id), "STORAGE_UNDO_RECEIPT").toMatchObject({state:"complete",items:[{state:"restored"}]}); expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_UNDO_INDEPENDENT_ORIGINAL_BYTES").toEqual(f.original); } finally { await reopened.close(); await journal.close(); db.close(); } @@ -37,17 +39,17 @@ test("changed bytes keep a mixed batch partial, changed consequences refuse, and try { const review = await f.removal.review("odysseus", "odysseus-mixed-0001", ["attachments/chart.pdf","attachments/score.pdf"]); writeFileSync(join(f.brain,"attachments/score.pdf"),"Athena's revised score"); - f.removal.start("odysseus",review.review.id,review.review.fingerprint); await f.removal.idle(); + await f.removal.start("odysseus",review.review.id,review.review.fingerprint); await f.removal.idle(); expect(f.removal.receipt("odysseus",review.review.id), "STORAGE_REMOVAL_EXACT_REVIEWED_VERSION").toMatchObject({state:"partial",items:[{state:"removed"},{state:"changed"}]}); expect(readFileSync(join(f.brain,"attachments/score.pdf"),"utf8")).toBe("Athena's revised score"); const undo = await f.removal.review("odysseus","odysseus-stale-undo",["attachments/chart.pdf"],review.review.id); writeFileSync(join(f.brain,"attachments/chart.pdf"),"Penelope's replacement"); - f.removal.start("odysseus",undo.review.id,undo.review.fingerprint); await f.removal.idle(); + await f.removal.start("odysseus",undo.review.id,undo.review.fingerprint); await f.removal.idle(); expect(f.removal.receipt("odysseus",undo.review.id).items[0]?.state, "STORAGE_UNDO_NEVER_OVERWRITES").toBe("changed"); expect(readFileSync(join(f.brain,"attachments/chart.pdf"),"utf8")).toBe("Penelope's replacement"); const reference = await f.removal.review("odysseus","odysseus-ref-000001",["attachments/score.pdf"]); writeFileSync(join(f.brain,"new-voyage.md"),"[New score](attachments/score.pdf)\n"); - f.removal.start("odysseus",reference.review.id,reference.review.fingerprint); await f.removal.idle(); + await f.removal.start("odysseus",reference.review.id,reference.review.fingerprint); await f.removal.idle(); expect(f.removal.receipt("odysseus",reference.review.id).items[0], "STORAGE_REMOVAL_CONSEQUENCES_RECHECKED").toMatchObject({state:"changed",reason:"consequences_changed"}); expect(existsSync(join(f.brain,"attachments/score.pdf"))).toBe(true); } finally { await f.close(); } @@ -63,7 +65,7 @@ test("ignored, protected, linked, executable and uncovered attachments never bec f.options.protectedPaths.push("attachments/private.pdf"); writeFileSync(join(f.brain,"attachments/private.pdf"),"protected"); const review = await f.removal.review("odysseus","odysseus-protected-1",["attachments/chart.pdf","attachments/score.pdf","attachments/link.pdf","attachments/exec.pdf","attachments/private.pdf"]); expect(review.review.files.every(file=>!file.eligible), "STORAGE_REMOVAL_PROTECTED_SELECTION").toBe(true); - expect(()=>f.removal.start("odysseus",review.review.id,review.review.fingerprint)).toThrow("Refresh"); + await expectRejection(f.removal.start("odysseus",review.review.id,review.review.fingerprint), "Refresh"); expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); } finally { await f.close(); } }); @@ -73,16 +75,16 @@ test("authority, item-intent storage and missing retained bytes each stop before try { const review = await f.removal.review("odysseus","odysseus-revoked-01",["attachments/chart.pdf"]); const hold = await f.gate.acquire("shared","turn:athena",{turn:true}); - f.removal.start("odysseus",review.review.id,review.review.fingerprint); f.revoke(); hold.release(); await f.removal.idle(); f.allow(); + await f.removal.start("odysseus",review.review.id,review.review.fingerprint); f.revoke(); hold.release(); await f.removal.idle(); f.allow(); expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_CURRENT_AUTHORITY").toEqual(f.original); const blocked = await f.removal.review("odysseus","odysseus-intent-001",["attachments/chart.pdf"]); f.db.exec("CREATE TRIGGER journal.refuse_removal_intent BEFORE UPDATE ON turn_journal_storage_operations WHEN json_extract(NEW.receipt,'$.items[0].state')='applying' BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); - f.removal.start("odysseus",blocked.review.id,blocked.review.fingerprint); await f.removal.idle(); + await f.removal.start("odysseus",blocked.review.id,blocked.review.fingerprint); await f.removal.idle(); expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_DURABLE_ITEM_INTENT").toEqual(f.original); f.db.exec("DROP TRIGGER refuse_removal_intent"); const unavailable = await f.removal.review("odysseus","odysseus-retain-001",["attachments/chart.pdf"]); f.db.exec("CREATE TRIGGER journal.refuse_original BEFORE INSERT ON turn_journal_reservations WHEN NEW.kind='copy' BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); - f.removal.start("odysseus",unavailable.review.id,unavailable.review.fingerprint); await f.removal.idle(); + await f.removal.start("odysseus",unavailable.review.id,unavailable.review.fingerprint); await f.removal.idle(); expect(existsSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_RETAIN_BEFORE_UNLINK").toBe(true); expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); } finally { await f.close(); } @@ -93,9 +95,9 @@ test("cancellation completes the current item acknowledgement and leaves later i try { const review=await f.removal.review("odysseus",id,["attachments/chart.pdf","attachments/score.pdf"]); const write=f.journal.writeOperation.bind(f.journal);let cancelled=false; - f.journal.writeOperation=(apply:()=>T):T=>{const result=write(apply);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(id) as {receipt:string}|null; - if(!cancelled&&row&&JSON.parse(row.receipt).items[0].state==="removed"){cancelled=true;f.removal.cancel("odysseus",id);}return result;}; - f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + f.journal.writeOperation=async(statements:readonly JournalStatement[])=>{const result=await write(statements);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(id) as {receipt:string}|null; + if(!cancelled&&row&&JSON.parse(row.receipt).items[0].state==="removed"){cancelled=true;await f.removal.cancel("odysseus",id);}return result;}; + await f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); expect(f.removal.receipt("odysseus",id),"STORAGE_CANCEL_PRESERVES_ACK_AND_UNSTARTED").toMatchObject({state:"partial",items:[{state:"removed",undoAvailable:true},{state:"not-started"}]}); expect(readFileSync(join(f.brain,"attachments/score.pdf"))).toEqual(f.original); }finally{await f.close();} @@ -104,12 +106,12 @@ test("cancellation completes the current item acknowledgement and leaves later i test("expired detail hides Undo and changed ignore policy cannot expand recovery coverage",async()=>{ const f=storageRemovalFixture(),id="odysseus-expiry-001"; try{ - const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);await f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); const undo=await f.removal.review("odysseus","odysseus-ignore-undo",["attachments/chart.pdf"],id); - writeFileSync(join(f.brain,".gitignore"),"attachments/chart.pdf\n");f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); + writeFileSync(join(f.brain,".gitignore"),"attachments/chart.pdf\n");await f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_UNDO_CURRENT_IGNORE_POLICY").toBe(false); expect(f.removal.receipt("odysseus",undo.review.id).items[0]?.state).toBe("protected"); - f.journal.writeOperation(() => f.db.query("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?").run(`storage_${id}`)); + await f.journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?",`storage_${id}`)]); expect(f.removal.receipt("odysseus",id).items[0],"STORAGE_UNDO_EXPIRY_VISIBLE").toMatchObject({undoAvailable:false,undoReason:"retained_detail_expired_or_unavailable"}); }finally{await f.close();} }); @@ -121,7 +123,7 @@ test("revocation after retained-original admission still refuses the file effect const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]); const retain=f.journal.retainBefore.bind(f.journal); f.journal.retainBefore=async(...args)=>{const original=await retain(...args);f.revoke();return original;}; - f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();f.allow(); + await f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();f.allow(); expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_REMOVAL_AUTHORITY_BEFORE_EFFECT").toBe(true); expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); }finally{await f.close();} @@ -130,11 +132,11 @@ test("revocation after retained-original admission still refuses the file effect test("Undo leases only originals still owned at the atomic retention admission boundary",async()=>{ const f=storageRemovalFixture(),id="odysseus-lease-original"; try{ - const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);await f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); const undo=await f.removal.review("odysseus","odysseus-lease-undo",["attachments/chart.pdf"],id),write=f.journal.writeOperation.bind(f.journal);let expired=false; - f.journal.writeOperation=(apply:()=>T):T=>{const result=write(apply);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(undo.review.id) as {receipt:string}|null; - if(!expired&&row&&JSON.parse(row.receipt).items[0].state==="applying"){expired=true;write(()=>f.db.query("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?").run(`storage_${id}`));}return result;}; - f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); + f.journal.writeOperation=async(statements:readonly JournalStatement[])=>{const result=await write(statements);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(undo.review.id) as {receipt:string}|null; + if(!expired&&row&&JSON.parse(row.receipt).items[0].state==="applying"){expired=true;await write([journalStatement("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?",`storage_${id}`)]);}return result;}; + await f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_UNDO_RETENTION_CHECK_AT_LEASE").toBe(false); }finally{await f.close();} }); diff --git a/packages/ui-server/tests/turn-journal.test.ts b/packages/ui-server/tests/turn-journal.test.ts index 6638e0ae62..ad998650cd 100644 --- a/packages/ui-server/tests/turn-journal.test.ts +++ b/packages/ui-server/tests/turn-journal.test.ts @@ -2,12 +2,14 @@ import { createNativeJournalFixture } from "./helpers/native-journal-fixture.js" import { Database } from "bun:sqlite"; import { createHash } from "node:crypto"; import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; import { closeSync, fstatSync, mkdtempSync, mkdirSync, openSync, rmSync, writeFileSync, readFileSync, statSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createUiDb } from "../src/db/client.js"; import { createNativeJournalCapture } from "../src/brain/native-journal-capture.js"; import { createTurnJournal } from "../src/brain/turn-journal.js"; +import { journalStatement } from "../src/brain/journal-store.js"; import { recordGovernanceEvent, queryGovernanceEvents } from "../src/governance/events.js"; import { createJournalRoutes } from "../src/routes/journal.js"; @@ -78,8 +80,8 @@ test("configuration coverage summary follows retained inventory, independently o await f.journal.finish("coverage"); for (const state of ["complete", "recording", "partial", "expired"] as const) { for (const incomplete of [false, true]) { - f.journal.writeOperation(() => f.db.query("UPDATE turn_journal_turns SET state=?,steering_inventory_missing=?,missing=? WHERE turn_id='coverage'") - .run(state, JSON.stringify(incomplete ? ["workspaces/412/brain.config.ts"] : []), JSON.stringify([{ path: "notes/odysseus.md", reason: "journal_capacity" }]))); + await f.journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET state=?,steering_inventory_missing=?,missing=? WHERE turn_id='coverage'", + state, JSON.stringify(incomplete ? ["workspaces/412/brain.config.ts"] : []), JSON.stringify([{ path: "notes/odysseus.md", reason: "journal_capacity" }]))]); expect(f.journal.summary("coverage"), "COVERAGE_COMES_FROM_CONFIGURATION_INVENTORY") .toHaveProperty("configurationCoverage", incomplete ? "incomplete" : state === "complete" ? "complete" : "unknown"); expect(f.journal.summary("coverage").changedFiles, "COVERAGE_NEVER_CHANGES_PATH_ACCOUNTING").toBe(state === "partial" ? null : 0); @@ -180,13 +182,13 @@ test("native admission refuses unavailable storage and changed-version capacity try { writeFileSync(join(f.brain, "too-large.txt"), Buffer.alloc(2 * 1024 * 1024, 65)); f.native.failNextCheckpoint(Error("native capacity fixture")); - await expect(f.begin("refused")).rejects.toThrow("native capacity fixture"); + await expectRejection(f.begin("refused"), "native capacity fixture"); expect(f.journal.summary("refused").state).toBe("unknown"); const stopped: Error[] = []; await f.journal.begin("running", { sessionId: "ithaca", actor: "voice", model: null, profile: "governed", policyHash: null }, error => stopped.push(error)); expect(f.db.query("SELECT count(*) AS n FROM turn_journal_objects").get(), "NATIVE_BASELINE_RETAINS_NO_UNCHANGED_CONTENT").toEqual({ n: 0 }); writeFileSync(join(f.brain, "too-large.txt"), Buffer.alloc(2 * 1024 * 1024, 66)); - await expect(f.journal.finish("running")).rejects.toThrow(); + await expectRejection(f.journal.finish("running")); expect(stopped.length, "JOURNAL_OVERFLOW_STOPS_TURN").toBeGreaterThan(0); expect(f.journal.details("ithaca", "running")).toMatchObject({ summary: { state: "partial", actor: "voice" }, missing: [{ path: ".", reason: "journal_capacity" }] }); expect(Bun.file(join(f.brain, "too-large.txt")).size, "JOURNAL_PARTIAL_WORK_REMAINS").toBe(2 * 1024 * 1024); @@ -198,7 +200,7 @@ test("failed policy admission ends its journal row and leaves no raw baseline pi try { // Concrete disk input failure after durable admission row: ignore policy is a directory. rmSync(join(f.brain, ".gitignore")); mkdirSync(join(f.brain, ".gitignore")); - await expect(f.begin("refused-manifest")).rejects.toThrow("native journal admission could not be prepared"); + await expectRejection(f.begin("refused-manifest"), "native journal admission could not be prepared"); await f.journal.finish("refused-manifest"); expect(f.db.query("SELECT ended_at FROM turn_journal_turns WHERE turn_id='refused-manifest'").get(), "JOURNAL_REFUSED_ADMISSION_ENDED").toMatchObject({ ended_at: expect.any(Number) }); expect(f.db.query("SELECT count(*) AS n FROM turn_journal_object_refs WHERE turn_id='refused-manifest'").get(), "JOURNAL_REFUSED_BASELINE_PINS_RELEASED").toEqual({ n: 0 }); @@ -238,7 +240,7 @@ test("cancellation while native admission awaits cannot create an active late tu const pending = f.journal.begin("cancelled", { sessionId: "ithaca", actor: "interactive", model: null, profile: "governed", policyHash: null }, () => {}, cancellation.signal); void pending.catch(() => {}); await started; cancellation.abort(Error("Odysseus cancelled admission")); resume(); - await expect(pending).rejects.toThrow("Odysseus cancelled admission"); + await expectRejection(pending, "Odysseus cancelled admission"); expect(f.journal.summary("cancelled").state, "JOURNAL_CANCELLED_ADMISSION_NEVER_RECORDING").toBe("unknown"); expect(f.journal.isHeld()).toBe(false); f.native.manager.checkpoint = checkpoint; @@ -261,7 +263,7 @@ test("cancelling an overlapping admission resumes a healthy peer after its share void pending.catch(() => {}); await started; expect(paused).toBe(true); cancellation.abort(Error("Odysseus cancelled overlapping admission")); resume(); - await expect(pending).rejects.toThrow("cancelled overlapping admission"); + await expectRejection(pending, "cancelled overlapping admission"); expect(completed, "CANCELLED_ADMISSION_PRESERVES_HEALTHY_PEER_HOLD").toEqual([true]); expect(paused).toBe(false); expect(f.journal.summary("cancelled-peer").state).toBe("unknown"); @@ -277,7 +279,7 @@ test("a retained raw stream pins its object until the reader ends", async () => try { await f.begin("odysseus"); writeFileSync(join(f.brain, "notes", "odysseus.md"), "Athena\n"); await f.journal.finish("odysseus"); const change = f.journal.details("ithaca", "odysseus")!.changes[0]!; - const stream = f.journal.version("ithaca", "odysseus", change.sequence, "before")!.stream(); + const stream = await f.journal.version("ithaca", "odysseus", change.sequence, "before")!.stream(); expect(f.db.query("SELECT count(*) AS n FROM turn_journal_reservations WHERE kind='read'").get()).toEqual({ n: 1 }); expect(await new Response(stream.body).text()).toBe("# Ithaca\r\n"); expect(f.db.query("SELECT count(*) AS n FROM turn_journal_reservations WHERE kind='read'").get()).toEqual({ n: 0 }); @@ -330,7 +332,7 @@ test("approved policy writes retain user attribution in the original active wind } await f.journal.finish("odysseus"); await f.journal.finish("athena"); let executed = false; - await expect(f.journal.observeUserPolicy("odysseus", () => { executed = true; })).rejects.toThrow("active complete observation window"); + await expectRejection(f.journal.observeUserPolicy("odysseus", () => { executed = true; }), "active complete observation window"); expect(executed, "JOURNAL_COMPLETED_TURN_NEVER_REVIVED").toBe(false); } finally { await f.close(); } }); @@ -403,11 +405,11 @@ test("runtime main-model binding only updates the correlated active journal", as await f.journal.begin("athena", { sessionId: "troy", actor: "voice", model: null, profile: "governed", policyHash: null }, error => { throw error; }); expect(f.journal.bindModel("odysseus", "troy", "main-model"), "MODEL_WRONG_SESSION_REJECTED").toBe(false); expect(f.journal.bindModel("odysseus", "ithaca", "main-model"), "MODEL_ACTIVE_BINDING").toBe(true); - expect(f.journal.summary("odysseus").model, "MODEL_ACTIVE_SUMMARY").toBe("main-model"); + await f.journal.idle(); expect(f.journal.summary("odysseus").model, "MODEL_ACTIVE_SUMMARY").toBe("main-model"); expect(f.journal.summary("athena").model, "MODEL_CONCURRENT_ISOLATION").toBeNull(); expect(f.journal.bindModel("athena", "ithaca", "wrong-model"), "MODEL_TWO_SESSIONS_CROSS_BIND_REJECTED").toBe(false); expect(f.journal.bindModel("athena", "troy", "voice-model")).toBe(true); - expect(f.journal.summary("athena").model).toBe("voice-model"); + await f.journal.idle(); expect(f.journal.summary("athena").model).toBe("voice-model"); await f.journal.finish("odysseus"); expect(f.journal.bindModel("odysseus", "ithaca", "late-model"), "MODEL_FINISHED_REJECTED").toBe(false); expect(f.journal.summary("odysseus").model).toBe("main-model"); diff --git a/packages/ui-server/tests/turn-revert.test.ts b/packages/ui-server/tests/turn-revert.test.ts index 1962ed31d2..408bc2e20a 100644 --- a/packages/ui-server/tests/turn-revert.test.ts +++ b/packages/ui-server/tests/turn-revert.test.ts @@ -1,5 +1,6 @@ import { createNativeJournalFixture } from "./helpers/native-journal-fixture.js"; import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,6 +8,7 @@ import { createKeyedLock } from "@schlessera/brain-ui-sdk/server"; import { createUiDb } from "../src/db/client.js"; import { createTurnJournal } from "../src/brain/turn-journal.js"; import { createWriterGate } from "../src/brain/writer-gate.js"; +import { journalStatement } from "../src/brain/journal-store.js"; import { createTurnReverts } from "../src/brain/turn-revert.js"; function fixture() { const root=mkdtempSync(join(tmpdir(),"odysseus-revert-")),brain=join(root,"brain"),path=join(root,"ui.db");mkdirSync(brain); @@ -19,26 +21,26 @@ function fixture() { return {root,brain,db,journal,gate,reverts,turn,get refreshes(){return refreshes;},revoke(){authorized=false;},async close(){await journal.close();await native.close();db.close();rmSync(root,{recursive:true,force:true});}}; } test("whole-turn revert journals the user action; Undo replays its observed bytes without invoking Git",async()=>{ - const f=fixture();try {await f.turn("return");expect(()=>f.reverts.start("odysseus","ithaca","return",{id:"odysseus_empty_0001",path:""}),"REVERT_EMPTY_PATH_NEVER_BROADENS_SELECTION").toThrow("No retained eligible paths");const receipt=f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_001"});await f.reverts.idle(receipt.id); + const f=fixture();try {await f.turn("return");await expectRejection(f.reverts.start("odysseus","ithaca","return",{id:"odysseus_empty_0001",path:""}), "No retained eligible paths", "REVERT_EMPTY_PATH_NEVER_BROADENS_SELECTION");const receipt=await f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_001"});await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id,"odysseus","ithaca","return"),"REVERT_COMPLETE_RECEIPT").toMatchObject({state:"complete",paths:[{path:"fleet.bin",state:"restored"},{path:"new.bin",state:"restored"}]}); expect(readFileSync(join(f.brain,"fleet.bin")),"REVERT_ORIGINAL_RAW_BYTES").toEqual(Buffer.from([0,255,10]));expect(existsSync(join(f.brain,"new.bin"))).toBe(false); const own=f.journal.details("ithaca",receipt.journalTurnId)!;expect(own.summary).toMatchObject({actor:"user",kind:"revert",sourceTurnId:"return",state:"complete"});expect(own.changes.every(item=>item.actor==="user")).toBe(true); - const undo=f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_undo_0001"});await f.reverts.idle(undo.id); + const undo=await f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_undo_0001"});await f.reverts.idle(undo.id); expect(readFileSync(join(f.brain,"fleet.bin"))).toEqual(Buffer.from([0,254,13]));expect(readFileSync(join(f.brain,"new.bin"))).toEqual(Buffer.from([0,253]));expect(f.refreshes).toBe(2); }finally{await f.close();} }); test("later bytes conflict while another path restores; original request identity never performs a second effect",async()=>{ const f=fixture();try{await f.turn("return");writeFileSync(join(f.brain,"fleet.bin"),"Athena's later plan"); - const receipt=f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_002"});await f.reverts.idle(receipt.id); + const receipt=await f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_002"});await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id,"odysseus","ithaca","return"),"REVERT_PARTIAL_CONFLICT_RECEIPT").toMatchObject({state:"partial",paths:[{path:"fleet.bin",state:"conflict"},{path:"new.bin",state:"restored"}]}); expect(readFileSync(join(f.brain,"fleet.bin"),"utf8")).toBe("Athena's later plan");expect(existsSync(join(f.brain,"new.bin"))).toBe(false); - writeFileSync(join(f.brain,"new.bin"),"Later new file");expect(f.reverts.start("odysseus","ithaca","return",{id:receipt.id}).state).toBe("partial");expect(readFileSync(join(f.brain,"new.bin"),"utf8"),"REVERT_IDEMPOTENT_NO_SECOND_EFFECT").toBe("Later new file"); - expect(()=>f.reverts.start("odysseus","ithaca","return",{id:receipt.id,path:"fleet.bin"}),"REVERT_IDENTITY_SELECTION_BOUND").toThrow("another request");expect(f.db.query("SELECT kind,actor FROM governance_events WHERE kind='revert_conflict'").get()).toEqual({kind:"revert_conflict",actor:"user"}); + writeFileSync(join(f.brain,"new.bin"),"Later new file");expect((await f.reverts.start("odysseus","ithaca","return",{id:receipt.id})).state).toBe("partial");expect(readFileSync(join(f.brain,"new.bin"),"utf8"),"REVERT_IDEMPOTENT_NO_SECOND_EFFECT").toBe("Later new file"); + await expectRejection(f.reverts.start("odysseus","ithaca","return",{id:receipt.id,path:"fleet.bin"}), "another request", "REVERT_IDENTITY_SELECTION_BOUND");expect(f.db.query("SELECT kind,actor FROM governance_events WHERE kind='revert_conflict'").get()).toEqual({kind:"revert_conflict",actor:"user"}); }finally{await f.close();} }); test("queued revert waits for the running turn; revoked user and voice or other-session sources cannot write",async()=>{ const f=fixture();try{await f.turn("return");const hold=await f.gate.acquire("shared","turn:athena",{turn:true}); - const receipt=f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_003"});expect(receipt.state).toBe("waiting");f.revoke();hold.release();await f.reverts.idle(receipt.id); + const receipt=await f.reverts.start("odysseus","ithaca","return",{id:"odysseus_revert_003"});expect(receipt.state).toBe("waiting");f.revoke();hold.release();await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id,"odysseus","ithaca","return")?.state).toBe("failed");expect(readFileSync(join(f.brain,"fleet.bin"))).toEqual(Buffer.from([0,254,13])); expect(()=>f.reverts.plan("other","return")).toThrow("not available");await f.turn("voice","voice");expect(()=>f.reverts.plan("ithaca","voice")).toThrow("not available"); }finally{await f.close();} @@ -57,18 +59,18 @@ test("authenticated history routes reject absent, delegated, voice, and cross-se test("user restoration recovers observed quarantined bytes without exposing a private quarantine pathname, and Undo removes it",async()=>{ const f=fixture();try{await f.turn("return");const manifest={entries:[]};f.db.query("INSERT INTO governed_turn_manifests (turn_id,brain,pid,pid_start,manifest,started_at,ended_at,reconciliation) VALUES (?,?,?,?,?,?,?,?)").run("return",f.brain,process.pid,"",JSON.stringify(manifest),Date.now()-1,Date.now(),JSON.stringify({turnId:"return",quarantined:[{path:"new.bin",to:join(f.root,"quarantine","held-file")}],changed:[],failed:[]}));rmSync(join(f.brain,"new.bin")); const plan=f.reverts.plan("ithaca","return","restore");expect(plan.paths.find(path=>path.path==="new.bin")).toMatchObject({state:"eligible",quarantined:true});expect(JSON.stringify(plan)).not.toContain(f.root); - const receipt=f.reverts.start("odysseus","ithaca","return",{id:"odysseus_restore_01",path:"new.bin",action:"restore"});await f.reverts.idle(receipt.id);expect(readFileSync(join(f.brain,"new.bin"))).toEqual(Buffer.from([0,253])); - const undo=f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_restore_undo"});await f.reverts.idle(undo.id);expect(existsSync(join(f.brain,"new.bin"))).toBe(false); + const receipt=await f.reverts.start("odysseus","ithaca","return",{id:"odysseus_restore_01",path:"new.bin",action:"restore"});await f.reverts.idle(receipt.id);expect(readFileSync(join(f.brain,"new.bin"))).toEqual(Buffer.from([0,253])); + const undo=await f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_restore_undo"});await f.reverts.idle(undo.id);expect(existsSync(join(f.brain,"new.bin"))).toBe(false); }finally{await f.close();} }); test("expired Undo detail and an interrupted applying receipt never repeat or promise recovery",async()=>{ - const f=fixture();try{await f.turn("return");const receipt=f.reverts.start("odysseus","ithaca","return",{id:"odysseus_expiry_001"});await f.reverts.idle(receipt.id); + const f=fixture();try{await f.turn("return");const receipt=await f.reverts.start("odysseus","ithaca","return",{id:"odysseus_expiry_001"});await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id,"odysseus","ithaca","return")?.undoAvailable).toBe(true); - f.journal.writeOperation(()=>f.db.query("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?").run(receipt.journalTurnId)); + await f.journal.writeOperation([journalStatement("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?",receipt.journalTurnId)]); expect(f.reverts.receipt(receipt.id,"odysseus","ithaca","return")?.undoAvailable,"REVERT_EXPIRED_UNDO_NOT_OFFERED").toBe(false); - f.journal.writeOperation(()=>f.db.query("UPDATE turn_journal_revert_operations SET state='applying',pid=2147483647,pid_start='',error=? WHERE id=?").run("".padEnd(64),receipt.id)); - const recovered=f.reverts.start("odysseus","ithaca","return",{id:receipt.id});expect(recovered).toMatchObject({state:"unknown",undoAvailable:false,error:"interrupted_partial_work"}); + await f.journal.writeOperation([journalStatement("UPDATE turn_journal_revert_operations SET state='applying',pid=2147483647,pid_start='',error=? WHERE id=?","".padEnd(64),receipt.id)]); + const recovered=await f.reverts.start("odysseus","ithaca","return",{id:receipt.id});expect(recovered).toMatchObject({state:"unknown",undoAvailable:false,error:"interrupted_partial_work"}); expect(readFileSync(join(f.brain,"fleet.bin")),"REVERT_INTERRUPTED_IDENTITY_NEVER_REPLAYS").toEqual(Buffer.from([0,255,10])); }finally{await f.close();} }); @@ -98,15 +100,15 @@ test("overlapping actual shell writes retain explicit conflict versions and user expect(versions.map(ref => f.journal.version("ithaca", ref.turnId, ref.sequence, ref.side)!.bytes(1024).toString()), "CONFLICT_OBSERVED_RAW_VERSIONS").toEqual(["Odysseus observed version\n", "Athena observed version\n"]); writeFileSync(join(f.brain, "fleet.bin-remote"), "Penelope's existing sibling\n"); const request = { id: "odysseus_keep_both_01", path: "fleet.bin", action: "keep_both" as const, version: versions[0]!, otherVersion: versions[1]!, expected }; - const receipt = f.reverts.start("odysseus", "ithaca", "shell-a", request); await f.reverts.idle(receipt.id); + const receipt = await f.reverts.start("odysseus", "ithaca", "shell-a", request); await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id, "odysseus", "ithaca", "shell-a"), "CONFLICT_KEEP_BOTH_RECORDED").toMatchObject({ state: "complete", undoAvailable: true }); expect(readFileSync(join(f.brain, "fleet.bin"), "utf8")).toBe("Odysseus observed version\n"); expect(readFileSync(join(f.brain, "fleet.bin-remote"), "utf8"), "CONFLICT_EXISTING_SIBLING_PRESERVED").toBe("Penelope's existing sibling\n"); expect(readFileSync(join(f.brain, "fleet.bin-remote-2"), "utf8")).toBe("Athena observed version\n"); expect(f.journal.details("ithaca", receipt.journalTurnId)!.changes.map(item => item.path)).toEqual(["fleet.bin", "fleet.bin-remote-2"]); expect(f.refreshes).toBe(1); - expect(f.reverts.start("odysseus", "ithaca", "shell-a", request).id).toBe(receipt.id); - const undo = f.reverts.start("odysseus", "ithaca", receipt.journalTurnId, { id: "odysseus_keep_both_undo" }); await f.reverts.idle(undo.id); + expect((await f.reverts.start("odysseus", "ithaca", "shell-a", request)).id).toBe(receipt.id); + const undo = await f.reverts.start("odysseus", "ithaca", receipt.journalTurnId, { id: "odysseus_keep_both_undo" }); await f.reverts.idle(undo.id); expect(readFileSync(join(f.brain, "fleet.bin"), "utf8")).toBe("Athena observed version\n"); expect(existsSync(join(f.brain, "fleet.bin-remote-2"))).toBe(false); } finally { await f.close(); } }); @@ -116,10 +118,10 @@ test("explicit conflict choice refuses later current bytes, forged observations, const details = await observedShellConflict(f), conflict = details.changes.at(-1)!.conflict!; expect(conflict, "CONFLICT_CHOICE_HAS_RETAINED_EVIDENCE").toBeDefined(); const request = { id: "odysseus_keep_version_01", path: "fleet.bin", action: "keep_version" as const, version: conflict.versions[0]!, expected: conflict.latest }; - expect(() => f.reverts.start("odysseus", "other", "shell-a", request)).toThrow(); - expect(() => f.reverts.start("odysseus", "ithaca", "shell-a", { ...request, version: { ...request.version, turnId: "shell-b" } }), "CONFLICT_SELECTION_CANNOT_BORROW_OBSERVATION").toThrow(); + await expectRejection(f.reverts.start("odysseus", "other", "shell-a", request)); + await expectRejection(f.reverts.start("odysseus", "ithaca", "shell-a", { ...request, version: { ...request.version, turnId: "shell-b" } }), undefined, "CONFLICT_SELECTION_CANNOT_BORROW_OBSERVATION"); writeFileSync(join(f.brain, "fleet.bin"), "Penelope's later work\n"); - const receipt = f.reverts.start("odysseus", "ithaca", "shell-a", request); await f.reverts.idle(receipt.id); + const receipt = await f.reverts.start("odysseus", "ithaca", "shell-a", request); await f.reverts.idle(receipt.id); expect(f.reverts.receipt(receipt.id, "odysseus", "ithaca", "shell-a"), "CONFLICT_REVIEWED_CURRENT_RAW_STATE").toMatchObject({ state: "partial", paths: [{ path: "fleet.bin", state: "conflict" }] }); expect(readFileSync(join(f.brain, "fleet.bin"), "utf8")).toBe("Penelope's later work\n"); } finally { await f.close(); } @@ -131,11 +133,11 @@ test("keep-both explicitly covers its final sibling under ignored names and Undo writeFileSync(join(f.brain,".gitignore"),"fleet.bin-remote*\n"); const details=await observedShellConflict(f), conflict=details.changes.at(-1)!.conflict!; const request={id:"odysseus_ignored_both",path:"fleet.bin",action:"keep_both" as const,version:conflict.versions[0]!,otherVersion:conflict.versions[1]!,expected:conflict.latest}; - const receipt=f.reverts.start("odysseus","ithaca","shell-a",request);await f.reverts.idle(receipt.id); + const receipt=await f.reverts.start("odysseus","ithaca","shell-a",request);await f.reverts.idle(receipt.id); const sibling=f.journal.details("ithaca",receipt.journalTurnId)!.changes.find(change=>change.path==="fleet.bin-remote")!; expect(sibling.after,"CONFLICT_IGNORED_SIBLING_EXPLICIT_RAW_COVERAGE").toMatchObject({ignored:false,hash:expect.any(String)}); expect(f.journal.version("ithaca",receipt.journalTurnId,sibling.sequence,"after")!.bytes(1024).toString()).toBe("Athena observed version\n"); - const undo=f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_ignored_undo"});await f.reverts.idle(undo.id); + const undo=await f.reverts.start("odysseus","ithaca",receipt.journalTurnId,{id:"odysseus_ignored_undo"});await f.reverts.idle(undo.id); expect(existsSync(join(f.brain,"fleet.bin-remote"))).toBe(false); } finally {await f.close();} }); diff --git a/packages/ui-server/tests/turn-scope.test.ts b/packages/ui-server/tests/turn-scope.test.ts index fd35d9f5a5..01ad5ee2f4 100644 --- a/packages/ui-server/tests/turn-scope.test.ts +++ b/packages/ui-server/tests/turn-scope.test.ts @@ -363,6 +363,7 @@ test("real host restart kills its paused namespace, records interruption and nev expect(packet.state).toBe("paused"); process.kill(packet.serverPid, "SIGKILL"); await host.exited; await wait(() => { try { return readFileSync(`/proc/${packet.workerPid}/stat`, "utf8").replace(/^.*\) /s, "").startsWith("Z"); } catch { return true; } }); const path = join(root, "ui.db"), db = createUiDb(path), native = createNativeJournalFixture(packet.brain, root), journal = createTurnJournal(db, { nativeManager: native.manager, brainPath: packet.brain, dbPath: path, objectsRoot: join(root, "objects") }); + await journal.ready; const recovered = createTurnScope(db, { brainPath: packet.brain, journal, connected: () => true, changed: () => {} }); try { const receipt = journal.summary("voyage").scope!.receipts[0]!; diff --git a/packages/ui-server/tests/write-worker.test.ts b/packages/ui-server/tests/write-worker.test.ts new file mode 100644 index 0000000000..908184e75b --- /dev/null +++ b/packages/ui-server/tests/write-worker.test.ts @@ -0,0 +1,156 @@ +import { expect, test } from "bun:test"; +import { expectRejection } from "./helpers/expect-rejection.js"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createUiDb } from "../src/db/client.js"; +import { openSqliteWriter } from "../src/db/write-worker.js"; +import * as probe from "./fixtures/write-worker-probe.js"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "write-worker-odysseus-")), path = join(root, "ui.db"), db = createUiDb(path), writer = openSqliteWriter(db); + const log = (table = "odysseus_log") => (db.query(`SELECT label FROM ${table} ORDER BY n`).all() as { label: string }[]).map(row => row.label); + return { root, path, db, writer, log, async close() { await writer.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; +} + +test("a file database commits on the write worker while the event loop keeps running", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + expect(f.writer.offloop, "WRITE_WORKER_OFFLOOP_FOR_FILES").toBe(true); + let last = performance.now(), worst = 0; + const timer = setInterval(() => { const now = performance.now(); worst = Math.max(worst, now - last); last = now; }, 5); + try { expect(await voyage.call("slow", "Ithaca", 400)).toBe(1); } finally { clearInterval(timer); } + // An inline commit holds the loop for the whole 400 ms; the worker's leaves it free. + expect(worst, "WRITE_WORKER_COMMIT_LEAVES_EVENT_LOOP").toBeLessThan(300); + expect(f.log()).toEqual(["Ithaca"]); + } finally { await f.close(); } +}); + +test("every write-worker commit runs at synchronous=FULL and a lowered commit is refused before it lands", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + expect(await voyage.call("level"), "WRITE_WORKER_CONNECTION_FULL").toBe(2); + await voyage.call("log", "Troy"); + await expectRejection(voyage.call("lowered", "Calypso"), "SQLITE_WRITE_BELOW_FULL", "WRITE_WORKER_REFUSES_TRANSACTION_BELOW_FULL"); + expect(f.log(), "WRITE_WORKER_REFUSED_COMMIT_ROLLED_BACK").toEqual(["Troy"]); + await expectRejection(voyage.call("loweredAutocommit", "Circe"), "SQLITE_WRITE_BELOW_FULL", "WRITE_WORKER_REPORTS_AUTOCOMMIT_BELOW_FULL"); + expect(await voyage.call("level"), "WRITE_WORKER_RESTORES_FULL").toBe(2); + await voyage.call("log", "Ithaca"); + expect(f.log().at(-1)).toBe("Ithaca"); + } finally { await f.close(); } +}); + +test("writes reach the database in submission order across channels, and refusals keep their fields", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }), crew = f.writer.channel(probe, { table: "odysseus_log" }); + // Nothing is awaited: the first call holds the worker while the rest queue. + const submitted: string[] = [], calls: Promise[] = [voyage.call("slow", "Troy", 100)]; + submitted.push("Troy"); + for (let index = 0; index < 60; index++) { + const label = `${index % 2 ? "crew" : "voyage"}-${index}`; + submitted.push(label); calls.push((index % 2 ? crew : voyage).call("log", label)); + } + await Promise.all(calls); + expect(f.log(), "WRITE_WORKER_SUBMISSION_ORDER").toEqual(submitted); + const refusal = await expectRejection(voyage.call("refuse", "Penelope waits"), "Penelope waits") as Error & { reason?: string }; + expect(refusal.reason, "WRITE_WORKER_REFUSAL_FIELDS").toBe("odysseus_refused"); + } finally { await f.close(); } +}); + +test("an app-connection transaction that reads, then writes, waits for its turn instead of failing", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + f.db.exec("CREATE TABLE IF NOT EXISTS odysseus_harbour (port TEXT)"); + // The worker commits while the app's transaction sits between its read and its write. + const commit = voyage.call("slow", "Aeolus", 150); + await Bun.sleep(20); + f.db.transaction(() => { + f.db.query("SELECT count(*) FROM odysseus_log").get(); + Bun.sleepSync(300); + f.db.query("INSERT INTO odysseus_harbour VALUES ('Ithaca')").run(); + })(); + await commit; + expect(f.db.query("SELECT port FROM odysseus_harbour").all(), "WRITE_WORKER_APP_TRANSACTION_TAKES_ITS_TURN").toEqual([{ port: "Ithaca" }]); + } finally { await f.close(); } +}); + +test("app writes, autocommit or not, never land while a worker procedure runs", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + const gap = voyage.call<[number, number]>("gap", 300); + await Bun.sleep(50); + f.db.query("INSERT INTO odysseus_log (label) VALUES ('Aeolus')").run(); + f.db.transaction(() => f.db.query("INSERT INTO odysseus_log (label) VALUES ('Circe')").run())(); + f.db.run("INSERT INTO odysseus_log (label) VALUES ('Scylla')"); + const [before, after] = await gap; + expect(after, "WRITE_WORKER_APP_WRITES_WAIT_FOR_THEIR_TURN").toBe(before); + expect(f.log()).toEqual(["Aeolus", "Circe", "Scylla"]); + } finally { await f.close(); } +}); + +test("a read-only app snapshot keeps its deferred mode and never holds the write lock", async () => { + const f = fixture(); + try { + const voyage = f.writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + const other = new Database(f.path); other.exec("PRAGMA busy_timeout = 0"); + try { + // Another connection still writes while the app connection reads its snapshot. + f.db.transaction(() => { + f.db.query("SELECT count(*) FROM odysseus_log").get(); + other.query("INSERT INTO odysseus_log (label) VALUES ('Penelope')").run(); + })(); + expect(f.log(), "WRITE_WORKER_READ_SNAPSHOT_STAYS_DEFERRED").toEqual(["Penelope"]); + } finally { other.close(); } + } finally { await f.close(); } +}); + +test("an attached database file is owned by the same write connection, at FULL", async () => { + const root = mkdtempSync(join(tmpdir(), "write-worker-attach-odysseus-")), path = join(root, "ui.db"), harbourPath = join(root, "harbour.db"); + const db = createUiDb(path); + db.query("ATTACH DATABASE ? AS harbour").run(harbourPath); db.exec("PRAGMA harbour.journal_mode = WAL"); + const writer = openSqliteWriter(db, { attach: [{ schema: "harbour", path: harbourPath }] }); + try { + const voyage = writer.channel(probe, { table: "harbour.odysseus_log" }); await voyage.ready; + expect(await voyage.call("levelOf", "harbour"), "WRITE_WORKER_ATTACHED_FULL").toBe(2); + await voyage.call("log", "Ithaca"); + const separate = new Database(harbourPath, { readonly: true }); + try { expect((separate.query("SELECT label FROM odysseus_log").all() as { label: string }[]).map(row => row.label), "WRITE_WORKER_COMMITS_ATTACHED_FILE").toEqual(["Ithaca"]); } + finally { separate.close(); } + await expectRejection(voyage.call("loweredIn", "harbour", "Calypso"), "SQLITE_WRITE_BELOW_FULL", "WRITE_WORKER_REFUSES_ATTACHED_BELOW_FULL"); + expect(await voyage.call("levelOf", "harbour"), "WRITE_WORKER_RESTORES_ATTACHED_FULL").toBe(2); + expect((db.query("SELECT label FROM harbour.odysseus_log").all() as { label: string }[]).map(row => row.label)).toEqual(["Ithaca"]); + } finally { await writer.close(); db.close(); rmSync(root, { recursive: true, force: true }); } +}); + +test("an in-memory database runs the same procedures inline", async () => { + const db = new Database(":memory:"), writer = openSqliteWriter(db); + try { + const voyage = writer.channel(probe, { table: "odysseus_log" }); await voyage.ready; + expect(writer.offloop).toBe(false); + await Promise.all([voyage.call("log", "Aeolus"), voyage.call("log", "Scylla")]); + expect((db.query("SELECT label FROM odysseus_log ORDER BY n").all() as { label: string }[]).map(row => row.label)).toEqual(["Aeolus", "Scylla"]); + } finally { await writer.close(); db.close(); } +}); + +test("an idle write worker does not keep its process alive", async () => { + const root = mkdtempSync(join(tmpdir(), "write-worker-exit-odysseus-")); + try { + const script = join(root, "idle.ts"); + writeFileSync(script, `import { createUiDb } from ${JSON.stringify(join(import.meta.dir, "../src/db/client.ts"))}; +import { openSqliteWriter } from ${JSON.stringify(join(import.meta.dir, "../src/db/write-worker.ts"))}; +import * as probe from ${JSON.stringify(join(import.meta.dir, "fixtures/write-worker-probe.ts"))}; +const db = createUiDb(${JSON.stringify(join(root, "ui.db"))}), voyage = openSqliteWriter(db).channel(probe, { table: "odysseus_log" }); +await voyage.call("log", "Ithaca"); console.log("committed");\n`); + const child = Bun.spawn([process.execPath, script], { stdout: "pipe", stderr: "pipe" }); + const exited = await Promise.race([child.exited, Bun.sleep(10_000).then(() => "running")]); + if (exited === "running") child.kill(); + expect(exited, "WRITE_WORKER_IDLE_DOES_NOT_HOLD_PROCESS").toBe(0); + expect(await new Response(child.stdout).text()).toBe("committed\n"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-server/tests/ws-activity.test.ts b/packages/ui-server/tests/ws-activity.test.ts index 950f78f342..78843b2ecc 100644 --- a/packages/ui-server/tests/ws-activity.test.ts +++ b/packages/ui-server/tests/ws-activity.test.ts @@ -9,7 +9,8 @@ import { afterEach, describe, expect, test } from "bun:test"; import type { AgentBackend, BackendBridge } from "@schlessera/brain-ui-sdk/server"; import { createStaticBackendRegistry } from "../src/agent/backend"; -import { createActivityStore, type ActivityStore } from "../src/activity/store"; +import { createActivityStore, type ActivityStore, type AwaitableActivityStore } from "../src/activity/store"; +import { ACTIVITY_WRITE_METHODS } from "../src/activity/store-writes"; import { createActivityStream, type ActivityStream } from "../src/activity/stream"; import { createUiDb } from "../src/db/client"; import { createRecordingObservability } from "../src/observability/index"; @@ -62,7 +63,7 @@ function setup( capabilities?: Partial; followUp?: AgentBackend["followUp"]; } = {}, - hostOptions: { toolPermissions?: ToolPermissions } = {} + hostOptions: { toolPermissions?: ToolPermissions; writes?: (store: ActivityStore) => AwaitableActivityStore } = {} ): Setup { const db = createUiDb(":memory:"); const observability = createRecordingObservability(); @@ -77,7 +78,7 @@ function setup( registry: createStaticBackendRegistry([backend], backend.id), catalog: createSessionCatalog(() => db), observability, - activity: { store, stream }, + activity: { store: hostOptions.writes?.(store) ?? store, stream }, ...(hostOptions.toolPermissions ? { toolPermissions: hostOptions.toolPermissions } : {}), @@ -110,6 +111,28 @@ function scriptedTurn(sessionId = "sess-1") { }; } +/** + * The store's writes as the write worker serves them: each resolves after it + * commits. While held, writes queue in order and commit on release. + */ +function heldWrites(store: ActivityStore) { + let holding = false; + const queued: (() => void)[] = []; + const view = Object.create(store) as AwaitableActivityStore; + for (const name of ACTIVITY_WRITE_METHODS) { + Object.defineProperty(view, name, { value: (...args: unknown[]) => new Promise((resolve, reject) => { + const commit = () => { try { resolve(Reflect.apply(store[name] as (...values: unknown[]) => unknown, store, args)); } catch (error) { reject(error); } }; + if (holding || queued.length) queued.push(commit); else commit(); + }) }); + } + return { + view, + hold() { holding = true; }, + release() { holding = false; for (const commit of queued.splice(0)) commit(); }, + pending: () => queued.length, + }; +} + function openAndSubscribe(s: Setup, sub: Record) { const ws = fakeSocket(); s.handlers.onOpen(undefined as never, ws); @@ -968,4 +991,41 @@ describe("activity stream over the ws path", () => { ws.frames().some((f) => f.type === "activity_delta" && f.span?.spanId === "sub-t1" && f.span?.outcome === "cancelled") ); }); + + test("a session's next turn waits until the previous turn's Activity writes commit (#2814)", async () => { + let releaseFirst!: () => void; + const firstHeld = new Promise((resolve) => { releaseFirst = resolve; }); + const prompts: string[] = []; + let pendingAtSecond = -1; + let writes!: ReturnType; + const s = setup(async (request) => { + prompts.push(request.prompt); + if (request.prompt === "second") pendingAtSecond = writes.pending(); + request.bridge.emit({ type: "session_info", sessionId: "sess-settle", isNew: prompts.length === 1 }); + if (request.prompt === "first") { await firstHeld; writes.hold(); } + request.bridge.emit({ type: "result", sessionId: "sess-settle", outcome: "success", durationMs: 1, numTurns: 1, isError: false }); + }, {}, { writes: (store) => (writes = heldWrites(store)).view }); + cleanup = () => { + s.stream.close(); + s.db.close(); + }; + const ws = fakeSocket(); + await s.handlers.onOpen(undefined as never, ws); + s.handlers.onMessage({ data: JSON.stringify({ type: "chat_message", text: "first" }) } as MessageEvent, ws); + await until(() => s.host.coordinator.bySession.has("sess-settle")); + s.handlers.onMessage({ data: JSON.stringify({ type: "chat_message", text: "second", sessionId: "sess-settle" }) } as MessageEvent, ws); + await until(() => ws.frames().some((f) => f.type === "status" && f.status === "queued")); + releaseFirst(); + await until(() => ws.frames().some((f) => f.type === "result")); + await until(() => writes.pending() > 0); + // The first turn has finished, but its terminal writes have not committed. + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(prompts, "ACTIVITY_SLOT_WAITS_FOR_RECORDER_WRITES").toEqual(["first"]); + writes.release(); + await until(() => ws.frames().filter((f) => f.type === "result").length === 2); + expect(prompts).toEqual(["first", "second"]); + expect(pendingAtSecond, "ACTIVITY_NEXT_TURN_AFTER_COMMITTED_WRITES").toBe(0); + const first = ws.frames().find((f) => f.type === "result")!; + expect(s.store.getSpan(`${first.turnId}:turn`)!.outcome).toBe("success"); + }); }); diff --git a/packages/ui-server/tests/ws-brain-application.test.ts b/packages/ui-server/tests/ws-brain-application.test.ts index 1d4b524320..3b03473f72 100644 --- a/packages/ui-server/tests/ws-brain-application.test.ts +++ b/packages/ui-server/tests/ws-brain-application.test.ts @@ -8,6 +8,8 @@ import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { createKeyedLock, type BackendBridge, type BrainApplicationResult } from "@schlessera/brain-ui-sdk/server"; import { createActivityStore } from "../src/activity/store.js"; +import { createOffloopActivityStore } from "../src/activity/store-writes.js"; +import { openSqliteWriter } from "../src/db/write-worker.js"; import { createActivityStream } from "../src/activity/stream.js"; import { createTurnRecorder } from "../src/activity/recorder.js"; import { createUiDb } from "../src/db/client.js"; @@ -73,14 +75,17 @@ test("history retains every maximum-length named effect without event truncation // #2835: the real session bridge waits index readers for queued application // updates, and the end of the turn flushes what is still queued, before its // Activity terminal, with each outcome appended after the application it follows. -async function deferredSession(options: { failIndex?: () => string | undefined; shared?: boolean; turn: (bridge: BackendBridge, index: { runs: string[][] }, search: (query: string) => Promise) => Promise }) { +async function deferredSession(options: { failIndex?: () => string | undefined; shared?: boolean; worker?: boolean; turn: (bridge: BackendBridge, index: { runs: string[][] }, search: (query: string) => Promise) => Promise }) { const root = mkdtempSync(join(tmpdir(), "ws-brain-index-")); cpSync(resolve("packages/core/fixtures/corpus"), root, { recursive: true }); const config = join(root, "brain.config.ts"); writeFileSync(config, readFileSync(config, "utf8").replace('"@schlessera/brain"', JSON.stringify(resolve("packages/core/src/index.ts")))); await inProcessPlanner.index(root, undefined); - const db = createUiDb(":memory:"); - const store = createActivityStore(db, { writer: "application-index-test" }); + // With `worker`, Activity commits on the write worker as the hosted server's do (#2814). + const dbRoot = options.worker ? mkdtempSync(join(tmpdir(), "ws-brain-index-db-")) : undefined; + const db = createUiDb(dbRoot ? join(dbRoot, "ui.db") : ":memory:"); + const writer = dbRoot ? openSqliteWriter(db) : undefined; + const store = writer ? createOffloopActivityStore(db, writer, { writer: "application-index-test" }) : createActivityStore(db, { writer: "application-index-test" }); const stream = createActivityStream(store); const index = { runs: [] as string[][] }; const real = boundaryPlanner(createFixtureHostBoundary({ brainPath: root })); @@ -101,7 +106,7 @@ async function deferredSession(options: { failIndex?: () => string | undefined; authorization.release(); await running; const events = (db.query("SELECT event_type, payload AS payload_json FROM activity_events WHERE event_type LIKE 'brain_application%' ORDER BY rowid").all() as { event_type: string; payload_json: string }[]) .map(e => ({ type: e.event_type, ...JSON.parse(e.payload_json).v })); - return { root, events, search, host, index, close: () => { host.close(); stream.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; + return { root, events, search, host, index, close: async () => { host.close(); stream.close(); await writer?.close(); db.close(); rmSync(root, { recursive: true, force: true }); if (dbRoot) rmSync(dbRoot, { recursive: true, force: true }); } }; } const scheria = (path: string, word: string) => ({ operation: "write" as const, path, content: `---\ntitle: ${path}\ntype: note\n---\nOdysseus meets ${word}.\n`, expectedBaseHash: null }); @@ -131,7 +136,29 @@ test("the real session bridge waits index readers and flushes the rest at turn e // after the run's terminal, so the second outcome proves the flush came first. const order = session.events.map(e => e.type); expect(order.lastIndexOf("brain_application_index")).toBeGreaterThan(order.lastIndexOf("brain_application")); - } finally { session.close(); } + } finally { await session.close(); } +}); + +test("turn-end index outcomes commit on the write worker before the turn's terminal (#2814, #2835)", async () => { + let failure: string | undefined = "maintenance worker timed out"; + const session = await deferredSession({ worker: true, failIndex: () => failure, turn: async bridge => { + expect((await bridge.applyBrain!(scheria("notes/scheria-games.md", "Laodamas"))).ok).toBe(true); + failure = undefined; + expect((await bridge.applyBrain!(scheria("notes/scheria-palace.md", "Arete"))).ok).toBe(true); + failure = "maintenance worker timed out"; + } }); + try { + // Every row was read from the database after the session returned: the + // flush's outcomes reached the worker after their applications and before + // the terminal, which the recorder would otherwise have dropped them behind. + const applications = session.events.filter(e => e.type === "brain_application"); + const indexes = session.events.filter(e => e.type === "brain_application_index"); + expect(applications).toHaveLength(2); + expect(indexes.map(e => e.applicationId), "ACTIVITY_INDEX_OUTCOME_COMMITTED_OFF_LOOP").toEqual(applications.map(e => e.applicationId)); + expect(indexes.every(e => e.indexed === false && e.indexError === "maintenance worker timed out")).toBe(true); + const order = session.events.map(e => e.type); + expect(order.indexOf("brain_application_index")).toBeGreaterThan(order.lastIndexOf("brain_application")); + } finally { await session.close(); } }); test("a failed index update refuses the reader, is on the Activity record, and stays queued past turn end (#2835)", async () => { @@ -155,5 +182,5 @@ test("a failed index update refuses the reader, is on the Activity record, and s failure = undefined; await session.host.indexBatch!.barrier(); expect(await session.search("Laodamas")).toEqual(["notes/scheria-games.md"]); - } finally { session.close(); } + } finally { await session.close(); } });