From 0fb2f74c617fff4f3e4a1a9ef7e124a6c6df48ac Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 13:15:24 +0200 Subject: [PATCH 01/44] test(ci): await complete workspace owner PID publication --- tests/workspace-lease.test.ts | 48 ++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/tests/workspace-lease.test.ts b/tests/workspace-lease.test.ts index 35da67ec03..b4d876ff94 100644 --- a/tests/workspace-lease.test.ts +++ b/tests/workspace-lease.test.ts @@ -15,7 +15,7 @@ afterEach(async () => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); -function fixture(leaseHelper = helper, pauseBeforeRemoval = false) { +function fixture(leaseHelper = helper, pauseBeforeRemoval = false, pauseEntryPublication = false) { const root = mkdtempSync(join(tmpdir(), "brain-workspace-lease-")); dirs.push(root); mkdirSync(join(root, "dist")); writeFileSync(join(root, "package.json"), JSON.stringify({ name: "odysseus-build-fixture", type: "module", exports: "./dist/index.js" })); @@ -29,7 +29,12 @@ import {setTimeout as delay} from "node:timers/promises"; const [root,mode,action,id]=process.argv.slice(2); const coordinated=await ensureWorkspaceLease(root,mode); if(coordinated!==undefined)process.exit(coordinated); -writeFileSync(join(root,id+"-entered"),String(process.pid)); +if(${pauseEntryPublication} && action==="hold") { + writeFileSync(join(root,id+"-entered"),""); + writeFileSync(join(root,id+"-publishing"),"waiting"); + while(!existsSync(join(root,id+"-publish")))await delay(5); +} +writeFileSync(join(root,id+"-entered"),String(process.pid)+"\\n"); if(action==="writer"||action==="hold") { if(action==="writer") { if(${pauseBeforeRemoval})while(!existsSync(join(root,id+"-continue")))await delay(5); @@ -59,17 +64,47 @@ function start(root: string, mode: "read" | "write", action: string, id: string) return { child, stdout: new Response(child.stdout).text(), stderr: new Response(child.stderr).text() }; } -async function waitForMarker(root: string, id: string, phase: "entered" | "removed") { +async function waitForMarker(root: string, id: string, phase: "entered" | "removed" | "publishing") { const deadline = Date.now() + 10_000; - while (!existsSync(join(root, id + "-" + phase))) { + const marker = join(root, id + "-" + phase); + for (;;) { + if (existsSync(marker)) { + if (phase !== "entered") return; + const raw = readFileSync(marker, "utf8"), pid = Number(raw); + // The final newline distinguishes a complete publication from a positive PID prefix. + if (/^[1-9]\d*\n$/.test(raw) && Number.isSafeInteger(pid)) return pid; + } if (Date.now() > deadline) throw new Error(`Timed out waiting for ${id}: ${phase}`); await Bun.sleep(5); } } -const entered = (root: string, id: string) => waitForMarker(root, id, "entered"); +const entered = async (root: string, id: string) => (await waitForMarker(root, id, "entered"))!; const removed = (root: string, id: string) => waitForMarker(root, id, "removed"); const release = (root: string, id: string) => writeFileSync(join(root, id + "-release"), "release\n"); +test("entry readiness waits for the complete positive PID through empty and partial native publication", async () => { + const root = fixture(helper, false, true), other = fixture(); + const owner = start(root, "write", "hold", "owner"); + await waitForMarker(root, "owner", "publishing"); + expect(readFileSync(join(root, "owner-entered"), "utf8")).toBe(""); + let observed: number | undefined; + const readiness = entered(root, "owner").then(() => { observed = Number(readFileSync(join(root, "owner-entered"), "utf8")); }); + const independent = start(other, "read", "reader", "empty-control"); + expect(await independent.child.exited, await independent.stderr).toBe(0); + expect(observed, "LEASE_EMPTY_PID_PUBLICATION_MUST_WAIT").toBeUndefined(); + writeFileSync(join(root, "owner-entered"), "1"); + const partial = start(other, "read", "reader", "partial-control"); + expect(await partial.child.exited, await partial.stderr).toBe(0); + expect(observed, "LEASE_PARTIAL_POSITIVE_PID_MUST_WAIT").toBeUndefined(); + writeFileSync(join(root, "owner-publish"), "publish"); + await readiness; + expect(Number.isSafeInteger(observed)).toBe(true); expect(observed!).toBeGreaterThan(0); + expect(process.kill(observed!, 0)).toBe(true); + const waiter = start(root, "write", "reader", "waiter"); owner.child.kill("SIGTERM"); + expect(await owner.child.exited).not.toBe(0); expect(await waiter.child.exited).toBe(0); + expect(() => process.kill(observed!, 0)).toThrow(); +}); + test("writer removal readiness waits beyond entry notification through a controlled gap", async () => { const root = fixture(helper, true), other = fixture(); const writer = start(root, "write", "writer", "writer"); @@ -135,8 +170,7 @@ test("shared verification overlaps, blocks a writer and rejects a nested build b for (const signal of ["SIGTERM", "SIGKILL"] as const) test(`${signal} of the outer command stops its owner and admits the waiting command`, async () => { const root = fixture(); const owner = start(root, "write", "hold", "owner"); - await entered(root, "owner"); - const ownedPid = Number(readFileSync(join(root, "owner-entered"), "utf8")); + const ownedPid = await entered(root, "owner"); const waiter = start(root, "write", "reader", "waiter"); owner.child.kill(signal); expect(await owner.child.exited).not.toBe(0); From 4028a60e974196f8e3f4eb76a8633c97214018c4 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 14:31:14 +0200 Subject: [PATCH 02/44] docs(work-issue): check stale pull merge refs before confirmation --- .agents/skills/work-issue/references/ci-and-merge.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.agents/skills/work-issue/references/ci-and-merge.md b/.agents/skills/work-issue/references/ci-and-merge.md index acfcfa0dd3..67ea8e4763 100644 --- a/.agents/skills/work-issue/references/ci-and-merge.md +++ b/.agents/skills/work-issue/references/ci-and-merge.md @@ -28,7 +28,11 @@ leaves `proof` nonpassing; convert to draft and mark ready after the final push to request candidate confirmation. Open/reopen ready events also request it. The candidate gate checks actual merge parents/tree against the live head/base. A changed base requires a fresh confirmation event even without a rebase; -valid immutable browser/layout retention remains available. Endurance is fresh +valid immutable browser/layout retention remains available. Read the actual +`refs/pull//merge` commit too: GitHub can retain its old base across a +draft/ready transition. When that ref still has old parents, refresh the branch +against live main before requesting confirmation; another same-head event alone +does not establish a new combined tree. Endurance is fresh and separately mandatory whenever selected. Assess relevant base changes before redoing heavy work; a new base requires confirmation, with valid browser/layout retention avoiding unnecessary execution. Under #1333, a selected browser/layout domain may intentionally skip fresh From cf0c5c1c8e131552f39d4b984068f7953e259891 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 11:35:12 +0200 Subject: [PATCH 03/44] CONTRACT: add bounded human-only Storage inventory and reference inspection --- .changeset/odysseus-storage-server.md | 9 + api-report/core.txt | 11 ++ api-report/ui-sdk.txt | 176 ++++++++++++++++++ api-report/ui-server.txt | 5 + docs/content-index-queries.md | 7 +- docs/decisions/claude-code-runtime.md | 10 +- docs/decisions/http-api-boundary.md | 8 +- docs/decisions/index-query-api.md | 2 +- docs/decisions/live-conversation.md | 2 +- docs/decisions/runtime-requirements.md | 2 +- docs/decisions/scheduled-tasks.md | 2 +- docs/decisions/session-principals.md | 6 +- docs/http-api-coverage.md | 12 +- docs/http-api.md | 70 ++++++- docs/integration-contract/http.md | 15 ++ docs/integration-contract/package-api.md | 16 +- .../live-conversation-investigation.md | 2 +- .../plans/async-collaboration-requirements.md | 2 +- docs/plans/async-collaboration.md | 2 +- docs/plans/navigation-recovery.md | 2 +- docs/plans/storage-accounting.md | 2 +- packages/core/src/queries/index.ts | 12 ++ packages/core/src/queries/snapshot.ts | 8 +- .../core/tests/storage-references.test.ts | 28 +++ packages/ui-backend-claude/src/backend.ts | 2 + packages/ui-backend-pi/src/backend.ts | 1 + packages/ui-sdk/src/protocol.ts | 2 + packages/ui-sdk/src/server/backend.ts | 2 + packages/ui-sdk/src/storage.ts | 54 ++++++ packages/ui-server/README.md | 2 + packages/ui-server/src/agent/backend.ts | 13 +- packages/ui-server/src/app.ts | 32 ++++ .../ui-server/src/brain/host-execution.ts | 3 + packages/ui-server/src/config/env.ts | 14 ++ packages/ui-server/src/core-queries.ts | 11 +- packages/ui-server/src/routes/storage.ts | 36 ++++ packages/ui-server/src/storage/collect.ts | 148 +++++++++++++++ packages/ui-server/src/storage/coordinator.ts | 122 ++++++++++++ packages/ui-server/src/storage/metadata.ts | 98 ++++++++++ packages/ui-server/src/storage/process.ts | 68 +++++++ .../ui-server/tests/backend-registry.test.ts | 17 ++ .../ui-server/tests/http-inventory.test.ts | 2 +- .../tests/http-supported-mounts.test.ts | 5 + .../tests/storage-collection.test.ts | 159 ++++++++++++++++ .../ui-server/tests/storage-metadata.test.ts | 53 ++++++ .../ui-server/tests/storage-routes.test.ts | 57 ++++++ 46 files changed, 1273 insertions(+), 39 deletions(-) create mode 100644 .changeset/odysseus-storage-server.md create mode 100644 packages/core/tests/storage-references.test.ts create mode 100644 packages/ui-sdk/src/storage.ts create mode 100644 packages/ui-server/src/routes/storage.ts create mode 100644 packages/ui-server/src/storage/collect.ts create mode 100644 packages/ui-server/src/storage/coordinator.ts create mode 100644 packages/ui-server/src/storage/metadata.ts create mode 100644 packages/ui-server/src/storage/process.ts create mode 100644 packages/ui-server/tests/storage-collection.test.ts create mode 100644 packages/ui-server/tests/storage-metadata.test.ts create mode 100644 packages/ui-server/tests/storage-routes.test.ts diff --git a/.changeset/odysseus-storage-server.md b/.changeset/odysseus-storage-server.md new file mode 100644 index 0000000000..780931881f --- /dev/null +++ b/.changeset/odysseus-storage-server.md @@ -0,0 +1,9 @@ +--- +"@schlessera/brain": minor +"@schlessera/brain-ui-sdk": minor +"@schlessera/brain-ui-server": minor +"@schlessera/brain-backend-claude": minor +"@schlessera/brain-backend-pi": minor +--- + +Add human-only bounded Storage inventory and reference inspection, with explicit store ownership, deduplicated object and capacity observations, partial coverage, configured backup roots, and backend-owned transcript metadata. This foundation does not register a Storage pane or execute cleanup. diff --git a/api-report/core.txt b/api-report/core.txt index beca8ca30d..44486462d7 100644 --- a/api-report/core.txt +++ b/api-report/core.txt @@ -191,6 +191,7 @@ export "./queries" (./src/queries/index.ts) readGraphMeta readGraphNeighborhood readLinkWalk + readStorageReferences readVoiceVocabulary export "./testing" (./src/testing/index.ts) @@ -1363,6 +1364,16 @@ signatures (every public export and the types it is made of) depth?: number; direction?: "outgoing" | "incoming" | "both"; }): QueryResult; + readStorageReferences (packages/core/src/queries/index.ts) + export function readStorageReferences(opts: Root & { + path: string; + }): QueryResult<{ + sources: Array<{ + path: string; + indexedAt: string; + }>; + truncated: boolean; + }>; readVoiceVocabulary (packages/core/src/queries/index.ts) export function readVoiceVocabulary(opts: Root & { limit: number; diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index 3ef3ac7931..ba08950b33 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -305,6 +305,17 @@ export "." (./src/index.ts) type SharedFileMeta type SpeechCapabilities type StateRoomObservation + type StorageBytes + type StorageCapacityView + type StorageCoverage + type StorageFileEntry + type StorageFilter + type StorageOwner + type StoragePage + type StorageReferenceEvidence + type StorageRoot + type StorageSnapshot + type StorageStatus type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -752,6 +763,17 @@ export "./client" (./src/client/index.ts) type ShowBlockPayload type SpeechCapabilities type StateRoomObservation + type StorageBytes + type StorageCapacityView + type StorageCoverage + type StorageFileEntry + type StorageFilter + type StorageOwner + type StoragePage + type StorageReferenceEvidence + type StorageRoot + type StorageSnapshot + type StorageStatus type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -1354,6 +1376,17 @@ export "./protocol" (./src/protocol.ts) type SharedFileMeta type SpeechCapabilities type StateRoomObservation + type StorageBytes + type StorageCapacityView + type StorageCoverage + type StorageFileEntry + type StorageFilter + type StorageOwner + type StoragePage + type StorageReferenceEvidence + type StorageRoot + type StorageSnapshot + type StorageStatus type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -2027,6 +2060,17 @@ export "./server" (./src/server/index.ts) SpeechTranscriptionError type StartTurnRequest type StateRoomObservation + type StorageBytes + type StorageCapacityView + type StorageCoverage + type StorageFileEntry + type StorageFilter + type StorageOwner + type StoragePage + type StorageReferenceEvidence + type StorageRoot + type StorageSnapshot + type StorageStatus type SubprocessEnvAudience type SubscriptionAuthAction type SurfaceTools @@ -3031,6 +3075,10 @@ signatures (every public export and the types it is made of) brainApplicationPolicy?(req: Pick): import("./brain-application.js").BrainApplicationPolicy; startTurn(req: StartTurnRequest): Promise; followUp?(req: FollowUpRequest): Promise; + storageRoots?(): readonly { + kind: "transcripts" | "archive"; + path: string; + }[]; listSessions(): Promise; getHistory(sessionId: string): Promise; } @@ -9171,6 +9219,134 @@ signatures (every public export and the types it is made of) allowanceBytes: number; observedAt: number; } + StorageBytes (packages/ui-sdk/src/storage.ts) + export interface StorageBytes { + logical: number | null; + allocated: number | null; + coverage: StorageCoverage; + } + StorageCapacityView (packages/ui-sdk/src/storage.ts) + export interface StorageCapacityView { + id: string; + rootIds: string[]; + total: number | null; + used: number | null; + free: number | null; + available: number | null; + provenance: "statfs"; + kind: "tmpfs" | "filesystem" | "unknown"; + identity: "resolved" | "unknown"; + quota: null; + measured: null; + other: null; + reason: string | null; + } + StorageCoverage (packages/ui-sdk/src/storage.ts) + export type StorageCoverage = "complete" | "partial" | "unknown"; + StorageFileEntry (packages/ui-sdk/src/storage.ts) + export interface StorageFileEntry { + id: string; + parentId: string | null; + rootId: string; + path: string; + name: string; + kind: "file" | "directory" | "symlink" | "special"; + objectId: string; + version: string; + capacityId: string | null; + aliasOf: string | null; + logical: number | null; + allocated: number | null; + links: number; + references: { + state: "unchecked"; + count: null; + observedAt: null; + unchecked: string[]; + }; + } + StorageFilter (packages/ui-sdk/src/storage.ts) + export interface StorageFilter { + rootId?: string; + parentId?: string; + owner?: StorageOwner | "all"; + type?: "pdf"; + largerThan?: number; + cursor?: string; + } + StorageOwner (packages/ui-sdk/src/storage.ts) + export type StorageOwner = "brain" | "app" | "operator"; + StoragePage (packages/ui-sdk/src/storage.ts) + export interface StoragePage { + scanId: string; + entries: StorageFileEntry[]; + nextCursor: string | null; + stale: boolean; + fileLogicalTotal: number | null; + matched: number; + totals: StorageBytes; + } + StorageReferenceEvidence (packages/ui-sdk/src/storage.ts) + export interface StorageReferenceEvidence { + entryId: string; + observedAt: number; + state: "observed" | "unavailable"; + sources: Array<{ + path: string; + indexedAt: string; + }>; + truncated: boolean; + newestIndexedAt: string | null; + freshness: "unchecked"; + unchecked: string[]; + } + StorageRoot (packages/ui-sdk/src/storage.ts) + export interface StorageRoot extends StorageBytes { + id: string; + label: string; + category: "content" | "git" | "index" | "database" | "staging" | "scratch" | "workspace" | "temporary" | "transcripts" | "journal" | "native" | "backup"; + owner: StorageOwner; + reason: string | null; + } + StorageSnapshot (packages/ui-sdk/src/storage.ts) + export interface StorageSnapshot { + id: string; + startedAt: number; + observedAt: number; + staleAfter: number; + coverage: StorageCoverage; + consistency: "live-observation"; + basis: "512 * st_blocks; shared extents may be charged more than once"; + roots: StorageRoot[]; + capacities: StorageCapacityView[]; + totals: Record<"all" | StorageOwner, StorageBytes>; + entries: number; + errors: Array<{ + rootId: string; + reason: string; + }>; + omittedErrors: number; + } + StorageStatus (packages/ui-sdk/src/storage.ts) + export interface StorageStatus { + version: 1; + state: "idle" | "scanning"; + activeScanId: string | null; + snapshot: StorageSnapshot | null; + previous: StorageSnapshot | null; + lastFailure: { + scanId: string; + at: number; + reason: string; + } | null; + limits: { + entries: number; + page: number; + roots: number; + milliseconds: number; + cacheBytes: number; + }; + } StoredShare (packages/ui-sdk/src/client/share-store.ts) export interface StoredShare { id: string; diff --git a/api-report/ui-server.txt b/api-report/ui-server.txt index e75b0fb8d3..9eada3803a 100644 --- a/api-report/ui-server.txt +++ b/api-report/ui-server.txt @@ -203,6 +203,7 @@ signatures (every public export and the types it is made of) BackendRegistry (packages/ui-server/src/agent/backend.ts) export interface BackendRegistry { getBackends(): Promise; + peekBackends?(): readonly AgentBackend[] | null; getBackendById(id: string): Promise; getDefaultBackend(): Promise; getDefaultBackendId(): Promise; @@ -848,6 +849,10 @@ signatures (every public export and the types it is made of) const SPAN_OUTCOMES: readonly ["success", "error", "timeout", "cancelled", "denied", "interrupted"]; ServerConfig (packages/ui-server/src/config/env.ts) export interface ServerConfig { + storage?: { + backupRoots: readonly string[]; + gitRoots: readonly string[]; + }; nativeBrain?: { socketPath: string; timeoutMs: number; diff --git a/docs/content-index-queries.md b/docs/content-index-queries.md index 712ad5ba0b..62a2086d2e 100644 --- a/docs/content-index-queries.md +++ b/docs/content-index-queries.md @@ -1,6 +1,6 @@ # Content-index query results -`@schlessera/brain/queries` provides nine synchronous operations over a brain's +`@schlessera/brain/queries` provides ten synchronous operations over a brain's existing disposable `brain.db`. Bun resolves TypeScript source; ordinary `types` and `default` conditions resolve the same build's declarations and JavaScript. The entry does not initialize config, modules, providers, vectors, agents or the @@ -22,10 +22,10 @@ own working directory. HTTP/tool adapters still own authentication and root selection. Document keys are root-relative slash paths: `./notes/ithaca.md` is normalized; empty, absolute, drive-prefixed, backslash, empty-segment, control character and `..` paths are invalid. The API reads indexed keys, not source files; -a missing indexed path is `not_found`. IDs identify nodes in this index/result, +a missing indexed path is `not_found` except Storage reference inspection, which returns an empty observed link list. IDs identify nodes in this index/result, not across rebuilding; retain document paths as durable keys. -A module hygiene check receives the same nine operations already bound to its +A module hygiene check receives the existing nine non-Storage operations already bound to its brain as `ctx.queries` (type `ContentIndexQueries`, exported from this entry), taking these options without `brainPath`. See [modules.md](modules.md#hygiene-checks-read-the-index-through-ctxqueries). @@ -40,6 +40,7 @@ taking these options without `brainPath`. See | `readGraphDiscovery` | `root?: string`, `direction?: "out" \| "both"`, `maxDepth?: number` | `Subgraph`; default precomputed root, out direction, depth 8 clamped 1..8. | | `readGraphMaintenance` | `staleDays?: number`, `now?: string` | `Maintenance`; stale age 180 days clamped 1..3650; optional real UTC ISO instant (`YYYY-MM-DDTHH:mm:ss[.sss]Z`) makes the date-prefix cutoff deterministic, otherwise current time. | | `readLinkWalk` | Required `path: string`; `depth?: number`, `direction?: "outgoing" \| "incoming" \| "both"` | `LinkWalk`; depth 1, both directions; depth clamped 1..4. Unresolved raw link text, cycle/deduplication and touched-node behavior match `brain_graph`. | +| `readStorageReferences` | Required `path: string` | Up to 200 distinct incoming resolved-link sources, ordered by path, with each source’s `indexedAt`; `truncated` reports more. No source bytes are read. The index file is opened read-only with SQLite NOFOLLOW. An empty list does not prove no references; current source changes, unresolved/unindexed links and external consumers remain unchecked. | | `readVoiceVocabulary` | Required positive integer `limit: number` | `VoiceVocabulary`; at most that many ranked terms; extractor version 2, matching current tag/title/path/link/content scoring and stoplists. No pronunciation file or cache access. | | `listIndexDocuments` | `type?: string`, `tag?: string`, `status?: string`, `relevance?: string`, `limit?: number` | `ListedDocument[]`; limit 20 clamped 1..100, updated descending, current listing filter/tag behavior; archived only when `status: "archived"`. Null status is excluded by the default non-archived predicate. | | `findIndexDocuments` | `type?: string`, `excludeStatus?: string`, `pathSuffix?: string` | `DocumentCandidate[]`; complete path-ordered metadata selection, no cap. Literal suffix including `%`, `_` and an optional leading slash, never a LIKE pattern. Supplied exclusion also excludes null status. | diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index 26663c3098..148666332d 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -41,10 +41,10 @@ next to it. `/usr/local/bin/claude` and resolved with `env.CLAUDE_CODE_PATH || "/usr/local/bin/claude"`. Since #213 it has no default (`name: "CLAUDE_CODE_PATH"`, - `packages/ui-server/src/config/env.ts:504-504`) and is null when unset - (`claudeCodePath`, `packages/ui-server/src/config/env.ts:1070`). The whole `agent` block is copied into the backend's module + `packages/ui-server/src/config/env.ts:506-506`) and is null when unset + (`claudeCodePath`, `packages/ui-server/src/config/env.ts:1075`). The whole `agent` block is copied into the backend's module config (`config: { ...agent }`, - `packages/ui-server/src/agent/backend.ts:497`), read back as a string + `packages/ui-server/src/agent/backend.ts:500`), read back as a string (`const claudeCodePath`, `packages/ui-backend-claude/src/module.ts:266-270`) and handed to the SDK (`backend.claudeCodePath`, `packages/ui-backend-claude/src/sdk-options.ts:263`). Because of the `||` @@ -64,7 +64,7 @@ next to it. of this binary. The only version probe in the server is for the `brain` CLI (`Probe the selected content CLI`, `packages/ui-server/src/brain/client.ts:111-177`, called at - `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:381`). + `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:384`). `brain doctor` runs `claude mcp list` from `PATH` (`which("claude")`, `packages/core/src/cli/commands/doctor.ts:495-499`) — the user's own Claude Code on their own machine, to check the MCP registration, not the server's @@ -212,7 +212,7 @@ unnecessary. `node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs`. The boot probe below turns either into a refusal to start, for the same reason a missing backend refuses to boot (`A missing (or unrecognized) agent backend`, - `packages/ui-server/src/app.ts:326-331`). + `packages/ui-server/src/app.ts:329-334`). - **The published range is still a caret, and that bounds what this repo can guarantee.** `@schlessera/brain-backend-claude` depends on `^0.3.241`, and a host resolves it in its own lockfile. A host can bump the SDK — and so the diff --git a/docs/decisions/http-api-boundary.md b/docs/decisions/http-api-boundary.md index 3d36aba90f..4fc3e7883f 100644 --- a/docs/decisions/http-api-boundary.md +++ b/docs/decisions/http-api-boundary.md @@ -46,11 +46,11 @@ explicit supported rows, not silently excluded by this distinction. The source audit on 2026-09-30 used `e0875269b21b707482545be9e94901acb897893b`. The public health mount precedes auth -(`createHealthRoutes`, `packages/ui-server/src/app.ts:726`), the API guard follows -public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:746`), and the -WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:872`). +(`createHealthRoutes`, `packages/ui-server/src/app.ts:729`), the API guard follows +public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:749`), and the +WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:903`). Static serving is conditional (`if (options.staticRoot)`, -`packages/ui-server/src/app.ts:892-913`). That order establishes access; +`packages/ui-server/src/app.ts:923-944`). That order establishes access; it does not select compatibility status. A hermetic real-app launch, with temporary brain root, in-memory database, diff --git a/docs/decisions/index-query-api.md b/docs/decisions/index-query-api.md index 6baa0c374d..fe22e51931 100644 --- a/docs/decisions/index-query-api.md +++ b/docs/decisions/index-query-api.md @@ -118,7 +118,7 @@ neither justified a service or configurable query-provider seam. > **2026-10-05 — Implementation context (#697).** UI-server now declares the > optional peer and resolves the entry lazily, once per app -> (`createCoreQueryAccess`, `packages/ui-server/src/core-queries.ts:187-220`). The +> (`createCoreQueryAccess`, `packages/ui-server/src/core-queries.ts:192-227`). The > source-backed range is `>=0.40.0 <1.0.0`: the published 0.40.0 manifest is the > first to export `./queries`, its declarations name all nine operations and its > query source matches the source the server was migrated against; 0.39.0 has no diff --git a/docs/decisions/live-conversation.md b/docs/decisions/live-conversation.md index c80c196311..4fe837cc34 100644 --- a/docs/decisions/live-conversation.md +++ b/docs/decisions/live-conversation.md @@ -30,7 +30,7 @@ tool admission and execution, permissions, cancellation, terminal state and resynchronization. Engine task requests are advisory until mapped to committed user input. No native search, code-execution or alternate permission loop is enabled. Both adapters can delegate through the existing `AgentBackend` -(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:634-664`); +(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:634-666`); the host serializes work in a session and uses capability-checked follow-ups. The [voice-permission decision](voice-permission.md) binds this interface. diff --git a/docs/decisions/runtime-requirements.md b/docs/decisions/runtime-requirements.md index 369fd899b4..573a86c155 100644 --- a/docs/decisions/runtime-requirements.md +++ b/docs/decisions/runtime-requirements.md @@ -274,7 +274,7 @@ reciprocal host-version negotiation is involved. | `probeBrainCliVersion` / `brainCliCommand` (`export async function probeBrainCliVersion`, `packages/ui-server/src/brain/client.ts:112-177`) | Reuse executable selection, wrapper, deadline and cleanup; add host floor composition, strict comparison and invocation verification without duplicating the 0.33.0 constant. | | `installedAgentSdkVersion` (`export function installedAgentSdkVersion`, `packages/ui-backend-claude/src/runtime-probe.ts:55-61`) | Retain importer-relative resolution; validate package name/version and the owner's manifest range. | | `selectedSpawn` / `probeClaudeRuntime` (`export function selectedSpawn`, `packages/ui-backend-claude/src/runtime-probe.ts:69-96`; `export async function probeClaudeRuntime`, `packages/ui-backend-claude/src/runtime-probe.ts:127-147`) | Keep SDK-selected command capture, native/JS override argv, environment, timeout and group cleanup; add strict identity validation and composed runtime floors. | -| `probeBackendRuntimes` (`export async function probeBackendRuntimes`, `packages/ui-server/src/agent/backend.ts:390-437`) | Forward validated host requirements; retain active-backend selection and separate unmeasured-pair logging. | +| `probeBackendRuntimes` (`export async function probeBackendRuntimes`, `packages/ui-server/src/agent/backend.ts:392-439`) | Forward validated host requirements; retain active-backend selection and separate unmeasured-pair logging. | | `BackendModuleContext` / `BackendRuntimeReport` (`export interface BackendModuleContext`, `packages/ui-sdk/src/server/backend-module.ts:96-109`; `export interface BackendRuntimeReport`, `packages/ui-sdk/src/server/backend-module.ts:116-129`) | Extend the existing construction context for optional minimums. Keep report identities and `measured` meanings; a compatibility result must never occupy `measured.matches`. | | `MEASURED_RUNTIME`, its guard and `scripts/measure-claude-runtime.ts` | Keep the constant, exact pair guard, keyless behavior probe and dated measurement receipts. A permitted pair can still be unmeasured. | | Turn `reportRuntime` / `init` (`const reportRuntime`, `packages/ui-backend-claude/src/turn-runner.ts:158-165`; `if (msg.type === "system"`, `packages/ui-backend-claude/src/turn-runner.ts:532-547`) | Preserve actual per-turn SDK/runtime observations, billing/permission guards and normal failure terminal behavior. | diff --git a/docs/decisions/scheduled-tasks.md b/docs/decisions/scheduled-tasks.md index 66d138d9b2..5292efd6a7 100644 --- a/docs/decisions/scheduled-tasks.md +++ b/docs/decisions/scheduled-tasks.md @@ -23,7 +23,7 @@ Source inspected at `b5c30c180df5584584f261b446c4d440380d2441`: (`else if (item.status === "scheduled"`, `packages/ui-server/src/inbox/actions.ts:188-189`). The runtime ticks every minute (`INBOX_TICK_MS`, `packages/ui-server/src/inbox/runtime.ts:11-13`). - Production app wiring supplies recovery and budgets, without a dispatcher - (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:398-414`). + (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:401-417`). A scripted nonempty ready-item test still reports no claim, attempt or Activity root. That proves disabled wiring, not safe execution of a schedule. - Headless plumbing checks backend support, a usable principal, lifetime and diff --git a/docs/decisions/session-principals.md b/docs/decisions/session-principals.md index 5809a889e8..019e799eea 100644 --- a/docs/decisions/session-principals.md +++ b/docs/decisions/session-principals.md @@ -96,9 +96,9 @@ Both reviewers verified every row below against the source. | Cookie verify | `auth.ts:265-296` | strict `^(\d+)\.(\d+)$`, safe integers, future/expired rejection, `cookieEpoch === sessionsEpoch(db)` | | Global epoch | `auth.ts:302-318` | `settings` row; missing = 0; non-integer **throws** | | Global revoke | `auth.ts:321-330` | `setSetting` (overflow-guarded) + `clients.closeAll(1008, …)` | -| Auth routes | `authRoutes`, `app.ts:732` | mounted **before** the guard at `authGuard(authMode`, `app.ts:746`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | -| Passkey management | `passkeyManagementRoutes`, `app.ts:756` | mounted **after** the guard ("Mount AFTER the auth guard") | -| WS guard | `"/ws"`, `app.ts:872-889` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | +| Auth routes | `authRoutes`, `app.ts:735` | mounted **before** the guard at `authGuard(authMode`, `app.ts:749`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | +| Passkey management | `passkeyManagementRoutes`, `app.ts:759` | mounted **after** the guard ("Mount AFTER the auth guard") | +| WS guard | `"/ws"`, `app.ts:903-920` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | | WS upgrade | `ws/connection.ts:262-264` | `createWsUpgrade` ignores the request context | | WS admission | `ws/connection.ts:84` | `clients.add(ws)` — the socket's identity is unknown | | Turn record | `ws/run-session.ts:134-142` | recorder built from `{turnId, sessionId, billing}`; `RunningTurn` (`ws/turns.ts:55-77`) has no actor | diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index bd17379fcc..5733508b02 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:112-145`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:117-150`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -28,6 +28,11 @@ row's real mounting check. They do not establish mounting by themselves. | Supported operation | Mounted receipt | Detailed checks | | --- | --- | --- | +| `GET /api/storage` | `mounted supported handler: GET /api/storage` | ST | +| `POST /api/storage/scans` | `mounted supported handler: POST /api/storage/scans` | ST | +| `DELETE /api/storage/scans/:id` | `mounted supported handler: DELETE /api/storage/scans/:id` | ST | +| `GET /api/storage/scans/:id/entries` | `mounted supported handler: GET /api/storage/scans/:id/entries` | ST | +| `GET /api/storage/scans/:id/entries/:entryId/references` | `mounted supported handler: GET /api/storage/scans/:id/entries/:entryId/references` | ST | | `GET /api/activity/rollups` | `mounted supported handler: GET /api/activity/rollups` | R | | `GET /api/activity/runs` | `mounted supported handler: GET /api/activity/runs` | R | | `GET /api/activity/runs/:runId` | `mounted supported handler: GET /api/activity/runs/:runId` | R | @@ -455,3 +460,8 @@ files. `revert-gate.test.ts` proves old-turn maintenance/new-turn queue order; `revert-files.test.ts` proves binary/mode comparison and descriptor parent identity. The real Chrome `turn-journal-runtime.test.ts` exercises whole-turn review, reading focus/Escape, waiting and partial receipts, Undo and lost-response readback. + +ST: [Storage route authorization and mounted scans](../packages/ui-server/tests/storage-routes.test.ts), +[real CLI/store union and paging](../packages/ui-server/tests/storage-collection.test.ts), +[exact metadata roots](../packages/ui-server/tests/storage-metadata.test.ts), and +[bounded index evidence](../packages/core/tests/storage-references.test.ts). diff --git a/docs/http-api.md b/docs/http-api.md index faf9722941..7f20105961 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 135 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 140 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -26,6 +26,11 @@ client code has a gap. Source owners are listed after the table. | Method | Path | Class | Purpose | Consumers, existing promise and reason | | --- | --- | --- | --- | --- | +| GET | `/api/storage` | S | Read current bounded inventory state | Human-owner Storage inspection; no delegated or worker access. | +| POST | `/api/storage/scans` | S | Refresh configured stores | Human-owner Storage inspection; no delegated or worker access. | +| DELETE | `/api/storage/scans/:id` | S | Cancel an active scan | Human-owner Storage inspection; no delegated or worker access. | +| GET | `/api/storage/scans/:id/entries` | S | Read a filtered bounded page | Human-owner Storage inspection; no delegated or worker access. | +| GET | `/api/storage/scans/:id/entries/:entryId/references` | S | Inspect bounded indexed reference evidence | Human-owner Storage inspection; no delegated or worker access. | | GET | `/api/activity/digest` | I | Read digest and record app visit | React Activity presentation state; retained public client functions do not promise independent raw transport. | | POST | `/api/activity/digest/dismiss` | I | Record digest dismissal | React Activity presentation state; retained public client functions do not promise independent raw transport. | | POST | `/api/activity/digest/generate` | I | Generate digest manually | React Activity presentation state; retained public client functions do not promise independent raw transport. | @@ -168,7 +173,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 135 declared endpoints are mounted regardless of backend, renderer or +All 140 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1210,3 +1215,64 @@ These actions reuse the existing queue, identity-bound readback, partial receipts, journal and user Undo. No additional endpoint, automatic merge or agent permission is introduced. The reader calls versions observed, never exclusive "mine" or "theirs" ownership. + +## Storage inventory + +These five operations require a current `owner` or `ambient` principal. Agent, +delegated and restricted access is refused (403); unauthenticated/revoked access +is refused (401 by normal authentication). Authorization is checked again at +coordinator access and after asynchronous inspection. Responses are private, +no-store. Mutations retain the existing origin and JSON body protections. + +`GET /api/storage` returns SDK `StorageStatus` version 1 without starting work: +state, active scan ID, nullable latest/previous observations, last failure and +limits. `POST /api/storage/scans` accepts only `{}` and returns 202 with status. +It coalesces one scan per app instance; a new scan is limited to once every five +seconds. Roots, credentials and scan budgets are never request parameters. +`DELETE /api/storage/scans/:id` returns 202 while cancellation is being reaped; +an inactive identity returns 409. Initiator revocation cancels its scan. + +`GET /api/storage/scans/:id/entries` accepts optional `rootId`, `parentId` +(empty means root rows), `owner=all|brain|app|operator`, `type=pdf`, nonnegative +integer `largerThan` and opaque `cursor`. PDF matching is filename based and +case insensitive; byte thresholds are strictly greater than. A page has at most +200 entries, a next cursor, stale flag, match count and object-union subtotals. +Cursors bind the scan and filters and expire after 60 seconds; changed/expired +cursors return 409, malformed cursors/filters 400. Unknown/evicted scans return +404. Cache retains at most two observations and 96 MiB serialized data, each +at most 48 MiB, for at most ten minutes. One scan admits at most 100,000 entries, +32 roots and 35 seconds of collection; child cancellation is awaited. Kernel +uninterruptible I/O cannot have a hard wall-clock completion guarantee. + +The server invokes the installed host core package’s public CLI with an explicit +root, no executable brain configuration or inherited secrets, and the configured +exec wrapper. It never launches the brain repository’s selected executable on +the host. Missing/incompatible inventory support produces partial/unavailable +evidence rather than weakening that boundary. Metadata workers inspect only +exact configured stores, reject redirects below roots and mount crossings, and +never read attachment/database/backup contents for sizing. Backend setup is not +triggered by inventory. Explicit host backup/Git roots are described in server +configuration; no remote or host-wide filesystem discovery is performed. + +Root IDs, object IDs and versions are opaque per app lifetime. Absolute paths, +configuration contents and credentials are excluded. All/Brain/App/Operator +are overlapping ownership views over an object union, never additive totals. +Filesystem identity reconciles shared capacities; unsupported equivalence stays +unknown. Capacity rows are never summed. `measured` and `other` remain null +because shared extents do not establish exclusive attribution. Statfs caller +availability is not quota headroom. Native retained stores and private temporary +usage remain explicit unknown rows where their owners cannot provide a safe +measurement. Limits, permissions, unsupported metadata and live changes preserve +partialness; unknown values never become zero or a logical-size fallback. + +`GET /api/storage/scans/:id/entries/:entryId/references` inspects a retained +canonical file observation through the supported core query API. One inspection +per app runs at a time, with a five-second cancellation deadline and 1 MiB output +cap. The result is SDK `StorageReferenceEvidence`: up to 200 indexed source paths +and index timestamps, truncation, observation time, and explicit unchecked scope. +Index timestamps do not prove current source freshness. Unavailable index support +returns unavailable evidence; an empty observed list never proves no references. +No file body is read for inventory, type filtering or this index inspection. + +No endpoint authorizes deletion, expiry, maintenance or recovery. The complete +Storage pane is delivered separately with its approved removal journey. diff --git a/docs/integration-contract/http.md b/docs/integration-contract/http.md index 73323ac9ef..0ac153a134 100644 --- a/docs/integration-contract/http.md +++ b/docs/integration-contract/http.md @@ -481,3 +481,18 @@ Both effects remain user journal entries, with existing conflict-safe Undo. Partial writes or unavailable storage remain partial/unknown, not an atomic merge guarantee. Existing owner/ambient, voice, expired-detail, protected-path and cross-session restrictions remain binding. + +### Human-only Storage inventory (additive, #2010) + +The five [Storage inventory operations](../http-api.md#storage-inventory) and +SDK `StorageStatus`, `StorageSnapshot`, `StoragePage`, `StorageFileEntry`, +`StorageReferenceEvidence` and related types are additive contracts. They bind +authorization, configured-root scope, bounded output/lifetime, opaque cursors, +object/capacity deduplication and explicit partial/unknown semantics. This +transport foundation does not register a pane or authorize cleanup. + +Storage pages distinguish `fileLogicalTotal` (logical bytes per matching regular-file +entry, including aliases) from `totals` (the deduplicated physical-object union). +Empty unavailable ownership views carry null bytes, not zero. Capacity `kind` +identifies `tmpfs` as memory-backed; `filesystem` does not promise exclusive disk +ownership and `unknown` cannot be drawn as healthy capacity. diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index 2c7064f575..a6b5a4c1d7 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -734,7 +734,11 @@ records the approved request boundary and override policy. The concrete synchronous `@schlessera/brain/queries` entry supports `readGraphMeta`, `readGraphClusters`, `readGraphNeighborhood`, `readGraphDiscovery`, `readGraphMaintenance`, `readLinkWalk`, -`readVoiceVocabulary`, `listIndexDocuments` and `findIndexDocuments`. +`readVoiceVocabulary`, `listIndexDocuments`, `findIndexDocuments` and +`readStorageReferences`. The additive Storage operation takes `{brainPath, path}` +and returns the common query envelope with `{sources: [{path, indexedAt}], truncated}`. +It limits resolved incoming sources to 200, reads index metadata only, and refuses +a symlinked index. Zero observed sources never establishes that a file is unused. Its [complete API specification](../content-index-queries.md) is part of this contract: signatures/reachable exports, validation/defaults/caps/order, detached result and safe error envelopes, feature compatibility, read-only @@ -3196,3 +3200,13 @@ explanation. `showActions={false}` and closed exchanges still omit actions. The client preserves selected predefined answers beside custom text in a multi-select and the existing grouped-question submission path. No wire, CLI/MCP or answer-delivery contract changes. + +### Backend Storage roots (additive, #2010) + +`AgentBackend.storageRoots?()` is a synchronous host-only observation of already +owned transcript/archive roots: `readonly {kind: "transcripts" | "archive"; +path: string}[]`. It performs no setup or provider discovery. The server consults +only already initialized owners, never passes these paths to a worker bridge, +and converts them into opaque root IDs before transport. Older backends remain +valid and report unavailable transcript accounting. Claude supplies its exact +brain-specific live/archive leaves; Pi supplies its already resolved session root. diff --git a/docs/investigations/live-conversation-investigation.md b/docs/investigations/live-conversation-investigation.md index 6df9bd47d3..27cbb957cf 100644 --- a/docs/investigations/live-conversation-investigation.md +++ b/docs/investigations/live-conversation-investigation.md @@ -239,7 +239,7 @@ Backend reuse is structurally feasible for **both** adapters. `startTurn` is already asynchronous and accepts a host AbortSignal. A busy same-session turn is queued; new committed details use `followUp` only when the backend advertises it, otherwise remain queued/reviewable -(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:634-664`). +(`export interface AgentBackend`, `packages/ui-sdk/src/server/backend.ts:634-666`). Continued voice interaction never requires unsafe concurrent `startTurn`s. This is source evidence and design mapping, not a real provider/backend trial. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index ad0d90cf0c..e36aad82a5 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -226,7 +226,7 @@ Stated before the requirements because five of them derive from it. that is the shape to copy, including its `close()` lifecycle. - R19. **The cron backstop has independent authorization before the general guard.** Mount it on the existing listener before - (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:746`). + (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:749`). Authorize a boot-minted ephemeral token, rotated each boot and stored in a 0600 runtime file, with the actual socket address as an additional check. Proxy headers cannot authorize it. The poke succeeds in every auth mode diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 1b4a0e3735..1cbcfe5f7b 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -131,7 +131,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. owns manual triggers/history; it does not tick the inbox. - **Two connections:** (`export function createUiDb(`, `packages/ui-server/src/db/client.ts:25-37`) sets WAL, foreign keys and a 5-second busy timeout. Claims are immediate transactions. -- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:746`) follows public routes; +- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:749`) follows public routes; (`export function authGuard(`, `packages/ui-server/src/middleware/auth.ts:189-249`) binds principals in each auth mode. An internal poke needs independent token authorization before this guard. - **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:271-365`). Timeout unwind is diff --git a/docs/plans/navigation-recovery.md b/docs/plans/navigation-recovery.md index dd3eaec7f2..6819f745c2 100644 --- a/docs/plans/navigation-recovery.md +++ b/docs/plans/navigation-recovery.md @@ -397,7 +397,7 @@ implementation. Any PR moving these lines owns their citation repairs. - Activity resolution: (`.get("/activity/runs", (c) => {`, `packages/ui-server/src/routes/activity.ts:142-275`). -- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:746`). +- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:749`). - Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:878-922`). diff --git a/docs/plans/storage-accounting.md b/docs/plans/storage-accounting.md index 30937832d6..c3edf3ed5e 100644 --- a/docs/plans/storage-accounting.md +++ b/docs/plans/storage-accounting.md @@ -67,7 +67,7 @@ Source anchors, checked against the reviewed commit: The completed #94 and #398 provide existing stats/maintenance foundations. - C: snapshot fields (`snapshotOf`, `packages/core/src/lib/stats-history.ts:132-172`), thinning (`thin`, `packages/core/src/lib/stats-history.ts:232-244`). -- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:252-287`), +- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:255-290`), WAL store (`createUiDb`, `packages/ui-server/src/db/client.ts:25-37`), and private locations (`statePaths`, `packages/ui-server/src/brain/operational-state.ts:107-113`). - E: live/archive paths (`transcriptPaths`, `packages/ui-backend-claude/src/transcript-archive.ts:29-38`). diff --git a/packages/core/src/queries/index.ts b/packages/core/src/queries/index.ts index bc45948398..2995d208e0 100644 --- a/packages/core/src/queries/index.ts +++ b/packages/core/src/queries/index.ts @@ -160,3 +160,15 @@ export function findIndexDocuments(opts: Root & { function derived(version: number): void { if (version < 8) throw new QueryFailure("incompatible_index"); } + +/** Bounded incoming resolved-link evidence; absence never proves an attachment unused. */ +export function readStorageReferences(opts: Root & { path: string }): QueryResult<{ sources: Array<{ path: string; indexedAt: string }>; truncated: boolean }> { + let path = ""; + return withSnapshot(opts, "storage", () => { path = relativePath(opts.path); }, db => { + const rows = db.query(`SELECT DISTINCT source.path AS path, source.indexed_at AS indexedAt + FROM links JOIN documents source ON source.id=links.source_id + JOIN documents target ON target.id=links.target_id + WHERE target.path=? ORDER BY source.path LIMIT 201`).all(path) as Array<{ path: string; indexedAt: string }>; + return { sources: rows.slice(0, 200), truncated: rows.length > 200 }; + }); +} diff --git a/packages/core/src/queries/snapshot.ts b/packages/core/src/queries/snapshot.ts index dd5ca3fc54..e64e4485ec 100644 --- a/packages/core/src/queries/snapshot.ts +++ b/packages/core/src/queries/snapshot.ts @@ -1,4 +1,4 @@ -import { Database } from "bun:sqlite"; +import { Database, constants } from "bun:sqlite"; import { statSync } from "fs"; import { join } from "path"; import { SCHEMA_VERSION } from "../lib/db.js"; @@ -7,7 +7,7 @@ import type { QueryCode, QueryResult, Root } from "./types.js"; export class QueryFailure extends Error { constructor(readonly code: QueryCode) { super(code); } } -export type Feature = "graph" | "links" | "voice" | "list" | "find"; +export type Feature = "graph" | "links" | "voice" | "list" | "find" | "storage"; const GRAPH_COLUMNS = { graph_metrics: ["document_id", "in_degree", "out_degree", "community", "pagerank"], graph_communities: ["community", "size", "label", "top_terms"], @@ -52,7 +52,7 @@ function compatibility(db: Database, feature: Feature): { if (schemaVersion >= 8) Object.assign(tables, GRAPH_COLUMNS); } - if (feature === "links") { + if (feature === "links" || feature === "storage") { documents.push("id", "summary", "updated"); tables.links = ["source_id", "target_id", "target"]; } @@ -96,7 +96,7 @@ export function withSnapshot(opts: Root, feature: Feature, validate: () => vo const dbPath = join(opts.brainPath, "brain.db"); if (!statSync(dbPath).isFile()) throw new QueryFailure("unavailable_index"); - db = new Database(dbPath, { readonly: true }); + db = new Database(dbPath, feature === "storage" ? constants.SQLITE_OPEN_READONLY | constants.SQLITE_OPEN_NOFOLLOW : { readonly: true }); db.run("PRAGMA busy_timeout=100"); const connection = db; return db.transaction(() => { diff --git a/packages/core/tests/storage-references.test.ts b/packages/core/tests/storage-references.test.ts new file mode 100644 index 0000000000..43db5591ca --- /dev/null +++ b/packages/core/tests/storage-references.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, mkdirSync, symlinkSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readStorageReferences } from "../src/queries"; + +test("Storage references are bounded resolved-link evidence from a read-only index", () => { + const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")), db = new Database(join(root, "brain.db")); + try { + db.exec(`CREATE TABLE index_metadata(key TEXT,value TEXT); INSERT INTO index_metadata VALUES('schema_version','3'); + CREATE TABLE documents(id INTEGER PRIMARY KEY,path TEXT,title TEXT,type TEXT,summary TEXT,updated TEXT,indexed_at TEXT); + CREATE TABLE links(source_id INTEGER,target_id INTEGER,target TEXT); + INSERT INTO documents VALUES(1,'Troy.pdf','Chart','asset',NULL,NULL,'2026-07-12T00:00:00Z');`); + const add = db.query("INSERT INTO documents VALUES(?,?,?,'note',NULL,NULL,'2026-07-12T00:00:00Z')"), link = db.query("INSERT INTO links VALUES(?,1,'Troy.pdf')"); + db.transaction(() => { for (let i = 0; i < 205; i++) { add.run(i + 2, `notes/chart-${i}.md`, "Odysseus chart"); link.run(i + 2); } })(); + const result = readStorageReferences({ brainPath: root, path: "Troy.pdf" }); + expect(result.ok).toBe(true); + if (!result.ok) throw Error(result.error.code); + expect(result.value.sources, "STORAGE_REFERENCE_LIMIT").toHaveLength(200); + expect(result.value.truncated).toBe(true); + expect(result.value.sources[0]?.indexedAt).toBe("2026-07-12T00:00:00Z"); + expect(readStorageReferences({ brainPath: root, path: "Ithaca.pdf" })).toMatchObject({ ok: true, value: { sources: [], truncated: false } }); + const redirected = join(root, "other"); mkdirSync(redirected); symlinkSync(join(root, "brain.db"), join(redirected, "brain.db")); + expect(readStorageReferences({ brainPath: redirected, path: "Troy.pdf" }).ok, "STORAGE_REFERENCE_INDEX_SYMLINK_REFUSED").toBe(false); + expect(readStorageReferences({ brainPath: root, path: "../Troy.pdf" })).toMatchObject({ ok: false, error: { code: "invalid_input" } }); + } finally { db.close(); rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-backend-claude/src/backend.ts b/packages/ui-backend-claude/src/backend.ts index dfa36a06bc..f3304401c5 100644 --- a/packages/ui-backend-claude/src/backend.ts +++ b/packages/ui-backend-claude/src/backend.ts @@ -1,3 +1,4 @@ +import { transcriptPaths } from "./transcript-archive.js"; import { getSessionMessages as sdkGetSessionMessages, listSessions as sdkListSessions, @@ -149,6 +150,7 @@ export function createClaudeBackend(options: ClaudeBackendOptions): AgentBackend }, startTurn, followUp, + storageRoots() { const paths = transcriptPaths(options.brainPath); return [{ kind: "transcripts", path: paths.live }, { kind: "archive", path: paths.archive }]; }, listSessions(): Promise { return history.listSessions(); }, diff --git a/packages/ui-backend-pi/src/backend.ts b/packages/ui-backend-pi/src/backend.ts index 4375f2697f..972787c8cf 100644 --- a/packages/ui-backend-pi/src/backend.ts +++ b/packages/ui-backend-pi/src/backend.ts @@ -147,6 +147,7 @@ export function createPiBackend(options: CreatePiBackendOptions): AgentBackend { ...(images.length > 0 ? { images } : {}), }); }, + storageRoots() { return [{ kind: "transcripts", path: sessionDir }]; }, listSessions(): Promise { return listPiSessions(brainPath, sessionDir); }, diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index dccfb7bbec..dcb3a62a12 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3645,3 +3645,5 @@ export interface ServerScopeDecisionReceipt { reason?: string; } export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-journal.js"; + +export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence } from "./storage.js"; diff --git a/packages/ui-sdk/src/server/backend.ts b/packages/ui-sdk/src/server/backend.ts index 10ab9214b4..eca42979bc 100644 --- a/packages/ui-sdk/src/server/backend.ts +++ b/packages/ui-sdk/src/server/backend.ts @@ -657,6 +657,8 @@ export interface AgentBackend { * queues a message refused that way as the session's next turn. */ followUp?(req: FollowUpRequest): Promise; + /** Host-only read-only roots already owned by this backend. Never passed to a worker bridge. */ + storageRoots?(): readonly { kind: "transcripts" | "archive"; path: string }[]; /** Sessions this backend owns (its own transcript store). */ listSessions(): Promise; /** Normalized-at-read history; backends own raw transcripts. */ diff --git a/packages/ui-sdk/src/storage.ts b/packages/ui-sdk/src/storage.ts new file mode 100644 index 0000000000..45fcddef96 --- /dev/null +++ b/packages/ui-sdk/src/storage.ts @@ -0,0 +1,54 @@ +/** Human-only Storage HTTP contracts. No host path or credential is a wire field. */ +export type StorageCoverage = "complete" | "partial" | "unknown"; +export type StorageOwner = "brain" | "app" | "operator"; +export interface StorageBytes { logical: number | null; allocated: number | null; coverage: StorageCoverage } +export interface StorageRoot extends StorageBytes { + id: string; label: string; category: "content" | "git" | "index" | "database" | "staging" | "scratch" | "workspace" | "temporary" | "transcripts" | "journal" | "native" | "backup"; + owner: StorageOwner; reason: string | null; +} +export interface StorageCapacityView { + id: string; rootIds: string[]; total: number | null; used: number | null; free: number | null; available: number | null; + provenance: "statfs"; kind: "tmpfs" | "filesystem" | "unknown"; identity: "resolved" | "unknown"; quota: null; + /** Shared extents cannot be partitioned into exclusive measured/other usage. */ + measured: null; other: null; reason: string | null; +} +export interface StorageFileEntry { + id: string; parentId: string | null; rootId: string; path: string; name: string; + kind: "file" | "directory" | "symlink" | "special"; + objectId: string; version: string; capacityId: string | null; aliasOf: string | null; + logical: number | null; allocated: number | null; links: number; + references: { state: "unchecked"; count: null; observedAt: null; unchecked: string[] }; +} +export interface StorageSnapshot { + id: string; startedAt: number; observedAt: number; staleAfter: number; + coverage: StorageCoverage; consistency: "live-observation"; + basis: "512 * st_blocks; shared extents may be charged more than once"; + roots: StorageRoot[]; capacities: StorageCapacityView[]; + totals: Record<"all" | StorageOwner, StorageBytes>; + entries: number; errors: Array<{ rootId: string; reason: string }>; omittedErrors: number; +} +export interface StorageStatus { + version: 1; state: "idle" | "scanning"; activeScanId: string | null; + snapshot: StorageSnapshot | null; previous: StorageSnapshot | null; + lastFailure: { scanId: string; at: number; reason: string } | null; + limits: { entries: number; page: number; roots: number; milliseconds: number; cacheBytes: number }; +} +export interface StoragePage { + scanId: string; entries: StorageFileEntry[]; nextCursor: string | null; stale: boolean; + /** Logical bytes per matching regular-file entry, counting aliases individually. */ + fileLogicalTotal: number | null; + matched: number; totals: StorageBytes; +} +export interface StorageFilter { + rootId?: string; parentId?: string; owner?: StorageOwner | "all"; + type?: "pdf"; largerThan?: number; cursor?: string; +} + +export interface StorageReferenceEvidence { + entryId: string; observedAt: number; + state: "observed" | "unavailable"; + sources: Array<{ path: string; indexedAt: string }>; + truncated: boolean; newestIndexedAt: string | null; + /** Index freshness is not verified against current source bytes. */ + freshness: "unchecked"; unchecked: string[]; +} diff --git a/packages/ui-server/README.md b/packages/ui-server/README.md index e9c1899942..3c5e29822d 100644 --- a/packages/ui-server/README.md +++ b/packages/ui-server/README.md @@ -372,6 +372,8 @@ under the [service terms](../geo/README.md#configuration). | `BRAIN_UI_PRICING_TTL_HOURS` | How long a fetched model-pricing table stays fresh, in hours. | 24 | | `BRAIN_UI_SCHEDULE_INFERENCE_ORIGINS` | Comma-separated HTTPS origins (at most 16) of the model connection scheduled work would use. Operators review this audience with every schedule; unset refuses schedule proposals (unsupported_capability). Does not enable dispatch. | (unset; schedule proposals refused) | | `BRAIN_UI_SKILLS_GITHUB_TOKEN` | GitHub token used when installing skills from a private repository (Settings → Skills) — typically read-only Contents on the skill repos. Falls back to GITHUB_TOKEN. | $GITHUB_TOKEN | +| `BRAIN_UI_STORAGE_BACKUP_ROOTS` | JSON array of at most four absolute operator-approved local backup directories. Metadata only; no remote stores or automatic cleanup. | [] | +| `BRAIN_UI_STORAGE_GIT_ROOTS` | JSON array of at most sixteen absolute approved external Git directories for human-only Storage inventory. | [] | | `BRAIN_UI_SUBPROCESS_ENV_EXTRA` | Comma-separated environment variable names to admit to every child audience when an operator integration needs a variable outside the shipped allowlist. Names are trimmed; malformed entries are ignored; the control variable itself is never forwarded. | (empty) | | `BRAIN_UI_SYNC_GITHUB_TOKEN` | GitHub HTTPS credential for the host-owned sync mirror transport. Never passed to a maintenance worker that evaluates the brain. Falls back to GITHUB_TOKEN. | $GITHUB_TOKEN | | `BRAIN_UI_TURN_TEMP_BUDGET_BYTES` | What all running agent workers' turn-temp allowances may add up to, in bytes (a positive multiple of 4096, at least one allowance). A turn that would pass it is refused as worker_state_busy. Logical admission, not reserved RAM; invalid values refuse the boot. | BRAIN_UI_TURN_TEMP_BYTES × MAX_CONCURRENT_SESSIONS | diff --git a/packages/ui-server/src/agent/backend.ts b/packages/ui-server/src/agent/backend.ts index 1b718bc3db..3e4338eb09 100644 --- a/packages/ui-server/src/agent/backend.ts +++ b/packages/ui-server/src/agent/backend.ts @@ -56,6 +56,8 @@ export interface BackendRegistryOptions { export interface BackendRegistry { getBackends(): Promise; + /** Inspect already constructed owners without triggering runtime setup or discovery. */ + peekBackends?(): readonly AgentBackend[] | null; getBackendById(id: string): Promise; getDefaultBackend(): Promise; getDefaultBackendId(): Promise; @@ -475,6 +477,7 @@ export function createBackendRegistry(options: BackendRegistryOptions): BackendR }; const backendLog = options.log ? toBackendLog(options.log) : undefined; let cachedRegistry: Promise | null = null; + let loadedRegistry: RegistrySnapshot | null = null; async function buildRegistry(): Promise { const entries = activeFirstPartyBackends(agent); @@ -523,10 +526,10 @@ export function createBackendRegistry(options: BackendRegistryOptions): BackendR } const getRegistry = (): Promise => { - if (!cachedRegistry) cachedRegistry = buildRegistry(); + if (!cachedRegistry) cachedRegistry = buildRegistry().then(value => (loadedRegistry = value)); return cachedRegistry; }; - return makeRegistry(getRegistry, options); + return makeRegistry(getRegistry, options, () => loadedRegistry); } export function createStaticBackendRegistry( @@ -556,7 +559,7 @@ export function createStaticBackendRegistry( resolved, options.modelSource ?? null ); - return makeRegistry(async () => snapshot, options); + return makeRegistry(async () => snapshot, options, () => snapshot); } function makeRegistry( @@ -566,7 +569,8 @@ function makeRegistry( getDefaultModelId?: () => string | null; getBillingOverrides?: () => Record; log?: Logger; - } + }, + peek?: () => RegistrySnapshot | null ): BackendRegistry { let profileSnapshot: ProfileSnapshot | null = null; @@ -643,6 +647,7 @@ function makeRegistry( } return { + peekBackends: () => peek?.()?.backends ?? null, async getBackends() { return (await getRegistry()).backends; }, diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index 8704e73c20..e97a5d6dec 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -1,3 +1,6 @@ +import { createStorageCoordinator } from "./storage/coordinator.js"; +import { createStorageRoutes } from "./routes/storage.js"; +import type { StorageStore } from "./storage/collect.js"; import { recordGovernanceEvent } from "./governance/events.js"; import { createPolicyProposals } from "./brain/policy-proposal.js"; import { createAgentPolicyRoutes } from "./routes/agent-policy.js"; @@ -801,6 +804,34 @@ export async function createApp(options: CreateAppOptions = {}): Promise host.prepareHistory(id, messages) })); app.route("/api", createJournalRoutes({ journal })); + const storageStores: StorageStore[] = [{ path: journalRoot, label: "Retained file history", category: "journal", owner: "app", recursive: true }]; + for (const [path, label] of [[config.dbPath, "Operational database"], [nativeLeasePath, "Native lease database"]] as const) { + if (!path || path === ":memory:") continue; + for (const suffix of ["", "-wal", "-shm", "-journal"]) storageStores.push({ path: resolve(path + suffix), label: label + suffix, category: "database", owner: "app", recursive: false, optional: suffix !== "" }); + } + for (const [index, path] of (config.storage?.backupRoots ?? []).entries()) storageStores.push({ path, label: `Backup store ${index + 1}`, category: "backup", owner: "operator", recursive: true }); + const storage = createStorageCoordinator({ brainPath: config.brainPath, stores: storageStores, gitRoots: [...config.storage?.gitRoots ?? []], exec: config.exec, + authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, + unavailable: [ + { label: "Native checkpoints and output", category: "native", owner: "app", reason: "shared-native-allocation-unavailable" }, + { label: "Live temporary stores", category: "temporary", owner: "app", reason: "private-runtime-usage-unavailable" }, + ], + storesNow() { + const stores: StorageStore[] = [], unavailable: Array<{ label: string; category: "transcripts"; owner: "app"; reason: string }> = []; + const quarantine = hostExecution.quarantinePath(); + if (quarantine) stores.push({ path: quarantine, label: "Quarantined changes", category: "journal", owner: "app", recursive: true, optional: true }); + const backends = registry.peekBackends?.(); + if (!backends) unavailable.push({ label: "Backend transcripts", category: "transcripts", owner: "app", reason: "backend-owner-not-initialized" }); + for (const backend of backends ?? []) { + if (!backend.storageRoots) { unavailable.push({ label: `${backend.id} transcripts`, category: "transcripts", owner: "app", reason: "backend-storage-unsupported" }); continue; } + try { for (const location of backend.storageRoots()) stores.push({ path: location.path, label: `${backend.id} ${location.kind}`, category: "transcripts", owner: "app", recursive: true, optional: true }); } catch { unavailable.push({ label: `${backend.id} transcripts`, category: "transcripts", owner: "app", reason: "backend-roots-unavailable" }); } + } + return { stores, unavailable }; + }, + }); + bootCleanup.push(() => storage.close()); + app.route("/api", createStorageRoutes(storage)); + const reverts = createTurnReverts(db, { brainPath: config.brainPath, journal, gate: hostExecution.gate()!, lock: host.workerLock, isAuthorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, refresh() { host.governedTurns?.refresh(host.workerLock); } }); @@ -934,6 +965,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise { await Promise.allSettled([...host.turnTasks]); }); await attempt(() => subscription.close()); await attempt(() => intake.close()); + await attempt(() => storage.close()); await attempt(() => scratchPrune.close()); await attempt(() => scratchPins.close()); await attempt(() => reverts.close()); diff --git a/packages/ui-server/src/brain/host-execution.ts b/packages/ui-server/src/brain/host-execution.ts index 474ceaf091..85b3df59f6 100644 --- a/packages/ui-server/src/brain/host-execution.ts +++ b/packages/ui-server/src/brain/host-execution.ts @@ -52,6 +52,8 @@ export interface HostExecution { /** Null until the database is attached. */ gate(): WriterGate | null; manifests(): GovernedManifests | null; + /** Existing quarantine only; reading this does not create it. */ + quarantinePath(): string | undefined; /** * Reconciles every abandoned governed turn on an existing database (with * none, nothing is created, so a refused boot leaves no database behind), @@ -90,6 +92,7 @@ export function createHostExecution(options: { brainPath: string; dbPath: string boundary, gate: () => gate, manifests: () => manifests, + quarantinePath: () => quarantineDir, reconcileExisting() { if (options.dbPath === ":memory:" || !existsSync(options.dbPath)) return []; const db = createUiDb(options.dbPath); diff --git a/packages/ui-server/src/config/env.ts b/packages/ui-server/src/config/env.ts index d95a73468e..0455e22691 100644 --- a/packages/ui-server/src/config/env.ts +++ b/packages/ui-server/src/config/env.ts @@ -81,6 +81,8 @@ export interface EnvVarDescriptor { * documentation in both directions — add here and to the docs together. */ export const ENV_VARS: readonly EnvVarDescriptor[] = [ + { name: "BRAIN_UI_STORAGE_BACKUP_ROOTS", description: "JSON array of at most four absolute operator-approved local backup directories. Metadata only; no remote stores or automatic cleanup.", default: "[]", required: false }, + { name: "BRAIN_UI_STORAGE_GIT_ROOTS", description: "JSON array of at most sixteen absolute approved external Git directories for human-only Storage inventory.", default: "[]", required: false }, { name: "BRAIN_WORKER_DATA_ROOT", description: "Internal application-maintenance output root supplied by the trusted launch. Not an operator configuration input.", default: "unset outside maintenance", required: false }, { name: "BRAIN_UI_AUTONOMOUS_SPEND_USD_PER_DAY", required: false, default: "5", description: "Admission cap for non-subscription autonomous spend, including active reservations. Invalid values fail startup." }, { name: "BRAIN_UI_AUTONOMOUS_TURNS_PER_DAY", required: false, default: "0", description: "Daily model-bearing autonomous operation cap. Zero pauses admission until explicitly configured. Invalid values fail startup." }, @@ -750,6 +752,8 @@ export interface VoiceConfig { /** Fully-resolved server configuration. Plain data — safe to construct in tests. */ export interface ServerConfig { + /** Trusted host roots only; never accepted from Storage HTTP requests. */ + storage?: { backupRoots: readonly string[]; gitRoots: readonly string[] }; /** Host-only Unix service for managed native brain storage. */ nativeBrain?: { socketPath: string; timeoutMs: number; maxDeltaBytes: number; leasePath?: string }; /** Resolved by the host edge; optional for existing programmatic callers, which use SDK defaults. */ @@ -1022,6 +1026,7 @@ export function resolveServerConfig(env: EnvRecord = process.env): ServerConfig nativeBrain: resolveNativeBrainConfig(env), maintenanceTemp: hostMaintenanceTempLimits(env), brainPath, + storage: { backupRoots: storageRoots(env.BRAIN_UI_STORAGE_BACKUP_ROOTS, 4), gitRoots: storageRoots(env.BRAIN_UI_STORAGE_GIT_ROOTS, 16) }, askUserFormLimits: resolveAskUserFormLimits(Object.fromEntries([ ["maxDepth", env.BRAIN_UI_ASK_USER_FORM_MAX_DEPTH], ["maxNodes", env.BRAIN_UI_ASK_USER_FORM_MAX_NODES], @@ -1284,3 +1289,12 @@ export const DYNAMIC_ENV_READS: readonly DynamicEnvReadSpec[] = [ export function maintenanceDataRoot(): string | undefined { return process.env.BRAIN_WORKER_DATA_ROOT; } + +/** A finite explicit root list; validation does not touch or create these paths. */ +function storageRoots(raw: string | undefined, limit: number): string[] { + if (!raw) return []; + let values: unknown; + try { values = JSON.parse(raw); } catch { throw Error("Storage root lists must be JSON arrays."); } + if (!Array.isArray(values) || values.length > limit || values.some(value => typeof value !== "string" || !value.startsWith("/") || value.length > 4096 || value.includes("\0"))) throw Error("Storage roots must be a bounded list of absolute paths."); + return values; +} diff --git a/packages/ui-server/src/core-queries.ts b/packages/ui-server/src/core-queries.ts index dbb1aea1f8..50e098a3bf 100644 --- a/packages/ui-server/src/core-queries.ts +++ b/packages/ui-server/src/core-queries.ts @@ -71,6 +71,10 @@ export interface VoiceQueries { readVoiceVocabulary(opts: Root & { limit: number }): QueryResult<{ terms: string[]; extractorVersion: number }>; } +export interface StorageQueries { + readStorageReferences(opts: Root & { path: string }): QueryResult<{ sources: Array<{ path: string; indexedAt: string }>; truncated: boolean }>; +} + const GRAPH_OPERATIONS = [ "readGraphMeta", "readGraphClusters", @@ -104,6 +108,7 @@ export type CoreCapability = export interface CoreQueryAccess { graph(): CoreCapability; voice(): CoreCapability; + storage?(): CoreCapability; } /** How the package is located and loaded — the real `require`, except in tests. */ @@ -188,20 +193,21 @@ export function createCoreQueryAccess(options: { log?: Logger } = {}): CoreQuery let resolved: Resolved | undefined; let graph: CoreCapability | undefined; let voice: CoreCapability | undefined; + let storage: CoreCapability | undefined; const reported = new Set(); const entry = (): Resolved => { resolved ??= resolveEntry((loaderOverride ?? defaultLoader)()); return resolved; }; - const report = (name: "graph" | "voice", capability: CoreCapability): CoreCapability => { + const report = (name: "graph" | "voice" | "storage", capability: CoreCapability): CoreCapability => { if (!capability.ok && !reported.has(name)) { reported.add(name); const version = resolved && !resolved.ok ? resolved.version : undefined; options.log?.emit({ severityText: "WARN", body: - `${name === "graph" ? "graph view" : "index-derived voice vocabulary"} unavailable: ` + + `${name === "graph" ? "graph view" : name === "storage" ? "Storage reference evidence" : "index-derived voice vocabulary"} unavailable: ` + `install ${CORE_PACKAGE} ${CORE_QUERIES_RANGE} alongside this server`, attributes: { "core.reason": capability.reason, @@ -214,6 +220,7 @@ export function createCoreQueryAccess(options: { log?: Logger } = {}): CoreQuery }; return { + storage: () => (storage ??= report("storage", feature(entry(), ["readStorageReferences"]))), graph: () => (graph ??= report("graph", feature(entry(), GRAPH_OPERATIONS))), voice: () => (voice ??= report("voice", feature(entry(), VOICE_OPERATIONS))), }; diff --git a/packages/ui-server/src/routes/storage.ts b/packages/ui-server/src/routes/storage.ts new file mode 100644 index 0000000000..1312e02162 --- /dev/null +++ b/packages/ui-server/src/routes/storage.ts @@ -0,0 +1,36 @@ +import { Hono } from "hono"; +import { requireJson } from "../middleware/origin.js"; +import type { AppEnv } from "../app-env.js"; +import type { StorageFilter } from "@schlessera/brain-ui-sdk/protocol"; +import { StorageUnavailable, type StorageCoordinator } from "../storage/coordinator.js"; +/** Ordinary content authorization does not grant access to deployment inventory. */ +export function createStorageRoutes(storage: StorageCoordinator): Hono { + const routes = new Hono(); + routes.onError((error, c) => c.json({ error: error instanceof StorageUnavailable ? error.message : "Storage temporarily unavailable." }, error instanceof StorageUnavailable ? error.status : 503)); + routes.use("/storage*", async (c, next) => { + c.header("Cache-Control", "private, no-store"); + const principal = c.get("principal"); + if (!principal) return c.json({ error: "Authentication required." }, 401); + if (!["owner", "ambient"].includes(principal.kind)) return c.json({ error: "Storage requires human-owner access." }, 403); + try { storage.status(principal.id); await next(); } + catch (error) { return c.json({ error: error instanceof StorageUnavailable ? error.message : "Storage temporarily unavailable." }, error instanceof StorageUnavailable ? error.status : 503); } + }); + routes.get("/storage", c => c.json(storage.status(c.get("principal")!.id))); + routes.post("/storage/scans", requireJson(), async c => { + let body: unknown; try { body = await c.req.json(); } catch { return c.json({ error: "Expected an empty JSON object." }, 400); } + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).length) return c.json({ error: "Storage roots and options are host configured." }, 400); + return c.json(storage.start(c.get("principal")!.id), 202); + }); + routes.delete("/storage/scans/:id", c => c.json(storage.cancel(c.get("principal")!.id, c.req.param("id")), 202)); + routes.get("/storage/scans/:id/entries", c => { + const raw = c.req.query(), keys = ["rootId", "parentId", "owner", "type", "largerThan", "cursor"]; + if (Object.keys(raw).some(key => !keys.includes(key)) || raw.owner !== undefined && !["all", "brain", "app", "operator"].includes(raw.owner) + || raw.type !== undefined && raw.type !== "pdf" || [raw.rootId, raw.parentId].some(id => id !== undefined && id !== "" && !/^[a-f0-9]{32}$/.test(id)) + || raw.largerThan !== undefined && (!/^\d+$/.test(raw.largerThan) || !Number.isSafeInteger(Number(raw.largerThan)))) return c.json({ error: "Invalid Storage filters." }, 400); + const filter: StorageFilter = { ...(raw.rootId !== undefined ? { rootId: raw.rootId } : {}), ...(raw.parentId !== undefined ? { parentId: raw.parentId } : {}), + ...(raw.owner ? { owner: raw.owner as StorageFilter["owner"] } : {}), ...(raw.type ? { type: "pdf" } : {}), ...(raw.largerThan !== undefined ? { largerThan: Number(raw.largerThan) } : {}), ...(raw.cursor ? { cursor: raw.cursor } : {}) }; + return c.json(storage.page(c.get("principal")!.id, c.req.param("id"), filter)); + }); + routes.get("/storage/scans/:id/entries/:entryId/references", async c => c.json(await storage.references(c.get("principal")!.id, c.req.param("id"), c.req.param("entryId")))); + return routes; +} diff --git a/packages/ui-server/src/storage/collect.ts b/packages/ui-server/src/storage/collect.ts new file mode 100644 index 0000000000..441fa934c3 --- /dev/null +++ b/packages/ui-server/src/storage/collect.ts @@ -0,0 +1,148 @@ +import { createHmac } from "node:crypto"; +import { basename, dirname, resolve } from "node:path"; +import { z } from "zod"; +import type { StorageFileEntry, StorageRoot, StorageSnapshot, StorageOwner, StorageBytes } from "@schlessera/brain-ui-sdk/protocol"; +import { SHARE_STAGING_DIR } from "@schlessera/brain-ui-sdk/protocol"; +import type { ExecWrapperConfig } from "@schlessera/brain-ui-sdk/server"; +import { serverMetadata, storageCli, storageProcess } from "./process.js"; +import type { MetadataEntry, MetadataRoot } from "./metadata.js"; +export const STORAGE_SERVER_LIMITS = Object.freeze({ entries: 100_000, page: 200, roots: 32, milliseconds: 35_000, cacheBytes: 96 * 1024 * 1024 }); +const bytes = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).nullable(); +const coreInventory = z.object({ schema_version: z.literal(1), status: z.enum(["complete", "partial", "unsupported"]), + roots: z.array(z.object({ id: z.string(), kind: z.enum(["brain", "git"]), path: z.string().max(4096) })).max(65), + entries: z.array(z.object({ rootId: z.string(), path: z.string().max(16384), kind: z.enum(["file", "directory", "symlink", "special"]), objectId: z.string(), version: z.string(), logicalBytes: bytes, allocatedBytes: bytes, links: z.number().int().nonnegative() })).max(100_000), + errors: z.array(z.object({ reason: z.string().max(128) })).max(100), omittedErrors: z.number().int().nonnegative(), +}); +export interface StorageStore { path: string; label: string; category: StorageRoot["category"]; owner: StorageOwner; recursive: boolean; optional?: boolean } +export interface StorageCollection { summary: StorageSnapshot; entries: StorageFileEntry[]; bytes: number } +export interface StorageCollectOptions { + brainPath: string; stores: StorageStore[]; gitRoots?: string[]; exec?: ExecWrapperConfig; + /** Unknown private owner stores are explicit rows, never scans of host /tmp or broker state. */ + unavailable?: Array<{ label: string; category: StorageRoot["category"]; owner: StorageOwner; reason: string }>; + salt: string; +} +const validPath = (path: string) => path === "" || (!path.startsWith("/") && path.split("/").every(piece => piece !== "." && piece !== ".." && piece !== "") && !path.includes("\0")); +export function storageTotals(entries: StorageFileEntry[], complete = true): StorageBytes { + if (!entries.length && !complete) return { logical: null, allocated: null, coverage: "unknown" }; + const seen = new Set(); let logical = 0n, allocated = 0n, hasLogical = true, hasAllocation = true; + for (const entry of entries) { + if (seen.has(entry.objectId)) continue; + seen.add(entry.objectId); + if (entry.logical === null) hasLogical = false; else logical += BigInt(entry.logical); + if (entry.allocated === null) hasAllocation = false; else allocated += BigInt(entry.allocated); + } + const safe = (value: bigint) => value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null; + return { logical: hasLogical ? safe(logical) : null, allocated: hasAllocation ? safe(allocated) : null, coverage: complete && hasLogical && hasAllocation ? "complete" : "partial" }; +} +export async function collectStorage(options: StorageCollectOptions, id: string, signal: AbortSignal): Promise { + const startedAt = Date.now(), key = (value: string) => createHmac("sha256", options.salt).update(value).digest("hex").slice(0, 32); + const roots: StorageRoot[] = [], errors: StorageSnapshot["errors"] = [], entries: StorageFileEntry[] = []; + let omittedErrors = 0; + const failedRoots = new Set(); + const fail = (rootId: string, reason: string) => { failedRoots.add(rootId); if (errors.length < 100) errors.push({ rootId, reason }); else omittedErrors++; }; + const root = (identity: string, label: string, category: StorageRoot["category"], owner: StorageOwner) => { + const id = key(`root:${identity}`); let found = roots.find(item => item.id === id); + if (!found) { found = { id, label, category, owner, logical: null, allocated: null, coverage: "unknown", reason: null }; roots.push(found); } + return found; + }; + const brain = root("brain", "Brain files", "content", "brain"); + const sources = new Map(); + const rawEntries: MetadataEntry[] = []; + const coreRootMappings = new Map(); + try { + const raw = coreInventory.parse(await storageProcess([...storageCli(options.brainPath), "storage", "inventory", "--json"], { signal, exec: options.exec, + env: { BRAIN_ROOT: options.brainPath, BRAIN_STORAGE_GIT_ROOTS: JSON.stringify(options.gitRoots ?? []) } })); + if (raw.status !== "complete") fail(brain.id, "core-inventory-partial"); + for (const value of raw.errors) fail(brain.id, value.reason); + omittedErrors += raw.omittedErrors; + for (const item of raw.roots) { + if (sources.size >= 16) { fail(brain.id, "core-root-budget"); break; } + const view = item.kind === "brain" ? brain : root(`git:${item.path}`, "Git store", "git", "brain"); + coreRootMappings.set(item.id, { path: resolve(item.path), root: view }); + sources.set(view.id, { root: view, path: item.path, recursive: false }); + } + if (raw.status !== "complete") for (const mapped of coreRootMappings.values()) if (mapped.root.id !== brain.id) fail(mapped.root.id, "core-inventory-partial"); + for (const item of raw.entries) { + const mapped = coreRootMappings.get(item.rootId); + if (!mapped || !validPath(item.path)) { fail(brain.id, "core-entry-unavailable"); continue; } + let view = mapped.root, path = item.path; + if (view.id === brain.id) { + // Core reports its Git root separately, so collapse the canonical in-tree alias. + const git = [...coreRootMappings.values()].find(value => value.root.category === "git" && resolve(options.brainPath, path) === value.path); + if (git) { view = git.root; path = ""; } + else { + const containing = [...coreRootMappings.values()].find(value => value.root.category === "git" && resolve(options.brainPath, path).startsWith(`${value.path}/`)); + if (containing) { view = containing.root; path = resolve(options.brainPath, path).slice(containing.path.length + 1); } + } + if (view.id === brain.id) { + for (const [prefix, label, category, owner] of [ + [SHARE_STAGING_DIR, "Upload staging", "staging", "app"], [".brain/scratch", "Scratch", "scratch", "brain"], ["workspaces", "Workspaces", "workspace", "brain"], + ] as const) if (path === prefix || path.startsWith(`${prefix}/`)) { view = root(prefix, label, category, owner); sources.set(view.id, { root: view, path: resolve(options.brainPath, prefix), recursive: false, optional: true }); path = path.slice(prefix.length).replace(/^\//, ""); break; } + if (/^brain\.db(?:-wal|-shm|-journal)?$/.test(path)) { view = root("index", "Content index", "index", "brain"); sources.set(view.id, { root: view, path: options.brainPath, recursive: false }); } + } + } + rawEntries.push({ rootId: view.id, path, kind: item.kind, object: item.objectId, version: item.version, device: item.objectId.split(":")[0], logical: item.logicalBytes, allocated: item.allocatedBytes, links: item.links }); + } + } catch { fail(brain.id, signal.aborted ? "cancelled" : "storage-cli-unavailable-or-incompatible"); } + const appSources: MetadataRoot[] = []; + for (const store of options.stores) { + if (roots.length >= STORAGE_SERVER_LIMITS.roots - (options.unavailable?.length ?? 0)) { fail("", "root-budget"); break; } + const view = root(`store:${store.path}`, store.label, store.category, store.owner); + const source = { root: view, path: store.path, recursive: store.recursive, optional: store.optional }; + sources.set(view.id, source); appSources.push({ id: view.id, ...source }); + } + const probes: MetadataRoot[] = [...sources.values()].filter(source => !appSources.some(item => item.id === source.root.id)).map(source => ({ id: source.root.id, path: source.path, recursive: false, optional: source.optional })); + const capacities: StorageSnapshot["capacities"] = []; + const capacityByRoot = new Map(); + try { + const measured = await serverMetadata([...appSources, ...probes], signal, Math.max(1, STORAGE_SERVER_LIMITS.entries - rawEntries.length)); + for (const error of measured.errors) fail(error.rootId, error.reason); + omittedErrors += measured.omittedErrors; + if (measured.omittedErrors) for (const source of sources.values()) fail(source.root.id, "metadata-errors-omitted"); + const appIds = new Set(appSources.map(item => item.id)); + rawEntries.push(...measured.entries.filter(item => appIds.has(item.rootId))); + for (const value of measured.capacities) { + const capacityId = key(value.identity ? `fs:${value.filesystem}:${value.identity}` : `unknown:${value.rootId}`); + capacityByRoot.set(value.rootId, capacityId); + const previous = capacities.find(item => item.id === capacityId); + if (previous) { if (!previous.rootIds.includes(value.rootId)) previous.rootIds.push(value.rootId); continue; } + capacities.push({ id: capacityId, rootIds: [value.rootId], total: value.total, used: value.used, free: value.free, available: value.available, + provenance: "statfs", kind: value.filesystem === String(0x01021994) ? "tmpfs" : value.reason === "overlay-capacity-unknown" ? "unknown" : "filesystem", identity: value.identity ? "resolved" : "unknown", quota: null, measured: null, other: null, reason: value.reason ?? "shared-extent-partition-unknown" }); + } + } catch { for (const source of sources.values()) fail(source.root.id, signal.aborted ? "cancelled" : "metadata-unavailable"); } + if ((options.unavailable?.length ?? 0) > STORAGE_SERVER_LIMITS.roots - roots.length) fail("", "root-budget"); + for (const unavailable of (options.unavailable ?? []).slice(0, STORAGE_SERVER_LIMITS.roots - roots.length)) { const view = root(`unavailable:${unavailable.label}`, unavailable.label, unavailable.category, unavailable.owner); view.reason = unavailable.reason; fail(view.id, unavailable.reason); } + const representedRoots = new Set(rawEntries.filter(value => value.path === "").map(value => value.rootId)); + const ids = new Set(), objects = new Map(); + const versions = new Map(); + for (const value of rawEntries) { + if (entries.length >= STORAGE_SERVER_LIMITS.entries) { fail(value.rootId, "entry-budget"); break; } + const entryId = key(`${value.rootId}:${value.path}`); + if (ids.has(entryId)) continue; + ids.add(entryId); + const objectId = key(`object:${value.object}`), first = objects.get(objectId); + const observed = versions.get(objectId); + if (observed && observed.version !== value.version) { fail(value.rootId, "object-changed-during-scan"); fail(observed.rootId, "object-changed-during-scan"); } + versions.set(objectId, { version: value.version, rootId: value.rootId }); + objects.set(objectId, first ?? entryId); + entries.push({ id: entryId, parentId: value.path && (dirname(value.path) !== "." || representedRoots.has(value.rootId)) ? key(`${value.rootId}:${dirname(value.path) === "." ? "" : dirname(value.path)}`) : null, + rootId: value.rootId, path: value.path, name: value.path ? basename(value.path) : roots.find(item => item.id === value.rootId)?.label ?? "Store", + kind: value.kind, objectId, version: key(`version:${value.version}`), capacityId: capacityByRoot.get(value.rootId) ?? null, aliasOf: first ?? null, + logical: value.logical, allocated: value.allocated, links: value.links, + references: { state: "unchecked", count: null, observedAt: null, unchecked: ["indexed links", "unindexed files", "external consumers"] } }); + } + entries.sort((a, b) => a.rootId.localeCompare(b.rootId) || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + for (const view of roots) { + const values = entries.filter(entry => entry.rootId === view.id), issue = errors.find(error => error.rootId === view.id || !error.rootId) ?? (failedRoots.has(view.id) || failedRoots.has("") ? { reason: "additional-errors-omitted" } : undefined); + if (!values.length && issue) { view.reason ??= issue.reason; continue; } + Object.assign(view, storageTotals(values, !issue), { reason: issue?.reason ?? view.reason }); + } + const ownerOf = new Map(roots.map(view => [view.id, view.owner])); + const total = (owner: StorageOwner) => storageTotals(entries.filter(entry => ownerOf.get(entry.rootId) === owner), roots.filter(view => view.owner === owner).every(view => view.coverage === "complete")); + const observedAt = Date.now(); + const summary: StorageSnapshot = { id, startedAt, observedAt, staleAfter: observedAt + 60_000, coverage: errors.length ? "partial" : "complete", consistency: "live-observation", + basis: "512 * st_blocks; shared extents may be charged more than once", roots, capacities, totals: { all: storageTotals(entries, errors.length === 0), brain: total("brain"), app: total("app"), operator: total("operator") }, entries: entries.length, errors, omittedErrors }; + const size = Buffer.byteLength(JSON.stringify({ summary, entries })); + if (size > STORAGE_SERVER_LIMITS.cacheBytes / 2) throw Error("cache-budget"); + return { summary, entries, bytes: size }; +} diff --git a/packages/ui-server/src/storage/coordinator.ts b/packages/ui-server/src/storage/coordinator.ts new file mode 100644 index 0000000000..7b17e5523d --- /dev/null +++ b/packages/ui-server/src/storage/coordinator.ts @@ -0,0 +1,122 @@ +import { storageReferences } from "./process.js"; +import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import type { StorageFilter, StoragePage, StorageStatus, StorageReferenceEvidence } from "@schlessera/brain-ui-sdk/protocol"; +import { collectStorage, storageTotals, STORAGE_SERVER_LIMITS, type StorageCollection, type StorageCollectOptions } from "./collect.js"; +export class StorageUnavailable extends Error { + constructor(readonly status: 400 | 403 | 404 | 409 | 429 | 503, message: string) { super(message); } +} +export function createStorageCoordinator(options: Omit & { + authorized(principalId: string): boolean; + /** Host/backend owners resolve exact roots; this callback never receives request data. */ + storesNow?(): { stores: StorageCollectOptions["stores"]; unavailable: NonNullable }; +}) { + const salt = randomBytes(32).toString("hex"); + let current: StorageCollection | null = null, previous: StorageCollection | null = null; + let active: { id: string; principal: string; controller: AbortController; promise: Promise } | null = null; + let lastAttempt = 0, closed = false; + let referenceJob: { controller: AbortController; promise: Promise } | null = null; + let lastFailure: { scanId: string; at: number; reason: string } | null = null; + const authorized = (principal: string) => { if (!options.authorized(principal)) throw new StorageUnavailable(403, "Storage requires current human-owner access."); }; + const prune = () => { if (previous && Date.now() - previous.summary.observedAt > 600_000) previous = null; if (current && Date.now() - current.summary.observedAt > 600_000) current = null; }; + const status = (principal: string): StorageStatus => { + authorized(principal); prune(); + return { version: 1, state: active ? "scanning" : "idle", activeScanId: active?.id ?? null, snapshot: current?.summary ?? null, previous: previous?.summary ?? null, lastFailure, limits: STORAGE_SERVER_LIMITS }; + }; + const digest = (value: string) => createHmac("sha256", salt).update(value).digest("base64url"); + function collection(principal: string, id: string): StorageCollection { + authorized(principal); prune(); + const result = [current, previous].find(value => value?.summary.id === id); + if (!result) throw new StorageUnavailable(404, "Storage observation expired or unavailable; refresh."); + return result; + } + return { + status, + start(principal: string): StorageStatus { + authorized(principal); + if (closed) throw new StorageUnavailable(503, "Storage is shutting down."); + if (active) return status(principal); + if (Date.now() - lastAttempt < 5_000) throw new StorageUnavailable(429, "Wait before refreshing Storage again."); + lastAttempt = Date.now(); + const id = randomUUID(), controller = new AbortController(); + const job = { id, principal, controller, promise: Promise.resolve() }; + active = job; + const timeout = setTimeout(() => controller.abort(), STORAGE_SERVER_LIMITS.milliseconds); + const revocation = setInterval(() => { if (!options.authorized(principal)) controller.abort(); }, 250); + job.promise = (async () => { + try { + const dynamic = options.storesNow?.() ?? { stores: [], unavailable: [] }; + controller.signal.throwIfAborted(); + const result = await collectStorage({ ...options, stores: [...options.stores, ...dynamic.stores], unavailable: [...options.unavailable ?? [], ...dynamic.unavailable], salt }, id, controller.signal); + authorized(principal); controller.signal.throwIfAborted(); + if (result.summary.coverage === "complete" || !current || current.summary.coverage !== "complete") { previous = current; current = result; } + else previous = result; // A partial refresh never erases the last complete observation. + if (current && previous && current.bytes + previous.bytes > STORAGE_SERVER_LIMITS.cacheBytes) previous = null; + lastFailure = null; + } catch { lastFailure = { scanId: id, at: Date.now(), reason: controller.signal.aborted || !options.authorized(principal) ? "cancelled-or-revoked" : "collection-unavailable" }; } + finally { clearTimeout(timeout); clearInterval(revocation); if (active === job) active = null; } + })(); + return status(principal); + }, + cancel(principal: string, id: string): StorageStatus { + authorized(principal); + if (active?.id !== id) throw new StorageUnavailable(409, "That scan is no longer active."); + active.controller.abort(); return status(principal); + }, + page(principal: string, id: string, filter: StorageFilter): StoragePage { + const value = collection(principal, id); + const { cursor, ...criteria } = filter; + const filterKey = JSON.stringify(criteria); + let offset = 0; + if (cursor) { + if (cursor.length > 2048) throw new StorageUnavailable(400, "Invalid Storage cursor."); + const [encoded, signature, extra] = cursor.split("."); + const expected = Buffer.from(digest(encoded ?? "")), actual = Buffer.from(signature ?? ""); + if (extra || expected.length !== actual.length || !timingSafeEqual(expected, actual)) throw new StorageUnavailable(400, "Invalid Storage cursor."); + let token: { id: string; filter: string; offset: number; expires: number }; + try { token = JSON.parse(Buffer.from(encoded!, "base64url").toString()); } catch { throw new StorageUnavailable(400, "Invalid Storage cursor."); } + if (token.id !== id || token.filter !== filterKey || token.expires < Date.now() || !Number.isSafeInteger(token.offset) || token.offset < 0) throw new StorageUnavailable(409, "Storage cursor expired or filters changed."); + offset = token.offset; + } + const owners = new Map(value.summary.roots.map(root => [root.id, root.owner])); + const matches = value.entries.filter(entry => (!criteria.rootId || entry.rootId === criteria.rootId) + && (criteria.parentId === undefined || entry.parentId === (criteria.parentId || null)) + && (!criteria.owner || criteria.owner === "all" || owners.get(entry.rootId) === criteria.owner) + && (!criteria.type || entry.kind === "file" && /\.pdf$/i.test(entry.name)) + && (criteria.largerThan === undefined || entry.kind === "file" && entry.logical !== null && entry.logical > criteria.largerThan)); + const page = matches.slice(offset, offset + STORAGE_SERVER_LIMITS.page), next = offset + page.length; + let nextCursor: string | null = null; + if (next < matches.length) { const encoded = Buffer.from(JSON.stringify({ id, filter: filterKey, offset: next, expires: Date.now() + 60_000 })).toString("base64url"); nextCursor = `${encoded}.${digest(encoded)}`; } + const fileEntries = matches.filter(entry => entry.kind === "file"); + const logicalSum = fileEntries.reduce((sum, entry) => sum + BigInt(entry.logical ?? 0), 0n); + const fileLogicalTotal = fileEntries.some(entry => entry.logical === null) || logicalSum > BigInt(Number.MAX_SAFE_INTEGER) ? null : Number(logicalSum); + return { scanId: id, fileLogicalTotal, entries: page, nextCursor, stale: Date.now() >= value.summary.staleAfter, matched: matches.length, totals: storageTotals(matches, value.summary.coverage === "complete") }; + }, + async references(principal: string, id: string, entryId: string): Promise { + const value = collection(principal, id), entry = value.entries.find(item => item.id === entryId); + if (!entry || entry.kind !== "file") throw new StorageUnavailable(404, "File observation unavailable."); + if (referenceJob) throw new StorageUnavailable(429, "A reference inspection is already running."); + const root = value.summary.roots.find(item => item.id === entry.rootId); + const prefix = root?.category === "scratch" ? ".brain/scratch/" : root?.category === "workspace" ? "workspaces/" : ""; + const evidence: StorageReferenceEvidence = { entryId, observedAt: Date.now(), state: "unavailable", sources: [], truncated: false, newestIndexedAt: null, freshness: "unchecked", unchecked: ["unindexed files", "unresolved links", "current source changes", "external consumers"] }; + if (!root || !["content", "scratch", "workspace"].includes(root.category)) return evidence; + const controller = new AbortController(), timer = setTimeout(() => controller.abort(), 5_000); + const revoked = setInterval(() => { if (!options.authorized(principal)) controller.abort(); }, 250); + const job = { controller, promise: Promise.resolve(evidence) }; + referenceJob = job; + job.promise = (async () => { + try { + const result = await storageReferences(options.brainPath, prefix + entry.path, controller.signal) as { ok?: boolean; value?: { sources?: Array<{path:string;indexedAt:string}>; truncated?: boolean }; snapshot?: {newestIndexedAt?:string|null} }; + if (result.ok && Array.isArray(result.value?.sources) && result.value.sources.length <= 200 && result.value.sources.every(source => typeof source.path === "string" && !source.path.startsWith("/") && !source.path.split("/").includes("..") && typeof source.indexedAt === "string")) { + evidence.state = "observed"; evidence.sources = result.value.sources; evidence.truncated = result.value.truncated === true; evidence.newestIndexedAt = result.snapshot?.newestIndexedAt ?? null; + } + } catch { /* Explicit unavailable evidence, never a fabricated zero-reference verdict. */ } + finally { clearTimeout(timer); clearInterval(revoked); if (referenceJob === job) referenceJob = null; } + authorized(principal); return evidence; + })(); + return job.promise; + }, + async idle() { await active?.promise; }, + async close() { closed = true; active?.controller.abort(); referenceJob?.controller.abort(); await Promise.allSettled([active?.promise, referenceJob?.promise]); current = null; previous = null; }, + }; +} +export type StorageCoordinator = ReturnType; diff --git a/packages/ui-server/src/storage/metadata.ts b/packages/ui-server/src/storage/metadata.ts new file mode 100644 index 0000000000..104d4f1cc8 --- /dev/null +++ b/packages/ui-server/src/storage/metadata.ts @@ -0,0 +1,98 @@ +/** Host-owned metadata worker. Exact configured roots only; never reads file contents. */ +import { constants, openSync, closeSync, fstatSync, readFileSync, statfsSync, type BigIntStats } from "node:fs"; +import { opendir } from "node:fs/promises"; +import { resolve } from "node:path"; +const O_PATH = 1 << 21; +const fdPath = (fd: number, name = "") => `/proc/self/fd/${fd}${name ? `/${name}` : ""}`; +const safe = (n: bigint) => n >= 0n && n <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(n) : null; +export interface MetadataRoot { id: string; path: string; recursive: boolean; optional?: boolean } +export interface MetadataEntry { + rootId: string; path: string; kind: "file" | "directory" | "symlink" | "special"; + object: string; version: string; device: string; logical: number | null; allocated: number | null; links: number; +} +export interface MetadataCapacity { + rootId: string; device: string; filesystem: string; identity: string | null; + total: number | null; used: number | null; free: number | null; available: number | null; reason: string | null; +} +export interface MetadataResult { entries: MetadataEntry[]; capacities: MetadataCapacity[]; errors: { rootId: string; reason: string }[]; omittedErrors: number } +function openConfigured(path: string): number { + const pieces = resolve(path).split("/").filter(Boolean); + let fd = openSync("/", O_PATH | constants.O_DIRECTORY); + try { + for (let i = 0; i < pieces.length; i++) { + const next = openSync(fdPath(fd, pieces[i]), O_PATH | constants.O_NOFOLLOW | (i < pieces.length - 1 ? constants.O_DIRECTORY : 0)); + closeSync(fd); fd = next; + } + return fd; + } catch (error) { closeSync(fd); throw error; } +} +const mount = (fd: number) => { + const id = /^mnt_id:\s*(\d+)$/m.exec(readFileSync(`/proc/self/fdinfo/${fd}`, "utf8"))?.[1]; + if (!id) throw Error("identity-unavailable"); + return id; +}; +/** GNU stat reads filesystem identity through our already-held descriptor. No path selected by a request. */ +async function filesystemIdentity(fd: number): Promise { + const child = Bun.spawn(["/usr/bin/stat", "-f", "--format=%i", `/proc/${process.pid}/fd/${fd}`], { stdin: "ignore", stdout: "pipe", stderr: "ignore", env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } }); + const timer = setTimeout(() => child.kill("SIGKILL"), 1000); + try { + let bytes = 0, output = ""; + for await (const part of child.stdout) { bytes += part.length; if (bytes > 128) { child.kill("SIGKILL"); break; } output += new TextDecoder().decode(part); } + return await child.exited === 0 && /^[0-9a-f]{1,32}\n$/i.test(output) && !/^0+\n$/.test(output) ? output.trim().toLowerCase() : null; + } finally { clearTimeout(timer); child.kill("SIGKILL"); await child.exited; } +} +export async function collectMetadata(roots: MetadataRoot[], options: { signal?: AbortSignal; entries?: number; milliseconds?: number } = {}): Promise { + const result: MetadataResult = { entries: [], capacities: [], errors: [], omittedErrors: 0 }; + const limit = Math.min(100_000, options.entries ?? 100_000), deadline = performance.now() + Math.min(30_000, options.milliseconds ?? 30_000); + let visited = 0, payloadBytes = 0; + const fail = (rootId: string, reason: string) => { if (result.errors.length < 100) result.errors.push({ rootId, reason }); else result.omittedErrors++; }; + const stop = () => options.signal?.aborted || performance.now() >= deadline || visited >= limit || payloadBytes >= 32 * 1024 * 1024; + if (process.platform !== "linux") { for (const root of roots) fail(root.id, "unsupported-platform"); return result; } + for (const root of roots.slice(0, 32)) { + if (stop()) { fail(root.id, options.signal?.aborted ? "cancelled" : visited >= limit ? "entry-budget" : payloadBytes >= 32 * 1024 * 1024 ? "output-budget" : "time-budget"); continue; } + let rootFd: number | undefined; + try { + // Server roots and exact sidecars never follow path-component redirects. + const canonical = resolve(root.path); + rootFd = openConfigured(canonical); + const first = fstatSync(rootFd, { bigint: true }), mountId = mount(rootFd); + if (first.isSymbolicLink() || (!first.isFile() && !first.isDirectory())) { fail(root.id, "unsupported-root"); continue; } + const fs = statfsSync(fdPath(rootFd), { bigint: true }), unsupported = fs.type === 0x794c7630n; + let identity: string | null = null; + try { identity = unsupported ? null : await filesystemIdentity(rootFd); } catch { /* explicit unknown relationship */ } + result.capacities.push({ rootId: root.id, device: String(first.dev), filesystem: String(fs.type), identity, + total: unsupported ? null : safe(fs.blocks * fs.bsize), used: unsupported ? null : safe((fs.blocks - fs.bfree) * fs.bsize), free: unsupported ? null : safe(fs.bfree * fs.bsize), available: unsupported ? null : safe(fs.bavail * fs.bsize), reason: unsupported ? "overlay-capacity-unknown" : identity ? null : "capacity-identity-unknown" }); + const visit = async (fd: number, path: string, depth: number, info: BigIntStats): Promise => { + if (stop()) { fail(root.id, options.signal?.aborted ? "cancelled" : visited >= limit ? "entry-budget" : payloadBytes >= 32 * 1024 * 1024 ? "output-budget" : "time-budget"); return; } + visited++; + if (mount(fd) !== mountId) { fail(root.id, "mount-boundary"); return; } + const entry: MetadataEntry = { rootId: root.id, path, kind: info.isFile() ? "file" : info.isDirectory() ? "directory" : info.isSymbolicLink() ? "symlink" : "special", + object: `${info.dev}:${info.ino}`, version: `${info.dev}:${info.ino}:${info.mode}:${info.size}:${info.mtimeNs}:${info.ctimeNs}:${info.nlink}`, device: String(info.dev), logical: info.isFile() ? safe(info.size) : 0, + allocated: safe(info.blocks * 512n), links: Number(info.nlink) }; + payloadBytes += Buffer.byteLength(JSON.stringify(entry)); + if (payloadBytes >= 32 * 1024 * 1024) { fail(root.id, "output-budget"); return; } + result.entries.push(entry); + if (!info.isDirectory() || !root.recursive) return; + if (depth >= 64) { fail(root.id, "depth-budget"); return; } + const directory = await opendir(fdPath(fd), { bufferSize: 32 }); + try { + for await (const entry of directory) { + if (stop()) { fail(root.id, options.signal?.aborted ? "cancelled" : visited >= limit ? "entry-budget" : payloadBytes >= 32 * 1024 * 1024 ? "output-budget" : "time-budget"); break; } + let child: number | undefined; + try { child = openSync(fdPath(fd, entry.name), O_PATH | constants.O_NOFOLLOW); await visit(child, path ? `${path}/${entry.name}` : entry.name, depth + 1, fstatSync(child, { bigint: true })); } + catch { visited++; fail(root.id, "entry-unavailable"); } + finally { if (child !== undefined) closeSync(child); } + if (visited % 32 === 0) await new Promise(done => setTimeout(done, 0)); + } + } finally { try { await directory.close(); } catch { /* async iterator already closed */ } } + }; + await visit(rootFd, "", 0, first); + const finalFd = openConfigured(canonical); + try { const final = fstatSync(finalFd, { bigint: true }); if (final.dev !== first.dev || final.ino !== first.ino || mount(finalFd) !== mountId) fail(root.id, "root-changed"); } + finally { closeSync(finalFd); } + } catch (error) { if (!root.optional || (error as NodeJS.ErrnoException).code !== "ENOENT") fail(root.id, "root-unavailable"); } + finally { if (rootFd !== undefined) closeSync(rootFd); } + } + if (roots.length > 32) fail("", "root-budget"); + return result; +} diff --git a/packages/ui-server/src/storage/process.ts b/packages/ui-server/src/storage/process.ts new file mode 100644 index 0000000000..cd19ca03ed --- /dev/null +++ b/packages/ui-server/src/storage/process.ts @@ -0,0 +1,68 @@ +import { fileURLToPath } from "node:url"; +import { dirname, resolve, relative, isAbsolute } from "node:path"; +import { realpathSync, readFileSync } from "node:fs"; +import { wrapCommand, killWrapped, type ExecWrapperConfig } from "@schlessera/brain-ui-sdk/server"; +import type { MetadataRoot, MetadataResult } from "./metadata.js"; +const inside = (root: string, path: string) => { const rel = relative(root, path); return !rel || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith("../")); }; +/** Resolve the installed host package's public bin, never a content repository's executable. */ +export function storageCli(brainPath: string): string[] { + const manifestPath = realpathSync(fileURLToPath(import.meta.resolve("@schlessera/brain/package.json"))); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + if (manifest.name !== "@schlessera/brain" || typeof manifest.bin?.brain !== "string") throw Error("storage-cli-unavailable"); + const bin = realpathSync(resolve(dirname(manifestPath), manifest.bin.brain)), brain = realpathSync(brainPath); + if (inside(brain, manifestPath) || inside(brain, bin) || !inside(dirname(manifestPath), bin)) throw Error("storage-cli-untrusted"); + return [process.execPath, "--config=/dev/null", "--no-env-file", bin]; +} +/** No shell, inherited secrets, repository config or unbounded stdout. Waits for cancellation cleanup. */ +export async function storageProcess(argv: string[], options: { signal: AbortSignal; env?: Record; exec?: ExecWrapperConfig; bytes?: number }): Promise { + options.signal.throwIfAborted(); + const exec = options.exec ?? {}; + const child = Bun.spawn(wrapCommand(argv, exec.wrapper), { cwd: "/", env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8", ...options.env }, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); + let reason: string | null = null, force: ReturnType | undefined; + const helperAttempts: Promise[] = []; + const signal = (value: NodeJS.Signals) => { + if (exec.wrapper) helperAttempts.push(new Promise(done => { + killWrapped(child, exec, value, () => { reason = "cancellation-unconfirmed"; }, { helperTimeoutMs: 1000, onComplete: done }); + })); + else { try { process.kill(-child.pid, value); } catch { child.kill(value); } } + }; + const stop = (why: string) => { if (reason) return; reason = why; signal("SIGTERM"); force = setTimeout(() => signal("SIGKILL"), 1500); }; + const abort = () => stop("cancelled"); + options.signal.addEventListener("abort", abort, { once: true }); + if (options.signal.aborted) abort(); + const timer = setTimeout(() => stop("time-budget"), 32_000); + let bytes = 0; + const read = async (stream: ReadableStream) => { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) { bytes += chunk.byteLength; if (bytes > (options.bytes ?? 64 * 1024 * 1024)) { stop("output-budget"); continue; } chunks.push(chunk); } + return Buffer.concat(chunks).toString("utf8"); + }; + try { + const [out, , code] = await Promise.all([read(child.stdout), read(child.stderr), child.exited]); + if (reason || code !== 0) throw Error(reason ?? "collector-unavailable"); + return JSON.parse(out); + } finally { + clearTimeout(timer); if (force) clearTimeout(force); + options.signal.removeEventListener("abort", abort); + signal("SIGKILL"); + // The shared helper bounds its own cleanup; do not release our scan while + // an asynchronous cancellation attempt is still running. + await Promise.allSettled(helperAttempts); + await child.exited; + } +} +export async function serverMetadata(roots: MetadataRoot[], signal: AbortSignal, entries: number): Promise { + const source = `import {collectMetadata} from ${JSON.stringify(import.meta.resolve("./metadata.js"))}; +const controller=new AbortController();process.once("SIGTERM",()=>controller.abort()); +console.log(JSON.stringify(await collectMetadata(${JSON.stringify(roots)},{signal:controller.signal,entries:${entries}})));`; + return await storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal }) as MetadataResult; +} + +/** Isolated supported core query, with the same trusted-installation boundary as inventory. */ +export async function storageReferences(brainPath: string, path: string, signal: AbortSignal): Promise { + storageCli(brainPath); // Refuse any host installation redirected into content before loading code. + const source = `import {createCoreQueryAccess} from ${JSON.stringify(import.meta.resolve("../core-queries.js"))}; +const feature=createCoreQueryAccess().storage?.(); +console.log(JSON.stringify(feature?.ok?feature.queries.readStorageReferences(${JSON.stringify({ brainPath, path })}):{ok:false}));`; + return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal, bytes: 1024 * 1024 }); +} diff --git a/packages/ui-server/tests/backend-registry.test.ts b/packages/ui-server/tests/backend-registry.test.ts index ce1b848a78..259469dd26 100644 --- a/packages/ui-server/tests/backend-registry.test.ts +++ b/packages/ui-server/tests/backend-registry.test.ts @@ -50,6 +50,23 @@ function registryFor( } describe("backend registry", () => { + test("Storage inspects only initialized backend owners without loading discovery", async () => { + const registry = registryFor({}); + expect(registry.peekBackends?.(), "STORAGE_BACKEND_LAZY_OWNER").toBeNull(); + const initialized = await registry.getBackends(); + expect(registry.peekBackends?.()).toBe(initialized); + const roots = initialized[0]!.storageRoots!(); + expect(roots.map(root => root.kind)).toEqual(["transcripts", "archive"]); + expect(roots.every(root => root.path.endsWith(fixtureBrain.root.replace(/[^a-zA-Z0-9]/g, "-")))).toBe(true); + const pi = registryFor({ AGENT_BACKEND: "pi" }); + expect(pi.peekBackends?.()).toBeNull(); + const backend = (await pi.getBackends())[0]!; + expect(backend.storageRoots!()).toHaveLength(1); + expect(backend.storageRoots!()[0]?.kind).toBe("transcripts"); + expect(backend.storageRoots!()[0]?.path.startsWith(fixtureBrain.root)).toBe(false); + }); + + test("Claude-only deployment has one backend and one tagged provider", async () => { const registry = registryFor({}); const backends = await registry.getBackends(); diff --git a/packages/ui-server/tests/http-inventory.test.ts b/packages/ui-server/tests/http-inventory.test.ts index 9e7863e51e..7c9787fbe8 100644 --- a/packages/ui-server/tests/http-inventory.test.ts +++ b/packages/ui-server/tests/http-inventory.test.ts @@ -20,7 +20,7 @@ test("real app method/path inventory equals the complete classified HTTP specifi expect(proseCounts).toEqual([declared.length, declared.length]); } const middleware = [...new Set(t.routes.filter((r) => r.method === "ALL").map((r) => r.path))].sort(); - expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*"]); + expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*", "/api/storage*"]); expect(t.routes.filter((r) => r.method === "ALL" && r.path === "/*")).toHaveLength(staticRoot ? 3 : 2); expect(t.routes.some((r) => r.method === "HEAD" || r.method === "OPTIONS")).toBe(false); } finally { await t.close(); } diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index ef3baf6316..bd0e19065b 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -16,6 +16,11 @@ type Check = { path?: string; init?: RequestInit; status?: number; body?: object // More detailed existing behavior checks are linked by the coverage matrix; // this map is cross-checked with the spec and the actual Hono mounts. const checks: Record = { + "GET /api/storage": { body: { version: 1, state: "idle", snapshot: null } }, + "POST /api/storage/scans": { init: json("POST", {}), status: 202, body: { version: 1, state: "scanning" } }, + "DELETE /api/storage/scans/:id": { path: "/api/storage/scans/missing", init: { method: "DELETE" }, status: 409, body: { error: "That scan is no longer active." } }, + "GET /api/storage/scans/:id/entries": { path: "/api/storage/scans/missing/entries", status: 404, body: { error: "Storage observation expired or unavailable; refresh." } }, + "GET /api/storage/scans/:id/entries/:entryId/references": { path: "/api/storage/scans/missing/entries/missing/references", status: 404, body: { error: "Storage observation expired or unavailable; refresh." } }, "GET /api/health": { body: { status: "healthy" } }, "GET /api/hygiene/review": { body: { review: { version: 1, status: "idle" }, action: null } }, "POST /api/hygiene/review": { init: json("POST", { operation: "pause" }), body: { review: { version: 1, status: "paused" }, action: null } }, diff --git a/packages/ui-server/tests/storage-collection.test.ts b/packages/ui-server/tests/storage-collection.test.ts new file mode 100644 index 0000000000..900210e357 --- /dev/null +++ b/packages/ui-server/tests/storage-collection.test.ts @@ -0,0 +1,159 @@ +import { expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, mkdirSync, writeFileSync, linkSync, rmSync, openSync, closeSync, ftruncateSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { collectStorage } from "../src/storage/collect"; +import { createStorageCoordinator } from "../src/storage/coordinator"; +function fixture() { + const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")), brain = join(root, "brain"), app = join(root, "app"); + mkdirSync(brain); mkdirSync(app); + writeFileSync(join(brain, "brain.config.ts"), 'throw Error("STORAGE_CONFIG_MUST_NOT_RUN");'); + for (const [name, size] of [["Troy.PDF", 5_000_001], ["Ithaca.pdf", 5_000_000]] as const) { const fd = openSync(join(brain, name), "wx"); try { ftruncateSync(fd, size); } finally { closeSync(fd); } } + linkSync(join(brain, "Troy.PDF"), join(app, "chart.pdf")); + return { root, brain, app, close() { rmSync(root, { recursive: true, force: true }); } }; +} +test("real trusted CLI and exact app stores share one object union and capacity observation", async () => { + const f = fixture(), dbPath = join(f.app, "state.sqlite"), db = new Database(dbPath); + try { + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE chart (id INTEGER); INSERT INTO chart VALUES (1)"); + const result = await collectStorage({ brainPath: f.brain, salt: "odysseus", stores: [ + { path: f.app, label: "App", category: "journal", owner: "app", recursive: true }, + { path: dbPath, label: "DB", category: "database", owner: "app", recursive: false }, + { path: dbPath + "-wal", label: "WAL", category: "database", owner: "app", recursive: false }, + ] }, "scan-odysseus", new AbortController().signal); + expect(result.summary.errors, "STORAGE_REAL_CLI_AVAILABLE").toEqual([]); + expect(result.entries.some(entry => entry.name === "Troy.PDF")).toBe(true); + const troy = result.entries.find(entry => entry.name === "Troy.PDF")!, alias = result.entries.find(entry => entry.name === "chart.pdf")!; + expect(troy.objectId, "STORAGE_CROSS_OWNER_OBJECT_IDENTITY").toBe(alias.objectId); + expect(result.summary.totals.all.logical! < result.summary.totals.brain.logical! + result.summary.totals.app.logical!, "STORAGE_CROSS_OWNER_UNION").toBe(true); + expect(result.summary.capacities, "STORAGE_SHARED_CAPACITY_ONCE").toHaveLength(1); + expect(result.summary.capacities[0]?.identity).toBe("resolved"); + expect(result.summary.capacities[0]?.measured).toBeNull(); + expect(JSON.stringify(result)).not.toContain(f.root); + } finally { db.close(); f.close(); } +}); +test("coordinator coalesces scans, uses strict PDF threshold, binds cursors, and revalidates revoked access", async () => { + const f = fixture(); let allowed = true; + for (let index = 0; index < 205; index++) writeFileSync(join(f.brain, `chart-${index}.txt`), "Odysseus"); + const storage = createStorageCoordinator({ brainPath: f.brain, stores: [], authorized: id => allowed && id === "odysseus" }); + try { + expect(() => storage.status("athena")).toThrow("human-owner"); + const first = storage.start("odysseus"), second = storage.start("odysseus"); + expect(second.activeScanId, "STORAGE_COALESCED_SCAN").toBe(first.activeScanId); + await storage.idle(); + const status = storage.status("odysseus"); + expect(status.lastFailure).toBeNull(); expect(status.snapshot).not.toBeNull(); + const id = status.snapshot!.id; + const pdf = storage.page("odysseus", id, { type: "pdf", largerThan: 5_000_000 }); + expect(pdf.entries.map(entry => entry.name), "STORAGE_STRICT_PDF_THRESHOLD").toEqual(["Troy.PDF"]); + const page = storage.page("odysseus", id, {}); + expect(page.entries).toHaveLength(200); expect(page.nextCursor).not.toBeNull(); + const next = storage.page("odysseus", id, { cursor: page.nextCursor! }); + expect(next.entries.some(entry => page.entries.some(previous => previous.id === entry.id))).toBe(false); + expect(() => storage.page("odysseus", id, { type: "pdf", cursor: page.nextCursor! })).toThrow("filters changed"); + allowed = false; + expect(() => storage.page("odysseus", id, {}), "STORAGE_REVOKED_RESULT_REFUSED").toThrow("human-owner"); + } finally { await storage.close(); f.close(); } +}); + +test("revocation during a real scan never publishes its result", async () => { + const f = fixture(); let allowed = true; + const storage = createStorageCoordinator({ brainPath: f.brain, stores: [], authorized: () => allowed }); + try { + storage.start("odysseus"); allowed = false; + await storage.idle(); allowed = true; + expect(storage.status("odysseus"), "STORAGE_REVOKED_INFLIGHT_DISCARDED").toMatchObject({ snapshot: null, lastFailure: { reason: "cancelled-or-revoked" } }); + } finally { await storage.close(); f.close(); } +}); + +test("bounded process cancellation reaps the real child", async () => { + const { storageProcess } = await import("../src/storage/process"); + const { readFileSync, existsSync } = await import("node:fs"); + const f = fixture(), pidFile = join(f.app, "pid"), controller = new AbortController(); + const pending = storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", `await Bun.write(${JSON.stringify(pidFile)},String(process.pid));setInterval(()=>{},1000);`], { signal: controller.signal }); + try { + for (let i = 0; i < 200 && !existsSync(pidFile); i++) await new Promise(done => setTimeout(done, 5)); + expect(existsSync(pidFile)).toBe(true); + const pid = Number(readFileSync(pidFile, "utf8")); + controller.abort(); + await expect(pending).rejects.toThrow("cancelled"); + expect(() => process.kill(pid, 0), "STORAGE_CANCEL_REAPS_CHILD").toThrow(); + } finally { controller.abort(); await pending.catch(() => {}); f.close(); } +}); + +test("separate real tmpfs capacities remain separate and unknown stores never look empty", async () => { + const f = fixture(), other = mkdtempSync("/dev/shm/storage-odysseus-"); + try { + writeFileSync(join(other, "Ithaca.pdf"), "chart"); + const result = await collectStorage({ brainPath: f.brain, salt: "odysseus", stores: [{ path: other, label: "Memory store", category: "temporary", owner: "app", recursive: true }], unavailable: [{ label: "Backup", category: "backup", owner: "operator", reason: "unavailable" }] }, "separate", new AbortController().signal); + expect(result.summary.capacities, "STORAGE_SEPARATE_CAPACITIES").toHaveLength(2); + expect(result.summary.capacities.every(value => value.identity === "resolved")).toBe(true); + expect(result.summary.totals.operator.logical, "STORAGE_UNKNOWN_OWNER_NOT_ZERO").toBeNull(); + } finally { rmSync(other, { recursive: true, force: true }); f.close(); } +}); + +test("filtered file-entry total counts aliases while its allocation union does not", async () => { + const f = fixture(); linkSync(join(f.brain, "Troy.PDF"), join(f.brain, "Poseidon.PDF")); + const storage = createStorageCoordinator({ brainPath: f.brain, stores: [], authorized: () => true }); + try { + storage.start("odysseus"); await storage.idle(); + const page = storage.page("odysseus", storage.status("odysseus").snapshot!.id, { type: "pdf", largerThan: 5_000_000 }); + expect(page.matched).toBe(2); + expect(page.fileLogicalTotal, "STORAGE_PER_ENTRY_SELECTION_TOTAL").toBe(10_000_002); + expect(page.totals.logical).toBe(5_000_001); + } finally { await storage.close(); f.close(); } +}); + +test("reference inspection runs the supported query without repository configuration and marks unchecked consumers", async () => { + const f = fixture(), db = new Database(join(f.brain, "brain.db")); + db.exec(`CREATE TABLE index_metadata(key TEXT,value TEXT); INSERT INTO index_metadata VALUES('schema_version','3'); + CREATE TABLE documents(id INTEGER PRIMARY KEY,path TEXT,title TEXT,type TEXT,summary TEXT,updated TEXT,indexed_at TEXT); + CREATE TABLE links(source_id INTEGER,target_id INTEGER,target TEXT); + INSERT INTO documents VALUES(1,'Troy.PDF','Chart','asset',NULL,NULL,'2026-07-12T00:00:00Z'),(2,'notes/voyage.md','Voyage','note',NULL,NULL,'2026-07-12T00:00:00Z'); + INSERT INTO links VALUES(2,1,'Troy.PDF');`); + db.close(); + const storage = createStorageCoordinator({ brainPath: f.brain, stores: [], authorized: () => true }); + try { + storage.start("odysseus"); await storage.idle(); const id = storage.status("odysseus").snapshot!.id; + const entry = storage.page("odysseus", id, { type: "pdf", largerThan: 5_000_000 }).entries[0]!; + expect(await storage.references("odysseus", id, entry.id), "STORAGE_SUPPORTED_REFERENCE_EVIDENCE").toMatchObject({ state: "observed", sources: [{ path: "notes/voyage.md", indexedAt: "2026-07-12T00:00:00Z" }], freshness: "unchecked" }); + } finally { await storage.close(); f.close(); } +}); + +test("a partial refresh preserves complete evidence, signed cursors expire, and shutdown drops cached data", async () => { + const f = fixture(), descriptor = Object.getOwnPropertyDescriptor(Date, "now")!; + const realNow = Date.now; let offset = 0, unavailable = false; + for (let index = 0; index < 205; index++) writeFileSync(join(f.brain, `chart-${index}.txt`), "Odysseus"); + const storage = createStorageCoordinator({ brainPath: f.brain, stores: [], authorized: () => true, storesNow: () => ({ stores: [], unavailable: unavailable ? [{ label: "Unknown store", owner: "app", category: "temporary", reason: "unavailable" }] : [] }) }); + try { + Object.defineProperty(Date, "now", { ...descriptor, value: () => realNow() + offset }); + storage.start("odysseus"); await storage.idle(); const first = storage.status("odysseus").snapshot!; + expect(first.coverage).toBe("complete"); + const page = storage.page("odysseus", first.id, {}); expect(page.nextCursor).not.toBeNull(); + offset = 6000; unavailable = true; storage.start("odysseus"); await storage.idle(); + expect(storage.status("odysseus").snapshot?.id, "STORAGE_PARTIAL_PRESERVES_COMPLETE").toBe(first.id); + expect(storage.status("odysseus").previous?.coverage).toBe("partial"); + offset = 62_000; + expect(() => storage.page("odysseus", first.id, { cursor: page.nextCursor! }), "STORAGE_CURSOR_EXPIRY").toThrow("expired"); + offset = 610_000; + expect(storage.status("odysseus").snapshot, "STORAGE_CACHE_EXPIRY").toBeNull(); + await storage.close(); expect(() => storage.start("odysseus")).toThrow("shutting down"); + } finally { Object.defineProperty(Date, "now", descriptor); await storage.close(); f.close(); } +}); + +test("wrapped cancellation waits for bounded killer helper cleanup", async () => { + const { storageProcess } = await import("../src/storage/process"); + const { chmodSync, existsSync, readFileSync } = await import("node:fs"); + const f = fixture(), wrapper = join(f.app, "wrapper"), killer = join(f.app, "killer"), childPid = join(f.app, "child-pid"), helperPids = join(f.app, "helper-pids"); + writeFileSync(wrapper, '#!/bin/sh\nexec "$@"\n'); chmodSync(wrapper, 0o700); + writeFileSync(killer, `#!${process.execPath}\nimport{appendFileSync}from"node:fs";appendFileSync(${JSON.stringify(helperPids)},String(process.pid)+"\\n");setInterval(()=>{},1000);\n`); chmodSync(killer, 0o700); + const controller = new AbortController(), pending = storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", `await Bun.write(${JSON.stringify(childPid)},String(process.pid));setInterval(()=>{},1000);`], { signal: controller.signal, exec: { wrapper, killer } }); + try { + for (let i = 0; i < 200 && !existsSync(childPid); i++) await new Promise(done => setTimeout(done, 5)); + expect(existsSync(childPid)).toBe(true); controller.abort(); await expect(pending).rejects.toThrow(); + await new Promise(done => setTimeout(done, 50)); + expect(existsSync(helperPids)).toBe(true); + for (const pid of readFileSync(helperPids, "utf8").trim().split("\n").map(Number)) expect(() => process.kill(pid, 0), "STORAGE_KILL_HELPERS_REAPED").toThrow(); + } finally { controller.abort(); await pending.catch(() => {}); await new Promise(done => setTimeout(done, 1200)); f.close(); } +}); diff --git a/packages/ui-server/tests/storage-metadata.test.ts b/packages/ui-server/tests/storage-metadata.test.ts new file mode 100644 index 0000000000..deff3d4bdd --- /dev/null +++ b/packages/ui-server/tests/storage-metadata.test.ts @@ -0,0 +1,53 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, linkSync, symlinkSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { collectMetadata } from "../src/storage/metadata"; +import { storageTotals } from "../src/storage/collect"; +import type { StorageFileEntry } from "@schlessera/brain-ui-sdk/protocol"; + +test("server metadata stays within exact roots and does not follow descendants or sidecar redirects", async () => { + const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")); + try { + const store = join(root, "journal"); mkdirSync(store); + writeFileSync(join(root, "credential"), "private fixture bytes"); + writeFileSync(join(store, "Troy.PDF"), "chart"); + symlinkSync(join(root, "credential"), join(store, "link")); + symlinkSync(join(root, "credential"), join(root, "app.db-wal")); + const result = await collectMetadata([{ id: "journal", path: store, recursive: true }, { id: "sidecar", path: join(root, "app.db-wal"), recursive: false }]); + expect(result.entries.map(entry => entry.path).sort(), "STORAGE_EXACT_ROOTS").toEqual(["", "Troy.PDF", "link"]); + expect(result.entries.find(entry => entry.path === "link")?.kind).toBe("symlink"); + expect(result.errors, "STORAGE_SIDECAR_REDIRECT_REFUSED").toContainEqual({ rootId: "sidecar", reason: "unsupported-root" }); + expect(result.capacities).toHaveLength(1); + expect(result.capacities[0]?.rootId).toBe("journal"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("server metadata records hard-link identity once in the object union and preserves a partial budget", async () => { + const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")); + try { + writeFileSync(join(root, "Troy.pdf"), "12345"); linkSync(join(root, "Troy.pdf"), join(root, "Ithaca.pdf")); + const result = await collectMetadata([{ id: "charts", path: root, recursive: true }]); + const files = result.entries.filter(entry => entry.kind === "file"); + expect(files).toHaveLength(2); expect(files[0]!.object).toBe(files[1]!.object); + const entries = files.map(entry => ({ objectId: entry.object, logical: entry.logical, allocated: entry.allocated })) as StorageFileEntry[]; + expect(storageTotals(entries).logical, "STORAGE_OBJECT_UNION").toBe(5); + const partial = await collectMetadata([{ id: "charts", path: root, recursive: true }], { entries: 1 }); + expect(partial.entries).toHaveLength(1); expect(partial.errors).toContainEqual({ rootId: "charts", reason: "entry-budget" }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("actual same-device bind descendants are not traversed by the server metadata worker", async () => { + const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")); + try { + mkdirSync(join(root, "source")); mkdirSync(join(root, "mounted")); + writeFileSync(join(root, "source", "Troy.pdf"), "chart"); + const script = `import {collectMetadata} from ${JSON.stringify(import.meta.resolve("../src/storage/metadata.ts"))}; console.log(JSON.stringify(await collectMetadata([{id:"odysseus",path:${JSON.stringify(root)},recursive:true}])));`; + const child = Bun.spawn(["bwrap", "--unshare-user", "--unshare-net", "--bind", "/", "/", "--bind", join(root, "source"), join(root, "mounted"), "--proc", "/proc", "--dev", "/dev", process.execPath, "--config=/dev/null", "--no-env-file", "-e", script], { stdout: "pipe", stderr: "pipe" }); + const [out, err, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + expect(code, err).toBe(0); + const result = JSON.parse(out); + expect(result.errors, "STORAGE_BIND_MOUNT_REFUSED").toContainEqual({ rootId: "odysseus", reason: "mount-boundary" }); + expect(result.entries.some((entry: {path:string}) => entry.path === "mounted/Troy.pdf")).toBe(false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-server/tests/storage-routes.test.ts b/packages/ui-server/tests/storage-routes.test.ts new file mode 100644 index 0000000000..5e46ef226b --- /dev/null +++ b/packages/ui-server/tests/storage-routes.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from "bun:test"; +import { httpContractApp } from "./helpers/http-contract-app"; + +test("Storage inventory is mounted for the authenticated human without starting a scan on read", async () => { + const fixture = await httpContractApp(); + try { + const response = await fixture.fetch("/api/storage"); + expect(response.status, "STORAGE_OWNER_ROUTE").toBe(200); + expect(await response.json()).toMatchObject({ version: 1, state: "idle", snapshot: null }); + } finally { await fixture.close(); } +}); + +test("mounted Storage routes reject delegated and revoked principals and arbitrary roots", async () => { + const { generateSignedCookie } = await import("hono/cookie"); + const { createPrincipal } = await import("../src/db/principals"); + const secret = "storage-odysseus-cookie-secret-0123456789"; + const fixture = await httpContractApp({ env: { AUTH_MODE: "password", BRAIN_UI_PASSWORD_HASH: "unused-fixture-hash", COOKIE_SECRET: secret, BRAIN_UI_ALLOW_LOOPBACK_ORIGIN: "1" } }); + try { + const owner = createPrincipal(fixture.app.db, { authMethod: "password", label: "Odysseus", ttlSeconds: 3600 }); + const agent = createPrincipal(fixture.app.db, { authMethod: "delegated", createdBy: owner.id, label: "Athena", ttlSeconds: 3600 }); + const cookie = async (id: string) => (await generateSignedCookie("brain_ui_session", id, secret)).split(";")[0]!; + const ownerCookie = await cookie(owner.id), agentCookie = await cookie(agent.id); + for (const [method, path] of [["GET", "/api/storage"], ["POST", "/api/storage/scans"], ["DELETE", "/api/storage/scans/unknown"], ["GET", "/api/storage/scans/unknown/entries"], ["GET", "/api/storage/scans/unknown/entries/unknown/references"]]) { + const response = await fixture.fetch(path!, { method, headers: { cookie: agentCookie, "content-type": "application/json" }, ...(method === "POST" ? { body: "{}" } : {}) }); + expect(response.status, "STORAGE_DELEGATED_DENIED").toBe(403); + expect((await fixture.fetch(path!, { method, ...(method === "POST" ? { headers: { "content-type": "application/json" }, body: "{}" } : {}) })).status).toBe(401); + } + const post = (body: unknown) => fixture.fetch("/api/storage/scans", { method: "POST", headers: { cookie: ownerCookie, "content-type": "application/json" }, body: JSON.stringify(body) }); + expect((await fixture.fetch("/api/storage/scans", { method: "POST", headers: { cookie: ownerCookie, "content-type": "text/plain" }, body: "{}" })).status, "STORAGE_SCAN_REQUIRES_JSON").toBe(415); + expect((await fixture.fetch("/api/storage/scans", { method: "POST", headers: { cookie: ownerCookie, "content-type": "application/json", origin: "https://troy.example" }, body: "{}" })).status, "STORAGE_SCAN_ORIGIN_GUARD").toBe(403); + expect((await post({ root: "/" })).status, "STORAGE_REQUEST_CANNOT_EXPAND_ROOTS").toBe(400); + expect((await post({})).status).toBe(202); + let result: any; + for (let attempts = 0; attempts < 100; attempts++) { result = await (await fixture.fetch("/api/storage", { headers: { cookie: ownerCookie } })).json(); if (result.state === "idle") break; await new Promise(done => setTimeout(done, 20)); } + expect(result.state).toBe("idle"); expect(result.snapshot).not.toBeNull(); + const invalid = await fixture.fetch(`/api/storage/scans/${result.snapshot.id}/entries?cursor=forged`, { headers: { cookie: ownerCookie } }); + expect(invalid.status, "STORAGE_INVALID_CURSOR_IS_CLIENT_ERROR").toBe(400); + fixture.app.db.query("UPDATE principals SET revoked_at=? WHERE id=?").run(Date.now(), owner.id); + expect((await fixture.fetch("/api/storage", { headers: { cookie: ownerCookie } })).status, "STORAGE_REVOKED_OWNER_DENIED").toBe(401); + } finally { await fixture.close(); } +}); + +test("an unavailable backend owner leaves the real inventory usable with an explicit unknown row", async () => { + const { createStaticBackendRegistry } = await import("../src/agent/backend"); + const { makeFakeBackend } = await import("./helpers/fake-backend"); + const backend = makeFakeBackend({ id: "odysseus" }); + backend.storageRoots = () => { throw Error("owner scope unavailable"); }; + const fixture = await httpContractApp({ registry: createStaticBackendRegistry([backend]) }); + try { + expect((await fixture.fetch("/api/storage/scans", { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status).toBe(202); + let result: any; + for (let i = 0; i < 100; i++) { result = await (await fixture.fetch("/api/storage")).json(); if (result.state === "idle") break; await new Promise(done => setTimeout(done, 20)); } + expect(result.lastFailure).toBeNull(); + expect(result.snapshot.roots.some((root: {reason: string}) => root.reason === "backend-roots-unavailable"), "STORAGE_FAILED_OWNER_IS_PARTIAL").toBe(true); + expect(result.snapshot.entries).toBeGreaterThan(0); + } finally { await fixture.close(); } +}); From 247627603c6d2392d161801719d7e6a340503bb4 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 11:58:41 +0200 Subject: [PATCH 04/44] CONTRACT: keep host Storage inspection metadata-only --- .changeset/odysseus-storage-server.md | 2 +- api-report/core.txt | 99 +++++++++++++++++++ docs/decisions/design-kit/storage.md | 3 +- docs/http-api-coverage.md | 2 +- docs/http-api.md | 5 + docs/integration-contract/package-api.md | 30 ++++++ packages/core/README.md | 8 ++ packages/core/package.json | 5 + packages/core/src/lib/storage-inventory.ts | 50 +++++++++- packages/core/src/lib/storage-process.ts | 6 +- packages/core/src/storage.ts | 16 +++ .../core/tests/storage-host-reader.test.ts | 44 +++++++++ packages/ui-server/src/app.ts | 2 +- packages/ui-server/src/storage/collect.ts | 12 +-- packages/ui-server/src/storage/process.ts | 37 ++++--- .../tests/storage-collection.test.ts | 26 +++-- scripts/ci-pack.ts | 4 +- 17 files changed, 298 insertions(+), 53 deletions(-) create mode 100644 packages/core/src/storage.ts create mode 100644 packages/core/tests/storage-host-reader.test.ts diff --git a/.changeset/odysseus-storage-server.md b/.changeset/odysseus-storage-server.md index 780931881f..f72c52a9a7 100644 --- a/.changeset/odysseus-storage-server.md +++ b/.changeset/odysseus-storage-server.md @@ -6,4 +6,4 @@ "@schlessera/brain-backend-pi": minor --- -Add human-only bounded Storage inventory and reference inspection, with explicit store ownership, deduplicated object and capacity observations, partial coverage, configured backup roots, and backend-owned transcript metadata. This foundation does not register a Storage pane or execute cleanup. +Add the supported metadata-only `@schlessera/brain/storage` host reader, which never executes Git or repository configuration, and human-only bounded Storage inventory and reference inspection, with explicit store ownership, deduplicated object and capacity observations, partial coverage, configured backup roots, and backend-owned transcript metadata. This foundation does not register a Storage pane or execute cleanup. diff --git a/api-report/core.txt b/api-report/core.txt index 44486462d7..987e2a676a 100644 --- a/api-report/core.txt +++ b/api-report/core.txt @@ -194,6 +194,11 @@ export "./queries" (./src/queries/index.ts) readStorageReferences readVoiceVocabulary +export "./storage" (./src/storage.ts) + type StorageInventory + type StorageReadOptions + readStorageInventory + export "./testing" (./src/testing/index.ts) type AgentRunnerContractHarness type CompletionProviderContractHarness @@ -783,6 +788,8 @@ signatures (every public export and the types it is made of) const SCHEMA_VERSION: 15; SCRATCH_DIR (packages/core/src/lib/scratch.ts) const SCRATCH_DIR: ".brain/scratch"; + STORAGE_LIMITS (packages/core/src/lib/storage-inventory.ts) + const STORAGE_LIMITS: Readonly; ScratchFailure (packages/core/src/lib/scratch.ts) export interface ScratchFailure { path: string; @@ -901,6 +908,96 @@ signatures (every public export and the types it is made of) evaluatedAt: string; verdicts: StatsTrend[]; } + StorageCapacity (packages/core/src/lib/storage-inventory.ts) + export interface StorageCapacity { + id: string; + device: string; + mountId: string; + filesystemType: string; + totalBytes: number | null; + usedBytes: number | null; + freeBytes: number | null; + availableBytes: number | null; + reason: string | null; + provenance: "statfs"; + quota: null; + quotaReason: "unsupported"; + } + StorageEntry (packages/core/src/lib/storage-inventory.ts) + export interface StorageEntry { + id: string; + parentId: string | null; + rootId: string; + path: string; + name: string; + kind: StorageKind; + objectId: string; + capacityId: string; + logicalBytes: number | null; + allocatedBytes: number | null; + measurementReason: string | null; + version: string; + links: number; + pdfOver5MB: boolean; + aliasOf: string | null; + } + StorageInventory (packages/core/src/lib/storage-inventory.ts) + export interface StorageInventory { + schema_version: 1; + scanId: string; + revision: string; + startedAt: string; + finishedAt: string; + expiresAt: string; + status: "complete" | "partial" | "unsupported"; + consistency: "live-observation"; + allocationBasis: "512 * st_blocks; shared extents may be charged more than once"; + roots: { + id: string; + kind: "brain" | "git"; + path: string; + }[]; + entries: StorageEntry[]; + capacities: StorageCapacity[]; + pages: { + cursor: string; + parentId: string | null; + entryIds: string[]; + nextCursor: string | null; + }[]; + totals: { + entryLogicalBytes: number | null; + objectLogicalBytes: number | null; + objectAllocatedBytes: number | null; + objects: number; + entries: number; + lowerBound: boolean; + }; + errors: { + path: string; + reason: string; + }[]; + omittedErrors: number; + limits: typeof STORAGE_LIMITS; + } + StorageKind (packages/core/src/lib/storage-inventory.ts) + export type StorageKind = "file" | "directory" | "symlink" | "special"; + StorageLimits (packages/core/src/lib/storage-inventory.ts) + interface StorageLimits { + entries: number; + depth: number; + milliseconds: number; + errors: number; + page: number; + freshness: number; + gitBytes: number; + } + StorageReadOptions (packages/core/src/storage.ts) + export interface StorageReadOptions { + brainPath: string; + gitRoots?: readonly string[]; + signal?: AbortSignal; + } Subgraph (packages/core/src/queries/types.ts) export interface Subgraph { nodes: Node[]; @@ -1364,6 +1461,8 @@ signatures (every public export and the types it is made of) depth?: number; direction?: "outgoing" | "incoming" | "both"; }): QueryResult; + readStorageInventory (packages/core/src/storage.ts) + export function readStorageInventory(options: StorageReadOptions): Promise; readStorageReferences (packages/core/src/queries/index.ts) export function readStorageReferences(opts: Root & { path: string; diff --git a/docs/decisions/design-kit/storage.md b/docs/decisions/design-kit/storage.md index 99b0fd5e4a..f1fa704115 100644 --- a/docs/decisions/design-kit/storage.md +++ b/docs/decisions/design-kit/storage.md @@ -464,7 +464,8 @@ success and one conflict. Every step is available from a row and a button. Tracked implementation belongs under [#1779](https://github.com/schlessera/brain-kit/issues/1779); the issue tracker owns sequencing and completion. Core owns deterministic -content/Git inventory via supported CLI JSON machinery. ui-server resolves +content/Git inventory via the supported CLI and metadata-only public reader. +The host reader never runs Git or repository configuration. ui-server resolves its configured owners, merges capacities, authenticates transport and owns receipts. Backends keep transcript layout/lifetime ownership. ui-react composes the Settings pane; ui-kit supplies presentational map/list pieces if required. diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index 5733508b02..110e073562 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -462,6 +462,6 @@ The real Chrome `turn-journal-runtime.test.ts` exercises whole-turn review, reading focus/Escape, waiting and partial receipts, Undo and lost-response readback. ST: [Storage route authorization and mounted scans](../packages/ui-server/tests/storage-routes.test.ts), -[real CLI/store union and paging](../packages/ui-server/tests/storage-collection.test.ts), +[real public-reader/store union and paging](../packages/ui-server/tests/storage-collection.test.ts), [exact metadata roots](../packages/ui-server/tests/storage-metadata.test.ts), and [bounded index evidence](../packages/core/tests/storage-references.test.ts). diff --git a/docs/http-api.md b/docs/http-api.md index 7f20105961..79511718f9 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1218,6 +1218,11 @@ exclusive "mine" or "theirs" ownership. ## Storage inventory +The server uses the installed `@schlessera/brain/storage` metadata reader. It +never executes Git, the brain CLI, repository configuration, hooks or filters +for inspection. Only fixed Git administrative location markers and approved +host roots participate in discovery. + These five operations require a current `owner` or `ambient` principal. Agent, delegated and restricted access is refused (403); unauthenticated/revoked access is refused (401 by normal authentication). Authorization is checked again at diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index a6b5a4c1d7..dd762a0825 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3210,3 +3210,33 @@ only already initialized owners, never passes these paths to a worker bridge, and converts them into opaque root IDs before transport. Older backends remain valid and report unavailable transcript accounting. Claude supplies its exact brain-specific live/archive leaves; Pi supplies its already resolved session root. + +### Host Storage reader (additive, #2010) + +`@schlessera/brain/storage` supports host-shell authors who need bounded live +physical metadata without evaluating a content repository. Its sole operation +`readStorageInventory({brainPath, gitRoots?, signal?})` returns a promise of the +version-1 Storage inventory envelope, with the existing 100,000-entry/30-second +limits and isolated/reaped filesystem worker. `brainPath` is absolute; +`gitRoots` contains at most sixteen explicitly approved absolute external stores. +Invalid roots reject before traversal. The host must enforce user authorization; +this is not an agent-facing filesystem browser. + +This entry executes neither Git nor the brain CLI, and loads no repository +configuration/providers. It reads only fixed Git location markers (`.git`, +`commondir`, `objects/info/alternates`) through bounded non-followed descriptors, +approving external stores before reading their markers. It supports ordinary +worktrees and linked-worktree/alternate layouts; unsupported markers, redirects, +mount transitions and encodings remain partial evidence. It does not implement +Git configuration semantics or discover a repository from parent directories. +Host-selected root aliases retain the inventory's canonical-root semantics. + +The ordinary standalone `brain storage inventory` CLI retains its existing Git +resolver contract and must run through the maintenance boundary when invoked by +a governed server. The HTTP inventory uses the public metadata reader so it +measures the canonical live objects, not a maintenance worker's native copy. +The same structural inventory schema is consumed; available allocation is still +a qualified charge and never a reclaimability guarantee. Bun/types/default +exports resolve source/declarations/build from the same release. Older core +installations without this entry produce unavailable inventory, with no CLI or +internal-import fallback. diff --git a/packages/core/README.md b/packages/core/README.md index ee17b774a2..731800f617 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -67,3 +67,11 @@ index registry and completion-provider resolution `@schlessera/brain/internal` holds first-party native-handle search/context and write helpers. It has no compatibility guarantee and requires the same lockstep version; integrations should use the supported query results. + +### Host storage integration + +Host-shell authors can call `readStorageInventory` from +`@schlessera/brain/storage` to obtain bounded live metadata without executing +Git, repository configuration or the brain CLI. Enforce owner authorization and +choose explicit roots at the host boundary. The [host reader contract](../../docs/integration-contract/package-api.md#host-storage-reader-additive-2010) +defines resource bounds, Git marker support, unknown evidence and cancellation. diff --git a/packages/core/package.json b/packages/core/package.json index 49bfe214ea..bbdbe24dcd 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,6 +61,11 @@ "bun": "./src/internal-graph.ts", "types": "./dist/internal-graph.d.ts", "default": "./dist/internal-graph.js" + }, + "./storage": { + "bun": "./src/storage.ts", + "types": "./dist/storage.d.ts", + "default": "./dist/storage.js" } }, "files": [ diff --git a/packages/core/src/lib/storage-inventory.ts b/packages/core/src/lib/storage-inventory.ts index cd3834f1e9..7d738b358d 100644 --- a/packages/core/src/lib/storage-inventory.ts +++ b/packages/core/src/lib/storage-inventory.ts @@ -34,6 +34,8 @@ export interface StorageInventory { limits: typeof STORAGE_LIMITS; } interface Options { + /** Internal host-reader mode: Git administrative metadata only, no executable. */ + metadataGit?: boolean; signal?: AbortSignal; /** Trusted host configuration only; never a request parameter. */ gitRoots?: readonly string[]; @@ -61,6 +63,50 @@ function openRoot(path: string): number { } const inside = (root: string, path: string) => { const rel = relative(root, path); return rel === "" || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith(`..${sep}`)); }; +/** Read only Git's fixed location markers through held, non-followed descriptors. + * Config/includes, hooks, filters, helpers and object contents are never evaluated. */ +function metadataGitPaths(root: string, allowed: readonly string[]): { path: string; objects: boolean }[] { + const approved = (path: string) => inside(root, path) || allowed.some(value => inside(value, path)); + const text = (fd: number, parent: number): string => { + const stat = fstatSync(fd, { bigint: true }); + if (!stat.isFile() || mountId(fd) !== mountId(parent) || stat.size > BigInt(STORAGE_LIMITS.gitBytes)) throw Error("git-metadata-unsupported"); + const file = openSync(at(fd), constants.O_RDONLY | constants.O_NONBLOCK); + try { + const buffer = Buffer.alloc(STORAGE_LIMITS.gitBytes + 1), size = readSync(file, buffer, 0, buffer.length, 0); + if (size > STORAGE_LIMITS.gitBytes || version(fstatSync(file, { bigint: true })) !== version(stat)) throw Error("git-metadata-changed-or-over-budget"); + const value = buffer.subarray(0, size).toString("utf8").replace(/\n$/, ""); + if (!value || /[\r\n\0\ufffd]/.test(value)) throw Error("git-path-encoding-unsupported"); + return value; + } finally { closeSync(file); } + }; + let brain: number | undefined, marker: number | undefined, git: number | undefined, commonMarker: number | undefined; + try { + brain = openRoot(root); + try { marker = openSync(at(brain, ".git"), PATH | constants.O_NOFOLLOW); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } + const stat = fstatSync(marker, { bigint: true }); + let gitDir = resolve(root, ".git"); + if (mountId(marker) !== mountId(brain)) throw Error("mount-boundary"); + if (!stat.isDirectory()) { + const value = text(marker, brain); + if (!value.startsWith("gitdir: ")) throw Error("git-location-marker-unsupported"); + gitDir = resolve(root, value.slice(8)); + } + if (!approved(gitDir)) throw Error("external-git-root-not-approved"); + git = openRoot(gitDir); + let common = gitDir; + try { + commonMarker = openSync(at(git, "commondir"), PATH | constants.O_NOFOLLOW); + common = resolve(gitDir, text(commonMarker, git)); + } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (!approved(common)) throw Error("external-git-root-not-approved"); + const objects = resolve(common, "objects"); + return [...new Set([gitDir, common, objects])].sort(compare).map(path => ({ path, objects: path === objects })); + } finally { + for (const fd of [commonMarker, git, marker, brain]) if (fd !== undefined) closeSync(fd); + } +} + /** Bounded output and lifetime; every cancellation kills and awaits the child. No shell, hooks, filters or maintenance. */ async function gitPaths(root: string, signal: AbortSignal, deadline: number): Promise<{ path: string; objects: boolean }[]> { const env = Object.fromEntries(Object.entries(inheritedEnv({})).filter(([key]) => !key.startsWith("GIT_"))); @@ -205,7 +251,7 @@ export async function collectStorageInventory(rootPath: string, options: Options const roots: { path: string; kind: "brain" | "git" }[] = [{ path: root, kind: "brain" }]; if (!stop()) { try { - const paths = await gitPaths(root, signal, deadline); + const paths = options.metadataGit ? metadataGitPaths(root, allowedGitRoots) : await gitPaths(root, signal, deadline); const inspected = new Set(); // Only read alternate metadata after approving and opening its object // directory. `git count-objects` would traverse unapproved alternates. @@ -213,7 +259,7 @@ export async function collectStorageInventory(rootPath: string, options: Options if (stop()) break; if (index >= 64) { fail("git", "git-root-budget"); break; } const candidate = paths[index]; - const path = canonical(candidate.path); + const path = options.metadataGit ? resolve(candidate.path) : canonical(candidate.path); const resolutionId = `${candidate.objects ? "objects" : "store"}:${path}`; if (inspected.has(resolutionId)) continue; inspected.add(resolutionId); diff --git a/packages/core/src/lib/storage-process.ts b/packages/core/src/lib/storage-process.ts index 2cbee07ba4..df35e474ef 100644 --- a/packages/core/src/lib/storage-process.ts +++ b/packages/core/src/lib/storage-process.ts @@ -1,14 +1,14 @@ import { collectStorageInventory, STORAGE_LIMITS, type StorageInventory } from "./storage-inventory.js"; /** Isolate blocking filesystem calls from the CLI coordinator. No writable state or config loading. */ -export async function collectStorageProcess(root: string, gitRoots: readonly string[], signal: AbortSignal): Promise { - if (signal.aborted || process.platform !== "linux") return collectStorageInventory(root, { signal }); +export async function collectStorageProcess(root: string, gitRoots: readonly string[], signal: AbortSignal, metadataGit = false): Promise { + if (signal.aborted || process.platform !== "linux") return collectStorageInventory(root, { signal, metadataGit }); const startedAt = new Date().toISOString(); const source = `import {collectStorageInventory} from ${JSON.stringify(import.meta.resolve("./storage-inventory.js"))}; const controller=new AbortController(); process.once("SIGTERM",()=>controller.abort()); process.once("SIGINT",()=>controller.abort()); -console.log(JSON.stringify(await collectStorageInventory(${JSON.stringify(root)},{gitRoots:${JSON.stringify(gitRoots)},signal:controller.signal})));`; +console.log(JSON.stringify(await collectStorageInventory(${JSON.stringify(root)},{gitRoots:${JSON.stringify(gitRoots)},signal:controller.signal,metadataGit:${metadataGit}})));`; const child = Bun.spawn([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); let reason: string | undefined, killTimer: ReturnType | undefined; const kill = () => { try { process.kill(-child.pid, "SIGKILL"); } catch { /* already reaped */ } }; diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts new file mode 100644 index 0000000000..bc1c0fb994 --- /dev/null +++ b/packages/core/src/storage.ts @@ -0,0 +1,16 @@ +/** Supported metadata-only reader for host shells; never invokes Git or the brain CLI. */ +import { isAbsolute } from "node:path"; +import { collectStorageProcess } from "./lib/storage-process.js"; +import type { StorageInventory } from "./lib/storage-inventory.js"; +export type { StorageInventory } from "./lib/storage-inventory.js"; +export interface StorageReadOptions { + brainPath: string; + /** Explicitly approved external local Git stores; at most sixteen roots. */ + gitRoots?: readonly string[]; + signal?: AbortSignal; +} +export async function readStorageInventory(options: StorageReadOptions): Promise { + const paths = [options.brainPath, ...options.gitRoots ?? []]; + if (paths.length > 17 || paths.some(path => typeof path !== "string" || !isAbsolute(path) || path.length > 4096 || path.includes("\0"))) throw Error("Storage requires bounded absolute host roots."); + return collectStorageProcess(options.brainPath, options.gitRoots ?? [], options.signal ?? new AbortController().signal, true); +} diff --git a/packages/core/tests/storage-host-reader.test.ts b/packages/core/tests/storage-host-reader.test.ts new file mode 100644 index 0000000000..060a59bb9c --- /dev/null +++ b/packages/core/tests/storage-host-reader.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from "bun:test"; +import { chmodSync, existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +test("host metadata mode never executes Git or repository configuration", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-host-")), brain = join(root, "brain"), bin = join(root, "bin"), marker = join(root, "git-executed"); + mkdirSync(brain); mkdirSync(bin); mkdirSync(join(brain, ".git")); mkdirSync(join(brain, ".git", "objects")); + writeFileSync(join(brain, "brain.config.ts"), 'throw Error("REPOSITORY_CONFIG_EXECUTED");'); + writeFileSync(join(brain, ".git", "config"), '[include]\npath = /unapproved-host-config\n'); + writeFileSync(join(bin, "git"), `#!${process.execPath}\nawait Bun.write(${JSON.stringify(marker)},"Git executed");process.exit(1);\n`); chmodSync(join(bin, "git"), 0o700); + const source = `import {readStorageInventory} from ${JSON.stringify(import.meta.resolve("../src/storage.ts"))};console.log(JSON.stringify(await readStorageInventory({brainPath:${JSON.stringify(brain)}})));`; + try { + const child = Bun.spawn([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { env: { ...process.env, PATH: `${bin}:/usr/bin:/bin` }, stdout: "pipe", stderr: "pipe" }); + const [out, err, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + expect(code, err).toBe(0); + expect(existsSync(marker), "STORAGE_HOST_NEVER_RUNS_GIT").toBe(false); + expect(JSON.parse(out).status).toBe("complete"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("public host reader resolves approved worktree markers and refuses redirected administrative files", async () => { + const { readStorageInventory } = await import("../src/storage"); + const { symlinkSync, unlinkSync } = await import("node:fs"); + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-host-")), repo = join(root, "repo"), worktree = join(root, "voyage"), alternate = join(root, "alternate"); + mkdirSync(repo); mkdirSync(alternate); + const git = (path: string, ...args: string[]) => { const p = Bun.spawnSync(["git", "-C", path, ...args], { stdout: "pipe", stderr: "pipe" }); expect(p.exitCode, p.stderr.toString()).toBe(0); }; + try { + git(repo, "init"); git(alternate, "init", "--bare"); + writeFileSync(join(repo, "Ithaca.md"), "Odysseus returns to Ithaca.\n"); + git(repo, "add", "."); git(repo, "-c", "user.name=Odysseus", "-c", "user.email=odysseus@example.invalid", "commit", "-m", "Ithaca"); + git(repo, "worktree", "add", "-b", "voyage", worktree); + writeFileSync(join(repo, ".git", "objects", "info", "alternates"), join(alternate, "objects") + "\n"); + expect((await readStorageInventory({ brainPath: worktree })).errors, "STORAGE_HOST_APPROVED_ROOTS_ONLY").toContainEqual({ path: "git", reason: "external-git-root-not-approved" }); + const result = await readStorageInventory({ brainPath: worktree, gitRoots: [repo, alternate] }); + expect(result.status, JSON.stringify(result.errors)).toBe("complete"); + expect(result.roots.some(value => value.path === join(alternate, "objects"))).toBe(true); + const common = join(repo, ".git", "worktrees", "voyage", "commondir"), outside = join(root, "outside-marker"); + unlinkSync(common); writeFileSync(outside, "../..\n"); symlinkSync(outside, common); + expect((await readStorageInventory({ brainPath: worktree, gitRoots: [repo, alternate] })).errors, "STORAGE_HOST_MARKER_REDIRECT_REFUSED").toContainEqual({ path: "git", reason: "git-metadata-unsupported" }); + unlinkSync(common); writeFileSync(common, "../../redirect\n"); symlinkSync(join(repo, ".git"), join(repo, ".git", "redirect")); + expect((await readStorageInventory({ brainPath: worktree, gitRoots: [repo, alternate] })).status, "STORAGE_HOST_DIRECTORY_REDIRECT_REFUSED").toBe("partial"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index e97a5d6dec..11b9859a4c 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -810,7 +810,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise; salt: string; @@ -50,8 +49,7 @@ export async function collectStorage(options: StorageCollectOptions, id: string, const rawEntries: MetadataEntry[] = []; const coreRootMappings = new Map(); try { - const raw = coreInventory.parse(await storageProcess([...storageCli(options.brainPath), "storage", "inventory", "--json"], { signal, exec: options.exec, - env: { BRAIN_ROOT: options.brainPath, BRAIN_STORAGE_GIT_ROOTS: JSON.stringify(options.gitRoots ?? []) } })); + const raw = coreInventory.parse(await coreMetadata(options.brainPath, options.gitRoots ?? [], signal)); if (raw.status !== "complete") fail(brain.id, "core-inventory-partial"); for (const value of raw.errors) fail(brain.id, value.reason); omittedErrors += raw.omittedErrors; @@ -76,14 +74,14 @@ export async function collectStorage(options: StorageCollectOptions, id: string, } if (view.id === brain.id) { for (const [prefix, label, category, owner] of [ - [SHARE_STAGING_DIR, "Upload staging", "staging", "app"], [".brain/scratch", "Scratch", "scratch", "brain"], ["workspaces", "Workspaces", "workspace", "brain"], + [".git", "Git metadata", "git", "brain"], [SHARE_STAGING_DIR, "Upload staging", "staging", "app"], [".brain/scratch", "Scratch", "scratch", "brain"], ["workspaces", "Workspaces", "workspace", "brain"], ] as const) if (path === prefix || path.startsWith(`${prefix}/`)) { view = root(prefix, label, category, owner); sources.set(view.id, { root: view, path: resolve(options.brainPath, prefix), recursive: false, optional: true }); path = path.slice(prefix.length).replace(/^\//, ""); break; } if (/^brain\.db(?:-wal|-shm|-journal)?$/.test(path)) { view = root("index", "Content index", "index", "brain"); sources.set(view.id, { root: view, path: options.brainPath, recursive: false }); } } } rawEntries.push({ rootId: view.id, path, kind: item.kind, object: item.objectId, version: item.version, device: item.objectId.split(":")[0], logical: item.logicalBytes, allocated: item.allocatedBytes, links: item.links }); } - } catch { fail(brain.id, signal.aborted ? "cancelled" : "storage-cli-unavailable-or-incompatible"); } + } catch { fail(brain.id, signal.aborted ? "cancelled" : "storage-reader-unavailable-or-incompatible"); } const appSources: MetadataRoot[] = []; for (const store of options.stores) { if (roots.length >= STORAGE_SERVER_LIMITS.roots - (options.unavailable?.length ?? 0)) { fail("", "root-budget"); break; } diff --git a/packages/ui-server/src/storage/process.ts b/packages/ui-server/src/storage/process.ts index cd19ca03ed..8164e7be23 100644 --- a/packages/ui-server/src/storage/process.ts +++ b/packages/ui-server/src/storage/process.ts @@ -1,30 +1,30 @@ import { fileURLToPath } from "node:url"; -import { dirname, resolve, relative, isAbsolute } from "node:path"; +import { dirname, relative, isAbsolute } from "node:path"; import { realpathSync, readFileSync } from "node:fs"; -import { wrapCommand, killWrapped, type ExecWrapperConfig } from "@schlessera/brain-ui-sdk/server"; import type { MetadataRoot, MetadataResult } from "./metadata.js"; const inside = (root: string, path: string) => { const rel = relative(root, path); return !rel || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith("../")); }; -/** Resolve the installed host package's public bin, never a content repository's executable. */ -export function storageCli(brainPath: string): string[] { +/** Verify the installed public reader, never code or executables from content. */ +function coreReader(brainPath: string, subpath: "storage" | "queries"): string { const manifestPath = realpathSync(fileURLToPath(import.meta.resolve("@schlessera/brain/package.json"))); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); - if (manifest.name !== "@schlessera/brain" || typeof manifest.bin?.brain !== "string") throw Error("storage-cli-unavailable"); - const bin = realpathSync(resolve(dirname(manifestPath), manifest.bin.brain)), brain = realpathSync(brainPath); - if (inside(brain, manifestPath) || inside(brain, bin) || !inside(dirname(manifestPath), bin)) throw Error("storage-cli-untrusted"); - return [process.execPath, "--config=/dev/null", "--no-env-file", bin]; + if (manifest.name !== "@schlessera/brain") throw Error("storage-reader-unavailable"); + const entry = realpathSync(fileURLToPath(import.meta.resolve(`@schlessera/brain/${subpath}`))), brain = realpathSync(brainPath); + if (inside(brain, manifestPath) || inside(brain, entry) || !inside(dirname(manifestPath), entry)) throw Error("storage-reader-untrusted"); + return entry; +} +export async function coreMetadata(brainPath: string, gitRoots: string[], signal: AbortSignal): Promise { + const source = `import {readStorageInventory} from ${JSON.stringify(coreReader(brainPath, "storage"))}; +const controller=new AbortController();process.once("SIGTERM",()=>controller.abort()); +console.log(JSON.stringify(await readStorageInventory({...${JSON.stringify({ brainPath, gitRoots })},signal:controller.signal})));`; + return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal }); } /** No shell, inherited secrets, repository config or unbounded stdout. Waits for cancellation cleanup. */ -export async function storageProcess(argv: string[], options: { signal: AbortSignal; env?: Record; exec?: ExecWrapperConfig; bytes?: number }): Promise { +export async function storageProcess(argv: string[], options: { signal: AbortSignal; bytes?: number }): Promise { options.signal.throwIfAborted(); - const exec = options.exec ?? {}; - const child = Bun.spawn(wrapCommand(argv, exec.wrapper), { cwd: "/", env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8", ...options.env }, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); + const child = Bun.spawn(argv, { cwd: "/", env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8" }, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); let reason: string | null = null, force: ReturnType | undefined; - const helperAttempts: Promise[] = []; const signal = (value: NodeJS.Signals) => { - if (exec.wrapper) helperAttempts.push(new Promise(done => { - killWrapped(child, exec, value, () => { reason = "cancellation-unconfirmed"; }, { helperTimeoutMs: 1000, onComplete: done }); - })); - else { try { process.kill(-child.pid, value); } catch { child.kill(value); } } + try { process.kill(-child.pid, value); } catch { child.kill(value); } }; const stop = (why: string) => { if (reason) return; reason = why; signal("SIGTERM"); force = setTimeout(() => signal("SIGKILL"), 1500); }; const abort = () => stop("cancelled"); @@ -45,9 +45,6 @@ export async function storageProcess(argv: string[], options: { signal: AbortSig clearTimeout(timer); if (force) clearTimeout(force); options.signal.removeEventListener("abort", abort); signal("SIGKILL"); - // The shared helper bounds its own cleanup; do not release our scan while - // an asynchronous cancellation attempt is still running. - await Promise.allSettled(helperAttempts); await child.exited; } } @@ -60,7 +57,7 @@ console.log(JSON.stringify(await collectMetadata(${JSON.stringify(roots)},{signa /** Isolated supported core query, with the same trusted-installation boundary as inventory. */ export async function storageReferences(brainPath: string, path: string, signal: AbortSignal): Promise { - storageCli(brainPath); // Refuse any host installation redirected into content before loading code. + coreReader(brainPath, "queries"); // Refuse installation redirects into content. const source = `import {createCoreQueryAccess} from ${JSON.stringify(import.meta.resolve("../core-queries.js"))}; const feature=createCoreQueryAccess().storage?.(); console.log(JSON.stringify(feature?.ok?feature.queries.readStorageReferences(${JSON.stringify({ brainPath, path })}):{ok:false}));`; diff --git a/packages/ui-server/tests/storage-collection.test.ts b/packages/ui-server/tests/storage-collection.test.ts index 900210e357..7574a6cf3b 100644 --- a/packages/ui-server/tests/storage-collection.test.ts +++ b/packages/ui-server/tests/storage-collection.test.ts @@ -13,7 +13,7 @@ function fixture() { linkSync(join(brain, "Troy.PDF"), join(app, "chart.pdf")); return { root, brain, app, close() { rmSync(root, { recursive: true, force: true }); } }; } -test("real trusted CLI and exact app stores share one object union and capacity observation", async () => { +test("real trusted metadata reader and exact app stores share one object union and capacity observation", async () => { const f = fixture(), dbPath = join(f.app, "state.sqlite"), db = new Database(dbPath); try { db.exec("PRAGMA journal_mode=WAL; CREATE TABLE chart (id INTEGER); INSERT INTO chart VALUES (1)"); @@ -22,7 +22,7 @@ test("real trusted CLI and exact app stores share one object union and capacity { path: dbPath, label: "DB", category: "database", owner: "app", recursive: false }, { path: dbPath + "-wal", label: "WAL", category: "database", owner: "app", recursive: false }, ] }, "scan-odysseus", new AbortController().signal); - expect(result.summary.errors, "STORAGE_REAL_CLI_AVAILABLE").toEqual([]); + expect(result.summary.errors, "STORAGE_PUBLIC_READER_AVAILABLE").toEqual([]); expect(result.entries.some(entry => entry.name === "Troy.PDF")).toBe(true); const troy = result.entries.find(entry => entry.name === "Troy.PDF")!, alias = result.entries.find(entry => entry.name === "chart.pdf")!; expect(troy.objectId, "STORAGE_CROSS_OWNER_OBJECT_IDENTITY").toBe(alias.objectId); @@ -142,18 +142,14 @@ test("a partial refresh preserves complete evidence, signed cursors expire, and } finally { Object.defineProperty(Date, "now", descriptor); await storage.close(); f.close(); } }); -test("wrapped cancellation waits for bounded killer helper cleanup", async () => { - const { storageProcess } = await import("../src/storage/process"); - const { chmodSync, existsSync, readFileSync } = await import("node:fs"); - const f = fixture(), wrapper = join(f.app, "wrapper"), killer = join(f.app, "killer"), childPid = join(f.app, "child-pid"), helperPids = join(f.app, "helper-pids"); - writeFileSync(wrapper, '#!/bin/sh\nexec "$@"\n'); chmodSync(wrapper, 0o700); - writeFileSync(killer, `#!${process.execPath}\nimport{appendFileSync}from"node:fs";appendFileSync(${JSON.stringify(helperPids)},String(process.pid)+"\\n");setInterval(()=>{},1000);\n`); chmodSync(killer, 0o700); - const controller = new AbortController(), pending = storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", `await Bun.write(${JSON.stringify(childPid)},String(process.pid));setInterval(()=>{},1000);`], { signal: controller.signal, exec: { wrapper, killer } }); +test("in-tree Git metadata has its own ownership row without running repository code", async () => { + const f = fixture(); mkdirSync(join(f.brain, ".git")); mkdirSync(join(f.brain, ".git", "objects")); + writeFileSync(join(f.brain, ".git", "config"), "[include]\npath = /unapproved-host-config\n"); try { - for (let i = 0; i < 200 && !existsSync(childPid); i++) await new Promise(done => setTimeout(done, 5)); - expect(existsSync(childPid)).toBe(true); controller.abort(); await expect(pending).rejects.toThrow(); - await new Promise(done => setTimeout(done, 50)); - expect(existsSync(helperPids)).toBe(true); - for (const pid of readFileSync(helperPids, "utf8").trim().split("\n").map(Number)) expect(() => process.kill(pid, 0), "STORAGE_KILL_HELPERS_REAPED").toThrow(); - } finally { controller.abort(); await pending.catch(() => {}); await new Promise(done => setTimeout(done, 1200)); f.close(); } + const result = await collectStorage({ brainPath: f.brain, salt: "odysseus", stores: [] }, "git-metadata", new AbortController().signal); + const git = result.summary.roots.find(root => root.category === "git"); + expect(git, "STORAGE_GIT_OWNER_CATEGORY").toBeDefined(); + expect(result.entries.find(entry => entry.name === "config")?.rootId).toBe(git!.id); + expect(result.summary.errors).toEqual([]); + } finally { f.close(); } }); diff --git a/scripts/ci-pack.ts b/scripts/ci-pack.ts index 88d611c4f7..af7dafd559 100644 --- a/scripts/ci-pack.ts +++ b/scripts/ci-pack.ts @@ -87,8 +87,8 @@ export const PACK_TABLE: Record = { }, core: { // Codex reads a skill's manual-only policy from beside its SKILL.md. - files: ["dist/index.js", "dist/cli/brain.js", "dist/hooks/", "skills/sync/agents/openai.yaml"], - bun: ["@schlessera/brain"], + files: ["dist/index.js", "dist/cli/brain.js", "dist/storage.js", "dist/hooks/", "skills/sync/agents/openai.yaml"], + bun: ["@schlessera/brain", "@schlessera/brain/storage"], bunProbes: ["core-testing-suites"], node: "bun-only", nodeProbes: ["core-testing-suites"], From ca6fe98aad4e095ba1b296eb9d5fcf1a2a82b1c7 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 12:23:52 +0200 Subject: [PATCH 05/44] fix(storage): exclude directory metadata from logical file totals --- packages/ui-server/src/storage/collect.ts | 2 +- .../ui-server/tests/storage-collection.test.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/ui-server/src/storage/collect.ts b/packages/ui-server/src/storage/collect.ts index 5d86925235..ffb48727cb 100644 --- a/packages/ui-server/src/storage/collect.ts +++ b/packages/ui-server/src/storage/collect.ts @@ -27,7 +27,7 @@ export function storageTotals(entries: StorageFileEntry[], complete = true): Sto for (const entry of entries) { if (seen.has(entry.objectId)) continue; seen.add(entry.objectId); - if (entry.logical === null) hasLogical = false; else logical += BigInt(entry.logical); + if (entry.kind === "file") { if (entry.logical === null) hasLogical = false; else logical += BigInt(entry.logical); } if (entry.allocated === null) hasAllocation = false; else allocated += BigInt(entry.allocated); } const safe = (value: bigint) => value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null; diff --git a/packages/ui-server/tests/storage-collection.test.ts b/packages/ui-server/tests/storage-collection.test.ts index 7574a6cf3b..935c92797b 100644 --- a/packages/ui-server/tests/storage-collection.test.ts +++ b/packages/ui-server/tests/storage-collection.test.ts @@ -153,3 +153,17 @@ test("in-tree Git metadata has its own ownership row without running repository expect(result.summary.errors).toEqual([]); } finally { f.close(); } }); + +test("directory metadata does not erase known regular-file logical totals", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-logical-")); + mkdirSync(join(root, ".brain", "scratch"), { recursive: true }); + writeFileSync(join(root, ".brain", "scratch", "chart.txt"), "Ithaca map"); + linkSync(join(root, ".brain", "scratch", "chart.txt"), join(root, ".brain", "scratch", "alias.txt")); + try { + const result = await collectStorage({ brainPath: root, stores: [], salt: "odysseus" }, "logical-union", new AbortController().signal); + expect(result.summary.roots.find(value => value.category === "scratch")?.logical, "STORAGE_FILE_UNION_EXCLUDES_DIRECTORY_METADATA").toBe(10); + expect(result.summary.totals.brain.logical).toBe(10); + expect(result.summary.roots.find(value => value.category === "content")?.logical).toBe(0); + expect(result.summary.totals.all.coverage).toBe("complete"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); From 98b8875a14323e4bc70321feaa3541be7389a13c Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 12:50:13 +0200 Subject: [PATCH 06/44] docs(storage): describe the metadata reader consistently --- docs/http-api.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/http-api.md b/docs/http-api.md index 79511718f9..cac05d9c4c 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1249,12 +1249,12 @@ at most 48 MiB, for at most ten minutes. One scan admits at most 100,000 entries 32 roots and 35 seconds of collection; child cancellation is awaited. Kernel uninterruptible I/O cannot have a hard wall-clock completion guarantee. -The server invokes the installed host core package’s public CLI with an explicit -root, no executable brain configuration or inherited secrets, and the configured -exec wrapper. It never launches the brain repository’s selected executable on -the host. Missing/incompatible inventory support produces partial/unavailable -evidence rather than weakening that boundary. Metadata workers inspect only -exact configured stores, reject redirects below roots and mount crossings, and +The server resolves the installed host core package’s public metadata entry +outside the brain, with explicit roots and no inherited secrets. It never +launches the brain repository’s selected executable on the host. +Missing/incompatible inventory support produces partial/unavailable evidence +rather than falling back to CLI execution or internal imports. Metadata workers +inspect only exact configured stores, reject redirects below roots and mount crossings, and never read attachment/database/backup contents for sizing. Backend setup is not triggered by inventory. Explicit host backup/Git roots are described in server configuration; no remote or host-wide filesystem discovery is performed. From 60e627d4a2cfa7826355900de7c2fbe7ab00f0ac Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 13:14:06 +0200 Subject: [PATCH 07/44] test(storage): carry file kinds through object-union fixtures --- packages/ui-server/src/storage/collect.ts | 2 +- packages/ui-server/tests/storage-metadata.test.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ui-server/src/storage/collect.ts b/packages/ui-server/src/storage/collect.ts index ffb48727cb..b7c15e406f 100644 --- a/packages/ui-server/src/storage/collect.ts +++ b/packages/ui-server/src/storage/collect.ts @@ -21,7 +21,7 @@ export interface StorageCollectOptions { salt: string; } const validPath = (path: string) => path === "" || (!path.startsWith("/") && path.split("/").every(piece => piece !== "." && piece !== ".." && piece !== "") && !path.includes("\0")); -export function storageTotals(entries: StorageFileEntry[], complete = true): StorageBytes { +export function storageTotals(entries: readonly Pick[], complete = true): StorageBytes { if (!entries.length && !complete) return { logical: null, allocated: null, coverage: "unknown" }; const seen = new Set(); let logical = 0n, allocated = 0n, hasLogical = true, hasAllocation = true; for (const entry of entries) { diff --git a/packages/ui-server/tests/storage-metadata.test.ts b/packages/ui-server/tests/storage-metadata.test.ts index deff3d4bdd..69aae17075 100644 --- a/packages/ui-server/tests/storage-metadata.test.ts +++ b/packages/ui-server/tests/storage-metadata.test.ts @@ -4,7 +4,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { collectMetadata } from "../src/storage/metadata"; import { storageTotals } from "../src/storage/collect"; -import type { StorageFileEntry } from "@schlessera/brain-ui-sdk/protocol"; test("server metadata stays within exact roots and does not follow descendants or sidecar redirects", async () => { const root = mkdtempSync(join(tmpdir(), "storage-odysseus-")); @@ -30,7 +29,7 @@ test("server metadata records hard-link identity once in the object union and pr const result = await collectMetadata([{ id: "charts", path: root, recursive: true }]); const files = result.entries.filter(entry => entry.kind === "file"); expect(files).toHaveLength(2); expect(files[0]!.object).toBe(files[1]!.object); - const entries = files.map(entry => ({ objectId: entry.object, logical: entry.logical, allocated: entry.allocated })) as StorageFileEntry[]; + const entries = files.map(entry => ({ kind: entry.kind, objectId: entry.object, logical: entry.logical, allocated: entry.allocated })); expect(storageTotals(entries).logical, "STORAGE_OBJECT_UNION").toBe(5); const partial = await collectMetadata([{ id: "charts", path: root, recursive: true }], { entries: 1 }); expect(partial.entries).toHaveLength(1); expect(partial.errors).toContainEqual({ rootId: "charts", reason: "entry-budget" }); From a0b94a20bca0be522566f25bed0c72f39291909d Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 14:22:37 +0200 Subject: [PATCH 08/44] CONTRACT: feat(storage): project bounded subtree totals and searchable pages --- .changeset/odysseus-storage-server.md | 4 + api-report/ui-sdk.txt | 54 ++++++++++++ docs/http-api-coverage.md | 6 +- docs/http-api.md | 30 ++++++- docs/integration-contract/http.md | 9 +- packages/ui-sdk/src/protocol.ts | 2 +- packages/ui-sdk/src/storage.ts | 17 ++++ packages/ui-server/src/routes/storage.ts | 25 +++++- packages/ui-server/src/storage/coordinator.ts | 24 +++++- packages/ui-server/src/storage/tree.ts | 84 +++++++++++++++++++ .../tests/http-supported-mounts.test.ts | 1 + .../ui-server/tests/storage-routes.test.ts | 2 +- packages/ui-server/tests/storage-tree.test.ts | 79 +++++++++++++++++ 13 files changed, 322 insertions(+), 15 deletions(-) create mode 100644 packages/ui-server/src/storage/tree.ts create mode 100644 packages/ui-server/tests/storage-tree.test.ts diff --git a/.changeset/odysseus-storage-server.md b/.changeset/odysseus-storage-server.md index f72c52a9a7..9c1bf96631 100644 --- a/.changeset/odysseus-storage-server.md +++ b/.changeset/odysseus-storage-server.md @@ -7,3 +7,7 @@ --- Add the supported metadata-only `@schlessera/brain/storage` host reader, which never executes Git or repository configuration, and human-only bounded Storage inventory and reference inspection, with explicit store ownership, deduplicated object and capacity observations, partial coverage, configured backup roots, and backend-owned transcript metadata. This foundation does not register a Storage pane or execute cleanup. + +Provide bounded cached subtree projections and server-side search/sorting for +Storage charts and equivalent lists, with separate object-union area and subtotal +evidence so aliases do not inflate a chart. diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index ba08950b33..2e703e5be2 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -316,6 +316,9 @@ export "." (./src/index.ts) type StorageRoot type StorageSnapshot type StorageStatus + type StorageTreeFilter + type StorageTreeNode + type StorageTreePage type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -774,6 +777,9 @@ export "./client" (./src/client/index.ts) type StorageRoot type StorageSnapshot type StorageStatus + type StorageTreeFilter + type StorageTreeNode + type StorageTreePage type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -1387,6 +1393,9 @@ export "./protocol" (./src/protocol.ts) type StorageRoot type StorageSnapshot type StorageStatus + type StorageTreeFilter + type StorageTreeNode + type StorageTreePage type SubscriptionAuthAction type SystemStatus THINKING_LEVELS @@ -2071,6 +2080,9 @@ export "./server" (./src/server/index.ts) type StorageRoot type StorageSnapshot type StorageStatus + type StorageTreeFilter + type StorageTreeNode + type StorageTreePage type SubprocessEnvAudience type SubscriptionAuthAction type SurfaceTools @@ -9267,6 +9279,10 @@ signatures (every public export and the types it is made of) } StorageFilter (packages/ui-sdk/src/storage.ts) export interface StorageFilter { + search?: string; + sort?: "name" | "size"; + metric?: "logical" | "allocated"; + direction?: "asc" | "desc"; rootId?: string; parentId?: string; owner?: StorageOwner | "all"; @@ -9347,6 +9363,44 @@ signatures (every public export and the types it is made of) cacheBytes: number; }; } + StorageTreeFilter (packages/ui-sdk/src/storage.ts) + export interface StorageTreeFilter { + parentId?: string; + owner?: StorageOwner | "all"; + search?: string; + sort?: "name" | "size"; + metric?: "logical" | "allocated"; + direction?: "asc" | "desc"; + cursor?: string; + } + StorageTreeNode (packages/ui-sdk/src/storage.ts) + export interface StorageTreeNode { + id: string; + rootId: string; + entry: StorageFileEntry | null; + path: string; + name: string; + kind: "root" | "metadata" | StorageFileEntry["kind"]; + owner: StorageOwner; + totals: StorageBytes; + area: StorageBytes; + reason: string | null; + } + StorageTreePage (packages/ui-sdk/src/storage.ts) + export interface StorageTreePage { + scanId: string; + parentId: string | null; + name: string; + breadcrumbs: { + id: string | null; + name: string; + }[]; + nodes: StorageTreeNode[]; + totals: StorageBytes; + matched: number; + nextCursor: string | null; + stale: boolean; + } StoredShare (packages/ui-sdk/src/client/share-store.ts) export interface StoredShare { id: string; diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index 110e073562..f5d007b8a8 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:117-150`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:118-151`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -32,6 +32,7 @@ row's real mounting check. They do not establish mounting by themselves. | `POST /api/storage/scans` | `mounted supported handler: POST /api/storage/scans` | ST | | `DELETE /api/storage/scans/:id` | `mounted supported handler: DELETE /api/storage/scans/:id` | ST | | `GET /api/storage/scans/:id/entries` | `mounted supported handler: GET /api/storage/scans/:id/entries` | ST | +| `GET /api/storage/scans/:id/tree` | `mounted supported handler: GET /api/storage/scans/:id/tree` | ST | | `GET /api/storage/scans/:id/entries/:entryId/references` | `mounted supported handler: GET /api/storage/scans/:id/entries/:entryId/references` | ST | | `GET /api/activity/rollups` | `mounted supported handler: GET /api/activity/rollups` | R | | `GET /api/activity/runs` | `mounted supported handler: GET /api/activity/runs` | R | @@ -461,7 +462,8 @@ files. `revert-gate.test.ts` proves old-turn maintenance/new-turn queue order; The real Chrome `turn-journal-runtime.test.ts` exercises whole-turn review, reading focus/Escape, waiting and partial receipts, Undo and lost-response readback. -ST: [Storage route authorization and mounted scans](../packages/ui-server/tests/storage-routes.test.ts), +ST: [Cached hierarchy, union areas and query paging](../packages/ui-server/tests/storage-tree.test.ts), +[Storage route authorization and mounted scans](../packages/ui-server/tests/storage-routes.test.ts), [real public-reader/store union and paging](../packages/ui-server/tests/storage-collection.test.ts), [exact metadata roots](../packages/ui-server/tests/storage-metadata.test.ts), and [bounded index evidence](../packages/core/tests/storage-references.test.ts). diff --git a/docs/http-api.md b/docs/http-api.md index cac05d9c4c..92e530ea0b 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 140 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 141 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -30,6 +30,7 @@ client code has a gap. Source owners are listed after the table. | POST | `/api/storage/scans` | S | Refresh configured stores | Human-owner Storage inspection; no delegated or worker access. | | DELETE | `/api/storage/scans/:id` | S | Cancel an active scan | Human-owner Storage inspection; no delegated or worker access. | | GET | `/api/storage/scans/:id/entries` | S | Read a filtered bounded page | Human-owner Storage inspection; no delegated or worker access. | +| GET | `/api/storage/scans/:id/tree` | S | Read a bounded subtree projection | Human-owner Storage inspection; no delegated or worker access. | | GET | `/api/storage/scans/:id/entries/:entryId/references` | S | Inspect bounded indexed reference evidence | Human-owner Storage inspection; no delegated or worker access. | | GET | `/api/activity/digest` | I | Read digest and record app visit | React Activity presentation state; retained public client functions do not promise independent raw transport. | | POST | `/api/activity/digest/dismiss` | I | Record digest dismissal | React Activity presentation state; retained public client functions do not promise independent raw transport. | @@ -173,7 +174,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 140 declared endpoints are mounted regardless of backend, renderer or +All 141 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1223,7 +1224,7 @@ never executes Git, the brain CLI, repository configuration, hooks or filters for inspection. Only fixed Git administrative location markers and approved host roots participate in discovery. -These five operations require a current `owner` or `ambient` principal. Agent, +These six operations require a current `owner` or `ambient` principal. Agent, delegated and restricted access is refused (403); unauthenticated/revoked access is refused (401 by normal authentication). Authorization is checked again at coordinator access and after asynchronous inspection. Responses are private, @@ -1239,7 +1240,12 @@ an inactive identity returns 409. Initiator revocation cancels its scan. `GET /api/storage/scans/:id/entries` accepts optional `rootId`, `parentId` (empty means root rows), `owner=all|brain|app|operator`, `type=pdf`, nonnegative -integer `largerThan` and opaque `cursor`. PDF matching is filename based and +integer `largerThan`, `search`, `sort=name|size`, `metric=allocated|logical`, +`direction=asc|desc` and opaque `cursor`. Search matches case-insensitive entry +names and admits at most 128 UTF-8 bytes with no control characters. Ordering +applies to all matches before paging; unknown byte values sort last. Explicit +size ordering defaults descending, name ordering ascending; absent presentation +parameters retain the original root/path order. PDF matching is filename based and case insensitive; byte thresholds are strictly greater than. A page has at most 200 entries, a next cursor, stale flag, match count and object-union subtotals. Cursors bind the scan and filters and expire after 60 seconds; changed/expired @@ -1249,6 +1255,22 @@ at most 48 MiB, for at most ten minutes. One scan admits at most 100,000 entries 32 roots and 35 seconds of collection; child cancellation is awaited. Kernel uninterruptible I/O cannot have a hard wall-clock completion guarantee. +`GET /api/storage/scans/:id/tree` accepts `parentId` (a root or directory ID; +absent/empty selects all roots), `owner`, `search`, `sort`, `metric`, `direction` +and `cursor`. It projects the retained scan without filesystem reads, returning +SDK `StorageTreePage`: parent, breadcrumb names/IDs, at most 200 immediate child +nodes, full match count and subtotals. Search matches immediate child names; +default ordering is descending allocated size. A node's `totals` contains its +subtree object union; `area` assigns each object to one matching sibling in stable +root/path order. Alias names remain visible without inflating the displayed area. +The page's `totals` union covers all matching nodes, including later pages. +An explicit, non-actionable `metadata` node accounts for the current directory's +own allocation; file roots retain file identity. Partial/unknown owners remain +partial/unknown even with no matching rows. Tree cursors also bind the operation +and cannot be reused for flat entries. Projection has a three-second checked +CPU deadline over the bounded cached scan; incomplete topology, an unavailable +parent or an out-of-view parent returns 503 rather than omitting descendant bytes. + The server resolves the installed host core package’s public metadata entry outside the brain, with explicit roots and no inherited secrets. It never launches the brain repository’s selected executable on the host. diff --git a/docs/integration-contract/http.md b/docs/integration-contract/http.md index 0ac153a134..93874f9ee0 100644 --- a/docs/integration-contract/http.md +++ b/docs/integration-contract/http.md @@ -484,9 +484,9 @@ and cross-session restrictions remain binding. ### Human-only Storage inventory (additive, #2010) -The five [Storage inventory operations](../http-api.md#storage-inventory) and +The six [Storage inventory operations](../http-api.md#storage-inventory) and SDK `StorageStatus`, `StorageSnapshot`, `StoragePage`, `StorageFileEntry`, -`StorageReferenceEvidence` and related types are additive contracts. They bind +`StorageReferenceEvidence`, `StorageTreePage`, `StorageTreeNode` and related types are additive contracts. They bind authorization, configured-root scope, bounded output/lifetime, opaque cursors, object/capacity deduplication and explicit partial/unknown semantics. This transport foundation does not register a pane or authorize cleanup. @@ -496,3 +496,8 @@ entry, including aliases) from `totals` (the deduplicated physical-object union) Empty unavailable ownership views carry null bytes, not zero. Capacity `kind` identifies `tmpfs` as memory-backed; `filesystem` does not promise exclusive disk ownership and `unknown` cannot be drawn as healthy capacity. + +Hierarchy pages separate per-subtree object-union `totals` from disjoint sibling +`area`, preserve alias names and unknown coverage, and bind cursors to the tree +operation and its search/order/ownership criteria. Projection uses retained scan +evidence only. Entry search/order is additive; absent options preserve ordering. diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index dcb3a62a12..fd19abfaf5 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3646,4 +3646,4 @@ export interface ServerScopeDecisionReceipt { } export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-journal.js"; -export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence } from "./storage.js"; +export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; diff --git a/packages/ui-sdk/src/storage.ts b/packages/ui-sdk/src/storage.ts index 45fcddef96..1a950ede1b 100644 --- a/packages/ui-sdk/src/storage.ts +++ b/packages/ui-sdk/src/storage.ts @@ -40,6 +40,7 @@ export interface StoragePage { matched: number; totals: StorageBytes; } export interface StorageFilter { + search?: string; sort?: "name" | "size"; metric?: "logical" | "allocated"; direction?: "asc" | "desc"; rootId?: string; parentId?: string; owner?: StorageOwner | "all"; type?: "pdf"; largerThan?: number; cursor?: string; } @@ -52,3 +53,19 @@ export interface StorageReferenceEvidence { /** Index freshness is not verified against current source bytes. */ freshness: "unchecked"; unchecked: string[]; } + +/** A cached hierarchy projection; aliases retain names while contributing area once. */ +export interface StorageTreeNode { + id: string; rootId: string; entry: StorageFileEntry | null; path: string; name: string; + kind: "root" | "metadata" | StorageFileEntry["kind"]; owner: StorageOwner; + totals: StorageBytes; area: StorageBytes; reason: string | null; +} +export interface StorageTreeFilter { + parentId?: string; owner?: StorageOwner | "all"; search?: string; + sort?: "name" | "size"; metric?: "logical" | "allocated"; direction?: "asc" | "desc"; cursor?: string; +} +export interface StorageTreePage { + scanId: string; parentId: string | null; name: string; + breadcrumbs: { id: string | null; name: string }[]; + nodes: StorageTreeNode[]; totals: StorageBytes; matched: number; nextCursor: string | null; stale: boolean; +} diff --git a/packages/ui-server/src/routes/storage.ts b/packages/ui-server/src/routes/storage.ts index 1312e02162..85d22fc980 100644 --- a/packages/ui-server/src/routes/storage.ts +++ b/packages/ui-server/src/routes/storage.ts @@ -1,7 +1,7 @@ import { Hono } from "hono"; import { requireJson } from "../middleware/origin.js"; import type { AppEnv } from "../app-env.js"; -import type { StorageFilter } from "@schlessera/brain-ui-sdk/protocol"; +import type { StorageFilter, StorageTreeFilter } from "@schlessera/brain-ui-sdk/protocol"; import { StorageUnavailable, type StorageCoordinator } from "../storage/coordinator.js"; /** Ordinary content authorization does not grant access to deployment inventory. */ export function createStorageRoutes(storage: StorageCoordinator): Hono { @@ -23,14 +23,31 @@ export function createStorageRoutes(storage: StorageCoordinator): Hono { }); routes.delete("/storage/scans/:id", c => c.json(storage.cancel(c.get("principal")!.id, c.req.param("id")), 202)); routes.get("/storage/scans/:id/entries", c => { - const raw = c.req.query(), keys = ["rootId", "parentId", "owner", "type", "largerThan", "cursor"]; - if (Object.keys(raw).some(key => !keys.includes(key)) || raw.owner !== undefined && !["all", "brain", "app", "operator"].includes(raw.owner) + const raw = c.req.query(), keys = ["rootId", "parentId", "owner", "type", "largerThan", "cursor", "search", "sort", "metric", "direction"]; + if (Object.keys(raw).some(key => !keys.includes(key)) || !validPresentation(raw) || raw.owner !== undefined && !["all", "brain", "app", "operator"].includes(raw.owner) || raw.type !== undefined && raw.type !== "pdf" || [raw.rootId, raw.parentId].some(id => id !== undefined && id !== "" && !/^[a-f0-9]{32}$/.test(id)) || raw.largerThan !== undefined && (!/^\d+$/.test(raw.largerThan) || !Number.isSafeInteger(Number(raw.largerThan)))) return c.json({ error: "Invalid Storage filters." }, 400); const filter: StorageFilter = { ...(raw.rootId !== undefined ? { rootId: raw.rootId } : {}), ...(raw.parentId !== undefined ? { parentId: raw.parentId } : {}), - ...(raw.owner ? { owner: raw.owner as StorageFilter["owner"] } : {}), ...(raw.type ? { type: "pdf" } : {}), ...(raw.largerThan !== undefined ? { largerThan: Number(raw.largerThan) } : {}), ...(raw.cursor ? { cursor: raw.cursor } : {}) }; + ...(raw.owner ? { owner: raw.owner as StorageFilter["owner"] } : {}), ...(raw.type ? { type: "pdf" } : {}), ...(raw.largerThan !== undefined ? { largerThan: Number(raw.largerThan) } : {}), ...(raw.cursor ? { cursor: raw.cursor } : {}), ...presentation(raw) }; return c.json(storage.page(c.get("principal")!.id, c.req.param("id"), filter)); }); + routes.get("/storage/scans/:id/tree", c => { + const raw = c.req.query(), keys = ["parentId", "owner", "search", "sort", "metric", "direction", "cursor"]; + if (Object.keys(raw).some(key => !keys.includes(key)) || !validPresentation(raw) || raw.owner !== undefined && !["all", "brain", "app", "operator"].includes(raw.owner) + || raw.parentId !== undefined && raw.parentId !== "" && !/^[a-f0-9]{32}$/.test(raw.parentId)) return c.json({ error: "Invalid Storage filters." }, 400); + const filter: StorageTreeFilter = { ...(raw.parentId ? { parentId: raw.parentId } : {}), ...(raw.owner ? { owner: raw.owner as StorageTreeFilter["owner"] } : {}), ...(raw.cursor ? { cursor: raw.cursor } : {}), ...presentation(raw) }; + return c.json(storage.tree(c.get("principal")!.id, c.req.param("id"), filter)); + }); routes.get("/storage/scans/:id/entries/:entryId/references", async c => c.json(await storage.references(c.get("principal")!.id, c.req.param("id"), c.req.param("entryId")))); return routes; } + +function validPresentation(raw: Record) { + return (raw.search === undefined || Buffer.byteLength(raw.search) <= 128 && !/[\u0000-\u001f\u007f]/u.test(raw.search)) + && (raw.sort === undefined || ["name", "size"].includes(raw.sort)) + && (raw.metric === undefined || ["logical", "allocated"].includes(raw.metric)) + && (raw.direction === undefined || ["asc", "desc"].includes(raw.direction)); +} +function presentation(raw: Record): Pick { + return { ...(raw.search !== undefined ? {search:raw.search} : {}), ...(raw.sort ? {sort:raw.sort as "name"|"size"} : {}), ...(raw.metric ? {metric:raw.metric as "logical"|"allocated"} : {}), ...(raw.direction ? {direction:raw.direction as "asc"|"desc"} : {}) }; +} diff --git a/packages/ui-server/src/storage/coordinator.ts b/packages/ui-server/src/storage/coordinator.ts index 7b17e5523d..6ce296e6df 100644 --- a/packages/ui-server/src/storage/coordinator.ts +++ b/packages/ui-server/src/storage/coordinator.ts @@ -1,6 +1,7 @@ +import { projectStorageTree, sortStorageRows } from "./tree.js"; import { storageReferences } from "./process.js"; import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; -import type { StorageFilter, StoragePage, StorageStatus, StorageReferenceEvidence } from "@schlessera/brain-ui-sdk/protocol"; +import type { StorageFilter, StoragePage, StorageStatus, StorageReferenceEvidence, StorageTreeFilter, StorageTreePage } from "@schlessera/brain-ui-sdk/protocol"; import { collectStorage, storageTotals, STORAGE_SERVER_LIMITS, type StorageCollection, type StorageCollectOptions } from "./collect.js"; export class StorageUnavailable extends Error { constructor(readonly status: 400 | 403 | 404 | 409 | 429 | 503, message: string) { super(message); } @@ -82,7 +83,9 @@ export function createStorageCoordinator(options: Omit criteria.largerThan)); + if (criteria.sort || criteria.direction || criteria.metric) sortStorageRows(matches, criteria, entry => entry[criteria.metric ?? "allocated"]); const page = matches.slice(offset, offset + STORAGE_SERVER_LIMITS.page), next = offset + page.length; let nextCursor: string | null = null; if (next < matches.length) { const encoded = Buffer.from(JSON.stringify({ id, filter: filterKey, offset: next, expires: Date.now() + 60_000 })).toString("base64url"); nextCursor = `${encoded}.${digest(encoded)}`; } @@ -91,6 +94,25 @@ export function createStorageCoordinator(options: Omit entry.logical === null) || logicalSum > BigInt(Number.MAX_SAFE_INTEGER) ? null : Number(logicalSum); return { scanId: id, fileLogicalTotal, entries: page, nextCursor, stale: Date.now() >= value.summary.staleAfter, matched: matches.length, totals: storageTotals(matches, value.summary.coverage === "complete") }; }, + tree(principal: string, id: string, filter: StorageTreeFilter): StorageTreePage { + const value = collection(principal, id), { cursor, ...criteria } = filter, filterKey = "tree:" + JSON.stringify(criteria); + let offset = 0; + if (cursor) { + if (cursor.length > 2048) throw new StorageUnavailable(400, "Invalid Storage cursor."); + const [encoded, signature, extra] = cursor.split("."), expected = Buffer.from(digest(encoded ?? "")), actual = Buffer.from(signature ?? ""); + if (extra || expected.length !== actual.length || !timingSafeEqual(expected, actual)) throw new StorageUnavailable(400, "Invalid Storage cursor."); + let token: { id: string; filter: string; offset: number; expires: number }; + try { token = JSON.parse(Buffer.from(encoded!, "base64url").toString()); } catch { throw new StorageUnavailable(400, "Invalid Storage cursor."); } + if (token.id !== id || token.filter !== filterKey || token.expires < Date.now() || !Number.isSafeInteger(token.offset) || token.offset < 0) throw new StorageUnavailable(409, "Storage cursor expired or filters changed."); + offset = token.offset; + } + let projected: ReturnType; + try { projected = projectStorageTree(value, criteria); } catch { throw new StorageUnavailable(503, "Storage hierarchy unavailable; refresh this observation."); } + const nodes = projected.nodes.slice(offset, offset + STORAGE_SERVER_LIMITS.page), next = offset + nodes.length; + let nextCursor: string | null = null; + if (next < projected.nodes.length) { const encoded = Buffer.from(JSON.stringify({ id, filter: filterKey, offset: next, expires: Date.now() + 60_000 })).toString("base64url"); nextCursor = `${encoded}.${digest(encoded)}`; } + return { ...projected, nodes, scanId: id, nextCursor, matched: projected.nodes.length, stale: Date.now() >= value.summary.staleAfter }; + }, async references(principal: string, id: string, entryId: string): Promise { const value = collection(principal, id), entry = value.entries.find(item => item.id === entryId); if (!entry || entry.kind !== "file") throw new StorageUnavailable(404, "File observation unavailable."); diff --git a/packages/ui-server/src/storage/tree.ts b/packages/ui-server/src/storage/tree.ts new file mode 100644 index 0000000000..045fe10d84 --- /dev/null +++ b/packages/ui-server/src/storage/tree.ts @@ -0,0 +1,84 @@ +/** In-memory hierarchy projection over the bounded scan; no additional filesystem traversal. */ +import type { StorageBytes, StorageFileEntry, StorageTreeFilter, StorageTreeNode } from "@schlessera/brain-ui-sdk/protocol"; +import { storageTotals, type StorageCollection } from "./collect.js"; +const compare = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0; +export function sortStorageRows(rows: T[], filter: {sort?: "name" | "size"; metric?: "logical" | "allocated"; direction?: "asc" | "desc"}, bytes: (row: T) => number | null): T[] { + const direction = filter.direction === "asc" ? 1 : filter.direction === "desc" ? -1 : filter.sort === "name" ? 1 : -1; + return rows.sort((a,b) => { + if (filter.sort !== "name") { + const left = bytes(a), right = bytes(b); + if (left === null && right !== null) return 1; if (right === null && left !== null) return -1; + if (left !== null && right !== null && left !== right) return direction * (left - right); + } + return direction * (filter.sort === "name" ? compare(a.name.toLowerCase(),b.name.toLowerCase()) : 0) || compare(a.name,b.name) || compare(a.id,b.id); + }); +} +export function projectStorageTree(collection: StorageCollection, filter: StorageTreeFilter) { + const started = Date.now(), roots = new Map(collection.summary.roots.map(root => [root.id, root])), byId = new Map(collection.entries.map(entry => [entry.id,entry])); + const parentRoot = filter.parentId ? roots.get(filter.parentId) : undefined, parentEntry = filter.parentId ? byId.get(filter.parentId) : undefined; + if (filter.parentId && !parentRoot && (!parentEntry || parentEntry.kind !== "directory")) throw Error("Storage folder observation unavailable."); + const selectedRoot = parentRoot ?? (parentEntry ? roots.get(parentEntry.rootId) : undefined); + if (selectedRoot && filter.owner && filter.owner !== "all" && selectedRoot.owner !== filter.owner) throw Error("Storage folder is outside this ownership view."); + const deadline = () => { if (Date.now() - started > 3000) throw Error("Storage projection time bound reached."); }; + const nodes = new Map(), childPaths = new Map(); + const allowed = (rootId: string) => !!roots.get(rootId) && (!filter.owner || filter.owner === "all" || roots.get(rootId)!.owner === filter.owner); + if (!filter.parentId) for (const root of roots.values()) { + if (!allowed(root.id)) continue; + nodes.set(root.id,{node:{id:root.id,rootId:root.id,entry:null,path:"",name:root.label,kind:"root",owner:root.owner,totals:root,area:root,reason:root.reason},entries:[]}); + } else { + for (const entry of collection.entries) { + if (entry.rootId !== selectedRoot!.id || !allowed(entry.rootId)) continue; + const own = parentEntry ? entry.id === parentEntry.id : entry.path === ""; + const direct = parentEntry ? entry.parentId === parentEntry.id : !entry.path.includes("/") && entry.path !== ""; + if (!own && !direct) continue; + const id = entry.id, root = roots.get(entry.rootId)!, metadata = own && entry.kind === "directory"; + nodes.set(id,{node:{id,rootId:entry.rootId,entry:metadata ? null : entry,path:entry.path,name:metadata ? "Directory metadata" : entry.name,kind:metadata ? "metadata" : entry.kind,owner:root.owner,totals:root,area:root,reason:root.reason},entries:[]}); + childPaths.set(entry.path,id); + } + } + let count=0; + for (const entry of collection.entries) { + if (++count % 256 === 0) deadline(); + if (!allowed(entry.rootId) || selectedRoot && entry.rootId !== selectedRoot.id) continue; + let child: string | undefined; + if (!filter.parentId) child=entry.rootId; + else { + const prefix=parentEntry?.path ? parentEntry.path+"/" : ""; + if (parentEntry && entry.id===parentEntry.id || parentRoot && entry.path==="") child=entry.id; + else if (entry.path.startsWith(prefix)) { + const name=entry.path.slice(prefix.length).split("/",1)[0]!; + if (name) child=childPaths.get(prefix+name); + } + } + if (child && nodes.has(child)) nodes.get(child)!.entries.push(entry); + else if (!filter.parentId || entry.path.startsWith(parentEntry?.path ? parentEntry.path + "/" : "")) { + // A truncated or inconsistent scan must not silently lose descendant bytes. + throw Error("Storage hierarchy is incomplete."); + } + } + const search=filter.search?.toLowerCase() ?? "", selected=[...nodes.values()].filter(({node})=>!search || node.name.toLowerCase().includes(search)).sort((a,b)=>compare(a.node.rootId,b.node.rootId)||compare(a.node.path,b.node.path)||compare(a.node.id,b.node.id)); + const seen=new Set(), union: StorageFileEntry[]=[]; + for(const group of selected) { + deadline(); const root=roots.get(group.node.rootId)!,complete=root.coverage==="complete", area:StorageFileEntry[]=[]; + group.node.totals=storageTotals(group.entries,complete); + for (const entry of group.entries) { + if (++count % 256 === 0) deadline(); + if (!seen.has(entry.objectId)) { seen.add(entry.objectId); area.push(entry); union.push(entry); } + } + group.node.area=storageTotals(area,complete); + } + const complete=[...roots.values()].filter(root=>allowed(root.id)&&(!selectedRoot||root.id===selectedRoot.id)).every(root=>root.coverage==="complete"); + const totals:StorageBytes=storageTotals(union,complete), breadcrumbs:{id:string|null;name:string}[]=[{id:null,name:"All stores"}]; + if(selectedRoot) { + breadcrumbs.push({id:selectedRoot.id,name:selectedRoot.label}); + if(parentEntry) { + const chain:StorageFileEntry[]=[];let current:StorageFileEntry|undefined=parentEntry;const seenParents=new Set(); + while(current && current.path){if(seenParents.has(current.id)||chain.length>256)throw Error("Storage hierarchy unavailable.");seenParents.add(current.id);chain.push(current);current=current.parentId?byId.get(current.parentId):undefined;} + for(const entry of chain.reverse())breadcrumbs.push({id:entry.id,name:entry.name}); + } + } + const metric=filter.metric ?? "allocated"; + const sorted = sortStorageRows(selected.map(group=>group.node),filter,node=>node.area[metric]); + deadline(); + return {parentId:filter.parentId||null,name:parentEntry?.name??selectedRoot?.label??"All stores",breadcrumbs,totals,nodes:sorted}; +} diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index bd0e19065b..747b28409b 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -20,6 +20,7 @@ const checks: Record = { "POST /api/storage/scans": { init: json("POST", {}), status: 202, body: { version: 1, state: "scanning" } }, "DELETE /api/storage/scans/:id": { path: "/api/storage/scans/missing", init: { method: "DELETE" }, status: 409, body: { error: "That scan is no longer active." } }, "GET /api/storage/scans/:id/entries": { path: "/api/storage/scans/missing/entries", status: 404, body: { error: "Storage observation expired or unavailable; refresh." } }, + "GET /api/storage/scans/:id/tree": { path: "/api/storage/scans/missing/tree", status: 404, body: { error: "Storage observation expired or unavailable; refresh." } }, "GET /api/storage/scans/:id/entries/:entryId/references": { path: "/api/storage/scans/missing/entries/missing/references", status: 404, body: { error: "Storage observation expired or unavailable; refresh." } }, "GET /api/health": { body: { status: "healthy" } }, "GET /api/hygiene/review": { body: { review: { version: 1, status: "idle" }, action: null } }, diff --git a/packages/ui-server/tests/storage-routes.test.ts b/packages/ui-server/tests/storage-routes.test.ts index 5e46ef226b..add23c6136 100644 --- a/packages/ui-server/tests/storage-routes.test.ts +++ b/packages/ui-server/tests/storage-routes.test.ts @@ -20,7 +20,7 @@ test("mounted Storage routes reject delegated and revoked principals and arbitra const agent = createPrincipal(fixture.app.db, { authMethod: "delegated", createdBy: owner.id, label: "Athena", ttlSeconds: 3600 }); const cookie = async (id: string) => (await generateSignedCookie("brain_ui_session", id, secret)).split(";")[0]!; const ownerCookie = await cookie(owner.id), agentCookie = await cookie(agent.id); - for (const [method, path] of [["GET", "/api/storage"], ["POST", "/api/storage/scans"], ["DELETE", "/api/storage/scans/unknown"], ["GET", "/api/storage/scans/unknown/entries"], ["GET", "/api/storage/scans/unknown/entries/unknown/references"]]) { + for (const [method, path] of [["GET", "/api/storage"], ["POST", "/api/storage/scans"], ["DELETE", "/api/storage/scans/unknown"], ["GET", "/api/storage/scans/unknown/entries"], ["GET", "/api/storage/scans/unknown/tree"], ["GET", "/api/storage/scans/unknown/entries/unknown/references"]]) { const response = await fixture.fetch(path!, { method, headers: { cookie: agentCookie, "content-type": "application/json" }, ...(method === "POST" ? { body: "{}" } : {}) }); expect(response.status, "STORAGE_DELEGATED_DENIED").toBe(403); expect((await fixture.fetch(path!, { method, ...(method === "POST" ? { headers: { "content-type": "application/json" }, body: "{}" } : {}) })).status).toBe(401); diff --git a/packages/ui-server/tests/storage-tree.test.ts b/packages/ui-server/tests/storage-tree.test.ts new file mode 100644 index 0000000000..2065ec703a --- /dev/null +++ b/packages/ui-server/tests/storage-tree.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, linkSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { StoragePage, StorageStatus, StorageTreePage } from "@schlessera/brain-ui-sdk/protocol"; +import { collectStorage } from "../src/storage/collect"; +import { projectStorageTree } from "../src/storage/tree"; +import { httpContractApp } from "./helpers/http-contract-app"; + +test("tree uses subtree union bytes, accounts directory metadata and assigns alias area once", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-tree-")), brain = join(root,"brain"), app = join(root,"app"); + mkdirSync(join(brain,"charts/deep"), {recursive:true}); mkdirSync(join(brain,"maps")); mkdirSync(app); + const original = join(brain,"charts/deep/Troy.pdf"); writeFileSync(original, Buffer.alloc(12_345, 65)); + linkSync(original,join(brain,"maps/Ithaca.pdf")); linkSync(original,join(app,"chart.pdf")); + const single = join(app,"state.db"); writeFileSync(single,"Odysseus"); + try { + const scan = await collectStorage({brainPath:brain,salt:"odysseus",stores:[{path:app,label:"App",owner:"app",category:"journal",recursive:true},{path:single,label:"DB",owner:"app",category:"database",recursive:false}],unavailable:[{label:"Backup",owner:"operator",category:"backup",reason:"unavailable"}]},"tree",new AbortController().signal); + const all = projectStorageTree(scan,{}), brainRoot = scan.summary.roots.find(row=>row.category==="content")!; + expect(all.totals).toEqual(scan.summary.totals.all); + expect(all.nodes.reduce((sum,node)=>sum+(node.area.logical??0),0),"STORAGE_TREE_CROSS_ROOT_AREA_UNION").toBe(12_353); + const unknown = all.nodes.find(node=>node.owner==="operator")!; + expect(unknown.area).toEqual({logical:null,allocated:null,coverage:"unknown"}); + expect(projectStorageTree(scan,{owner:"operator",search:"nothing"}).totals.allocated,"STORAGE_TREE_EMPTY_UNKNOWN_FILTER").toBeNull(); + const tree = projectStorageTree(scan,{parentId:brainRoot.id}), charts = tree.nodes.find(node=>node.name==="charts")!, maps = tree.nodes.find(node=>node.name==="maps")!; + expect(tree.totals,"STORAGE_TREE_FOLDER_INCLUDES_DESCENDANTS").toEqual({logical:brainRoot.logical,allocated:brainRoot.allocated,coverage:brainRoot.coverage}); + expect(charts.totals.logical,"STORAGE_TREE_FOLDER_INCLUDES_DESCENDANTS").toBe(12_345); + expect(charts.totals.allocated).toBe((statSync(join(brain,"charts")).blocks+statSync(join(brain,"charts/deep")).blocks+statSync(original).blocks)*512); + expect(maps.totals.logical).toBe(12_345); + expect(maps.area.logical,"STORAGE_TREE_ALIAS_AREA_ONLY_ONCE").toBe(0); + expect(tree.nodes.reduce((sum,node)=>sum+(node.area.allocated??0),0)).toBe(brainRoot.allocated!); + expect(tree.nodes.find(node=>node.kind==="metadata")?.area.allocated,"STORAGE_TREE_DIRECTORY_METADATA").toBe(statSync(brain).blocks*512); + const nested = projectStorageTree(scan,{parentId:charts.id}), deep = nested.nodes.find(node=>node.name==="deep")!; + expect(nested.totals).toEqual(charts.totals); + const leaf = projectStorageTree(scan,{parentId:deep.id}); + expect(leaf.breadcrumbs.map(crumb=>crumb.name)).toEqual(["All stores","Brain files","charts","deep"]); + expect(leaf.nodes.find(node=>node.name==="Troy.pdf")?.area.logical).toBe(12_345); + expect(projectStorageTree(scan,{parentId:brainRoot.id,search:"MAP"}).totals.logical).toBe(12_345); + expect(()=>projectStorageTree(scan,{parentId:brainRoot.id,owner:"app"})).toThrow("ownership"); + const dbRoot = scan.summary.roots.find(row=>row.label==="DB")!; + expect(projectStorageTree(scan,{parentId:dbRoot.id}).nodes[0]?.kind,"STORAGE_TREE_FILE_ROOT_NOT_DIRECTORY").toBe("file"); + const broken = {...scan,entries:scan.entries.filter(row=>row.id!==charts.id)}; + expect(()=>projectStorageTree(broken,{parentId:brainRoot.id}),"STORAGE_TREE_MISSING_ANCESTOR_REFUSED").toThrow("incomplete"); + } finally { rmSync(root,{recursive:true,force:true}); } +}); + +test("mounted hierarchy and flat queries search and sort the complete scan before signed paging without rescanning", async () => { + const fixture = await httpContractApp(); + try { + const charts=join(fixture.brainPath,"charts");mkdirSync(charts); + for(let index=0;index<205;index++) writeFileSync(join(charts,`Troy-${String(index).padStart(3,"0")}.pdf`),"x".repeat(index+1)); + await fixture.fetch("/api/storage/scans",{method:"POST",headers:{"content-type":"application/json"},body:"{}"}); + let status:StorageStatus; + for(let attempt=0;;attempt++) { status=await (await fixture.fetch("/api/storage")).json() as StorageStatus;if(status.state==="idle")break;if(attempt>=200)throw Error("Scan did not finish");await new Promise(done=>setTimeout(done,10)); } + expect(status.lastFailure).toBeNull();const scan=status.snapshot!,base=`/api/storage/scans/${scan.id}`; + const roots=await (await fixture.fetch(base+"/tree")).json() as StorageTreePage; + const root=roots.nodes.find(node=>node.name==="Brain files")!; + const child=await (await fixture.fetch(base+`/tree?parentId=${root.id}`)).json() as StorageTreePage; + const folder=child.nodes.find(node=>node.name==="charts")!; + // The projection must use this retained observation, including after sources disappear. + rmSync(charts,{recursive:true}); + const query=`parentId=${folder.id}&search=troy&sort=size&metric=logical&direction=desc`; + const response=await fixture.fetch(base+"/tree?"+query),page=await response.json() as StorageTreePage; + expect(response.status,"STORAGE_TREE_MOUNTED_QUERY").toBe(200);expect(response.headers.get("cache-control")).toContain("no-store"); + expect(page.matched).toBe(205);expect(page.nodes).toHaveLength(200); + expect(page.nodes[0]?.name,"STORAGE_TREE_SORTS_BEFORE_PAGING").toBe("Troy-204.pdf"); + expect(page.totals.logical).toBe(205*206/2);expect(page.nextCursor).not.toBeNull(); + const tail=await (await fixture.fetch(base+"/tree?"+query+"&cursor="+encodeURIComponent(page.nextCursor!))).json() as StorageTreePage; + expect(tail.nodes.map(node=>node.name)).toEqual(["Troy-004.pdf","Troy-003.pdf","Troy-002.pdf","Troy-001.pdf","Troy-000.pdf"]); + expect(tail.nextCursor).toBeNull(); + expect((await fixture.fetch(base+"/tree?"+query.replace("desc","asc")+"&cursor="+encodeURIComponent(page.nextCursor!))).status,"STORAGE_TREE_CURSOR_BINDS_ORDER").toBe(409); + expect((await fixture.fetch(base+"/entries?cursor="+encodeURIComponent(page.nextCursor!))).status,"STORAGE_TREE_CURSOR_SEPARATE_FROM_FLAT").toBe(409); + const flat=await (await fixture.fetch(base+"/entries?search=TROY-204&sort=size&metric=logical")).json() as StoragePage; + expect(flat.entries.map(row=>row.name)).toEqual(["Troy-204.pdf"]);expect(flat.fileLogicalTotal).toBe(205); + for(const suffix of ["search="+"x".repeat(129),"sort=unknown","metric=unknown","direction=unknown","parentId=unknown","root=/"]) { + expect((await fixture.fetch(base+"/tree?"+suffix)).status).toBe(400); + } + expect((await fixture.fetch(base+"/tree?cursor=forged")).status).toBe(400); + } finally { await fixture.close(); } +}); From 680aa582e5d22030197d91efa30d2f91e10142e3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 12:56:55 +0200 Subject: [PATCH 09/44] CONTRACT: feat(storage): add durable owner maintenance --- .changeset/storage-maintenance-owners.md | 10 + api-report/ui-sdk.txt | 89 ++++++++ api-report/ui-server.txt | 10 +- docs/decisions/claude-code-runtime.md | 8 +- docs/decisions/http-api-boundary.md | 8 +- docs/decisions/runtime-requirements.md | 4 +- docs/decisions/scheduled-tasks.md | 2 +- docs/decisions/session-principals.md | 6 +- docs/hosting/README.md | 2 +- docs/http-api-coverage.md | 14 +- docs/http-api.md | 70 +++++- docs/inbox-actions.md | 2 +- docs/integration-contract/cli.md | 21 ++ docs/integration-contract/package-api.md | 12 + .../plans/async-collaboration-requirements.md | 2 +- docs/plans/async-collaboration.md | 2 +- docs/plans/navigation-recovery.md | 2 +- docs/plans/storage-accounting.md | 10 +- docs/plans/storage-lifecycle.md | 6 +- packages/core/README.md | 5 + packages/core/src/cli/brain.ts | 5 +- packages/core/src/cli/commands/storage.ts | 34 ++- packages/core/src/lib/git-storage.ts | 14 ++ packages/core/tests/storage-pack.test.ts | 41 ++++ packages/ui-sdk/src/protocol.ts | 2 +- .../ui-sdk/src/server/maintenance-worker.ts | 26 ++- packages/ui-sdk/src/storage.ts | 28 +++ packages/ui-server/README.md | 15 +- .../migrations/051_storage_maintenance.sql | 16 ++ packages/ui-server/src/app.ts | 8 + packages/ui-server/src/brain/client.ts | 48 +++- packages/ui-server/src/brain/host-boundary.ts | 17 +- packages/ui-server/src/cron/scratch-prune.ts | 53 +++-- packages/ui-server/src/inbox/cleanup.ts | 45 ++-- packages/ui-server/src/inbox/runtime.ts | 2 + .../src/routes/storage-maintenance.ts | 31 +++ packages/ui-server/src/storage/coordinator.ts | 19 +- packages/ui-server/src/storage/maintenance.ts | 170 ++++++++++++++ .../ui-server/tests/cli-inventory.test.ts | 18 +- .../fixtures/storage-maintenance-worker.ts | 14 ++ .../ui-server/tests/http-inventory.test.ts | 2 +- .../tests/http-supported-mounts.test.ts | 4 + .../tests/storage-maintenance-owners.test.ts | 85 +++++++ .../tests/storage-maintenance-routes.test.ts | 43 ++++ .../tests/storage-maintenance.test.ts | 212 ++++++++++++++++++ 45 files changed, 1135 insertions(+), 102 deletions(-) create mode 100644 .changeset/storage-maintenance-owners.md create mode 100644 packages/core/tests/storage-pack.test.ts create mode 100644 packages/ui-server/migrations/051_storage_maintenance.sql create mode 100644 packages/ui-server/src/routes/storage-maintenance.ts create mode 100644 packages/ui-server/src/storage/maintenance.ts create mode 100644 packages/ui-server/tests/fixtures/storage-maintenance-worker.ts create mode 100644 packages/ui-server/tests/storage-maintenance-owners.test.ts create mode 100644 packages/ui-server/tests/storage-maintenance-routes.test.ts create mode 100644 packages/ui-server/tests/storage-maintenance.test.ts diff --git a/.changeset/storage-maintenance-owners.md b/.changeset/storage-maintenance-owners.md new file mode 100644 index 0000000000..e3a259847a --- /dev/null +++ b/.changeset/storage-maintenance-owners.md @@ -0,0 +1,10 @@ +--- +"@schlessera/brain": minor +"@schlessera/brain-ui-sdk": minor +"@schlessera/brain-ui-server": minor +--- + +Expose explicit owner-only Storage housekeeping with durable operation receipts, +qualified before/after observations and bounded credential-free workers. Add +non-destructive `brain storage pack --json`; retain scratch pins, inbox settlement, +recovery evidence and existing retention policy across failures and cancellation. diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index 2e703e5be2..28e4c21bd8 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -310,6 +310,12 @@ export "." (./src/index.ts) type StorageCoverage type StorageFileEntry type StorageFilter + type StorageMaintenanceItem + type StorageMaintenanceKind + type StorageMaintenanceObservation + type StorageMaintenanceReceipt + type StorageMaintenanceRequest + type StorageMaintenanceRow type StorageOwner type StoragePage type StorageReferenceEvidence @@ -771,6 +777,12 @@ export "./client" (./src/client/index.ts) type StorageCoverage type StorageFileEntry type StorageFilter + type StorageMaintenanceItem + type StorageMaintenanceKind + type StorageMaintenanceObservation + type StorageMaintenanceReceipt + type StorageMaintenanceRequest + type StorageMaintenanceRow type StorageOwner type StoragePage type StorageReferenceEvidence @@ -1387,6 +1399,12 @@ export "./protocol" (./src/protocol.ts) type StorageCoverage type StorageFileEntry type StorageFilter + type StorageMaintenanceItem + type StorageMaintenanceKind + type StorageMaintenanceObservation + type StorageMaintenanceReceipt + type StorageMaintenanceRequest + type StorageMaintenanceRow type StorageOwner type StoragePage type StorageReferenceEvidence @@ -2074,6 +2092,12 @@ export "./server" (./src/server/index.ts) type StorageCoverage type StorageFileEntry type StorageFilter + type StorageMaintenanceItem + type StorageMaintenanceKind + type StorageMaintenanceObservation + type StorageMaintenanceReceipt + type StorageMaintenanceRequest + type StorageMaintenanceRow type StorageOwner type StoragePage type StorageReferenceEvidence @@ -9290,6 +9314,71 @@ signatures (every public export and the types it is made of) largerThan?: number; cursor?: string; } + StorageMaintenanceItem (packages/ui-sdk/src/storage.ts) + export interface StorageMaintenanceItem { + id: string; + state: "running" | "cleaned" | "failed"; + reason: string | null; + } + StorageMaintenanceKind (packages/ui-sdk/src/storage.ts) + export type StorageMaintenanceKind = "pack-git" | "prune-scratch" | "cleanup-staging"; + StorageMaintenanceObservation (packages/ui-sdk/src/storage.ts) + export interface StorageMaintenanceObservation { + observedAt: number; + coverage: StorageCoverage; + basis: StorageSnapshot["basis"]; + stores: StorageRoot[]; + capacities: StorageCapacityView[]; + attribution: "concurrent-writers-possible"; + } + StorageMaintenanceReceipt (packages/ui-sdk/src/storage.ts) + export interface StorageMaintenanceReceipt { + id: string; + operation: StorageMaintenanceKind; + owner: "core" | "inbox"; + scope: StorageRoot["category"][]; + requestedAt: number; + startedAt: number | null; + completedAt: number | null; + state: "waiting" | "running" | "complete" | "partial" | "refused" | "cancelled" | "unknown"; + reason: string | null; + headroom: { + requiredBytes: null; + reservation: false; + note: string; + }; + before: StorageMaintenanceObservation | null; + after: StorageMaintenanceObservation | null; + outcome: { + items: StorageMaintenanceItem[]; + truncated: boolean; + removed: number | null; + failed: number | null; + deferred: number | null; + packed: number | null; + pinnedFiles: number | null; + pinnedBytes: number | null; + overageBytes: number | null; + suspended: boolean; + } | null; + } + StorageMaintenanceRequest (packages/ui-sdk/src/storage.ts) + export interface StorageMaintenanceRequest { + id: string; + operation: StorageMaintenanceKind; + } + StorageMaintenanceRow (packages/ui-sdk/src/storage.ts) + export interface StorageMaintenanceRow { + id: StorageMaintenanceKind; + label: string; + owner: "core" | "inbox"; + scope: StorageRoot["category"][]; + effect: string; + available: boolean; + reason: string | null; + headroom: StorageMaintenanceReceipt["headroom"]; + latest: StorageMaintenanceReceipt | null; + } StorageOwner (packages/ui-sdk/src/storage.ts) export type StorageOwner = "brain" | "app" | "operator"; StoragePage (packages/ui-sdk/src/storage.ts) diff --git a/api-report/ui-server.txt b/api-report/ui-server.txt index 9eada3803a..4b29b676bc 100644 --- a/api-report/ui-server.txt +++ b/api-report/ui-server.txt @@ -277,10 +277,10 @@ signatures (every public export and the types it is made of) }): Promise; validate(): Promise; skillsSync(): Promise; - scratchPrune(opts?: { - signal?: AbortSignal; + scratchPrune(opts?: MaintenanceRequest & { pinsFile?: string; }): Promise; + storagePack?(opts?: MaintenanceRequest): Promise; } BrainClientOptions (packages/ui-server/src/brain/client.ts) export interface BrainClientOptions { @@ -692,6 +692,12 @@ signatures (every public export and the types it is made of) tempDomain?: MaintenanceTempDomain; maskPaths?: readonly string[]; } + MaintenanceRequest (packages/ui-server/src/brain/client.ts) + export interface MaintenanceRequest { + signal?: AbortSignal; + assertAuthorized?: () => void; + bounded?: boolean; + } MaintenanceTempDomain (packages/ui-sdk/src/server/maintenance-temp-budget.ts) export interface MaintenanceTempDomain { statePath: string; diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index 148666332d..0dd62e9cca 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -63,8 +63,8 @@ next to it. - **Updated.** Nothing in the tree installs, updates, pins or reads the version of this binary. The only version probe in the server is for the `brain` CLI (`Probe the selected content CLI`, - `packages/ui-server/src/brain/client.ts:111-177`, called at - `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:384`). + `packages/ui-server/src/brain/client.ts:121-187`, called at + `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:386`). `brain doctor` runs `claude mcp list` from `PATH` (`which("claude")`, `packages/core/src/cli/commands/doctor.ts:495-499`) — the user's own Claude Code on their own machine, to check the MCP registration, not the server's @@ -212,7 +212,7 @@ unnecessary. `node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs`. The boot probe below turns either into a refusal to start, for the same reason a missing backend refuses to boot (`A missing (or unrecognized) agent backend`, - `packages/ui-server/src/app.ts:329-334`). + `packages/ui-server/src/app.ts:331-336`). - **The published range is still a caret, and that bounds what this repo can guarantee.** `@schlessera/brain-backend-claude` depends on `^0.3.241`, and a host resolves it in its own lockfile. A host can bump the SDK — and so the @@ -243,7 +243,7 @@ than refuses on a mismatch.** pair is one nobody measured. - **At boot, from the binary a turn would spawn.** The same shape as the `brain` CLI probe (`Probe the selected content CLI`, - `packages/ui-server/src/brain/client.ts:111`). The SDK's + `packages/ui-server/src/brain/client.ts:121`). The SDK's resolver is not exported, so the probe must not re-implement it. The SDK resolves the binary when a query is built, and fails there if none is found (`node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs:228`); it then hands diff --git a/docs/decisions/http-api-boundary.md b/docs/decisions/http-api-boundary.md index 4fc3e7883f..d978c25eca 100644 --- a/docs/decisions/http-api-boundary.md +++ b/docs/decisions/http-api-boundary.md @@ -46,11 +46,11 @@ explicit supported rows, not silently excluded by this distinction. The source audit on 2026-09-30 used `e0875269b21b707482545be9e94901acb897893b`. The public health mount precedes auth -(`createHealthRoutes`, `packages/ui-server/src/app.ts:729`), the API guard follows -public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:749`), and the -WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:903`). +(`createHealthRoutes`, `packages/ui-server/src/app.ts:731`), the API guard follows +public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:751`), and the +WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:910`). Static serving is conditional (`if (options.staticRoot)`, -`packages/ui-server/src/app.ts:923-944`). That order establishes access; +`packages/ui-server/src/app.ts:930-951`). That order establishes access; it does not select compatibility status. A hermetic real-app launch, with temporary brain root, in-memory database, diff --git a/docs/decisions/runtime-requirements.md b/docs/decisions/runtime-requirements.md index 573a86c155..0f8ed10218 100644 --- a/docs/decisions/runtime-requirements.md +++ b/docs/decisions/runtime-requirements.md @@ -194,7 +194,7 @@ Paths may appear in local diagnostics; public fixtures use temporary paths. | Requirement | Initial value | Evidence and limit of the claim | | --- | --- | --- | | Claude backend's SDK range | Retain `^0.3.241` | It is the published declaration (`"@anthropic-ai/claude-agent-sdk"`, `packages/ui-backend-claude/package.json:49`). Registry metadata on 2026-09-30 confirms 0.3.241 exists. This is a declared compatibility bound, not proof every allowed release passed current measurements. | -| ui-server's content CLI floor | Retain `0.33.0` | `--` support is the reason documented immediately above `MIN_BRAIN_CLI_VERSION` (`MIN_BRAIN_CLI_VERSION`, `packages/ui-server/src/brain/client.ts:90`); the initial boot refusal test used 0.32.9. The current test exercises the corrected floor (`a 0.40.0 brain repo pin`, `packages/ui-server/tests/brain-client.test.ts:242-254`). | +| ui-server's content CLI floor | Retain `0.33.0` | `--` support is the reason documented immediately above `MIN_BRAIN_CLI_VERSION` (`MIN_BRAIN_CLI_VERSION`, `packages/ui-server/src/brain/client.ts:100`); the initial boot refusal test used 0.32.9. The current test exercises the corrected floor (`a 0.40.0 brain repo pin`, `packages/ui-server/tests/brain-client.test.ts:242-254`). | | Bun engine floor | Retain existing `>=1.3.5` declarations | Existing engine metadata and the doctor's CVE-based warning (`const MIN_BUN`, `packages/core/src/cli/commands/doctor.ts:45`; `function checkRuntime`, `packages/core/src/cli/commands/doctor.ts:82-89`). This record introduces no new doctor verdict or blanket boot gate. | | Pi SDK requirements | Retain existing exact `0.87.1` declarations | `packages/ui-backend-pi/package.json`; a host floor composes with each actual imported Pi dependency's existing exact requirement. The Claude range ruling does not authorize changing Pi's pins. | | Additional default Claude runtime floor | None | There is no evidenced numerical incompatibility threshold in the existing record. The successful pairs are observations, not proof that the preceding release fails. A new library floor needs a feature requirement or reproducible incompatibility and its versioning review. | @@ -271,7 +271,7 @@ reciprocal host-version negotiation is involved. | Existing boundary | How this design uses it | | --- | --- | -| `probeBrainCliVersion` / `brainCliCommand` (`export async function probeBrainCliVersion`, `packages/ui-server/src/brain/client.ts:112-177`) | Reuse executable selection, wrapper, deadline and cleanup; add host floor composition, strict comparison and invocation verification without duplicating the 0.33.0 constant. | +| `probeBrainCliVersion` / `brainCliCommand` (`export async function probeBrainCliVersion`, `packages/ui-server/src/brain/client.ts:122-187`) | Reuse executable selection, wrapper, deadline and cleanup; add host floor composition, strict comparison and invocation verification without duplicating the 0.33.0 constant. | | `installedAgentSdkVersion` (`export function installedAgentSdkVersion`, `packages/ui-backend-claude/src/runtime-probe.ts:55-61`) | Retain importer-relative resolution; validate package name/version and the owner's manifest range. | | `selectedSpawn` / `probeClaudeRuntime` (`export function selectedSpawn`, `packages/ui-backend-claude/src/runtime-probe.ts:69-96`; `export async function probeClaudeRuntime`, `packages/ui-backend-claude/src/runtime-probe.ts:127-147`) | Keep SDK-selected command capture, native/JS override argv, environment, timeout and group cleanup; add strict identity validation and composed runtime floors. | | `probeBackendRuntimes` (`export async function probeBackendRuntimes`, `packages/ui-server/src/agent/backend.ts:392-439`) | Forward validated host requirements; retain active-backend selection and separate unmeasured-pair logging. | diff --git a/docs/decisions/scheduled-tasks.md b/docs/decisions/scheduled-tasks.md index 5292efd6a7..208b8ee262 100644 --- a/docs/decisions/scheduled-tasks.md +++ b/docs/decisions/scheduled-tasks.md @@ -23,7 +23,7 @@ Source inspected at `b5c30c180df5584584f261b446c4d440380d2441`: (`else if (item.status === "scheduled"`, `packages/ui-server/src/inbox/actions.ts:188-189`). The runtime ticks every minute (`INBOX_TICK_MS`, `packages/ui-server/src/inbox/runtime.ts:11-13`). - Production app wiring supplies recovery and budgets, without a dispatcher - (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:401-417`). + (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:403-419`). A scripted nonempty ready-item test still reports no claim, attempt or Activity root. That proves disabled wiring, not safe execution of a schedule. - Headless plumbing checks backend support, a usable principal, lifetime and diff --git a/docs/decisions/session-principals.md b/docs/decisions/session-principals.md index 019e799eea..f9ba7e73f8 100644 --- a/docs/decisions/session-principals.md +++ b/docs/decisions/session-principals.md @@ -96,9 +96,9 @@ Both reviewers verified every row below against the source. | Cookie verify | `auth.ts:265-296` | strict `^(\d+)\.(\d+)$`, safe integers, future/expired rejection, `cookieEpoch === sessionsEpoch(db)` | | Global epoch | `auth.ts:302-318` | `settings` row; missing = 0; non-integer **throws** | | Global revoke | `auth.ts:321-330` | `setSetting` (overflow-guarded) + `clients.closeAll(1008, …)` | -| Auth routes | `authRoutes`, `app.ts:735` | mounted **before** the guard at `authGuard(authMode`, `app.ts:749`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | -| Passkey management | `passkeyManagementRoutes`, `app.ts:759` | mounted **after** the guard ("Mount AFTER the auth guard") | -| WS guard | `"/ws"`, `app.ts:903-920` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | +| Auth routes | `authRoutes`, `app.ts:737` | mounted **before** the guard at `authGuard(authMode`, `app.ts:751`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | +| Passkey management | `passkeyManagementRoutes`, `app.ts:761` | mounted **after** the guard ("Mount AFTER the auth guard") | +| WS guard | `"/ws"`, `app.ts:910-927` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | | WS upgrade | `ws/connection.ts:262-264` | `createWsUpgrade` ignores the request context | | WS admission | `ws/connection.ts:84` | `clients.add(ws)` — the socket's identity is unknown | | Turn record | `ws/run-session.ts:134-142` | recorder built from `{turnId, sessionId, billing}`; `RunningTurn` (`ws/turns.ts:55-77`) has no actor | diff --git a/docs/hosting/README.md b/docs/hosting/README.md index b1e3df1c0a..d7963554f3 100644 --- a/docs/hosting/README.md +++ b/docs/hosting/README.md @@ -44,7 +44,7 @@ the separately authorized cache measurement procedure. **The server refuses to boot against `@schlessera/brain` older than 0.42.0.** The floor is enforced at -(`MIN_BRAIN_CLI_VERSION`, `packages/ui-server/src/brain/client.ts:90`) and covers +(`MIN_BRAIN_CLI_VERSION`, `packages/ui-server/src/brain/client.ts:100`) and covers all commands and flags the server invokes, including 0.42.0’s scratch pin input. Hygiene review requires 0.41.0's selection, repair, recheck and fingerprint-bound dispositions; module settings and `stats --history` require 0.40.0. The command/flag inventory test guards the diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index f5d007b8a8..23cc396b3f 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:118-151`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:122-155`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -28,6 +28,10 @@ row's real mounting check. They do not establish mounting by themselves. | Supported operation | Mounted receipt | Detailed checks | | --- | --- | --- | +| `GET /api/storage/maintenance` | `mounted supported handler: GET /api/storage/maintenance` | SM | +| `POST /api/storage/maintenance` | `mounted supported handler: POST /api/storage/maintenance` | SM | +| `GET /api/storage/maintenance/:id` | `mounted supported handler: GET /api/storage/maintenance/:id` | SM | +| `DELETE /api/storage/maintenance/:id` | `mounted supported handler: DELETE /api/storage/maintenance/:id` | SM | | `GET /api/storage` | `mounted supported handler: GET /api/storage` | ST | | `POST /api/storage/scans` | `mounted supported handler: POST /api/storage/scans` | ST | | `DELETE /api/storage/scans/:id` | `mounted supported handler: DELETE /api/storage/scans/:id` | ST | @@ -467,3 +471,11 @@ ST: [Cached hierarchy, union areas and query paging](../packages/ui-server/tests [real public-reader/store union and paging](../packages/ui-server/tests/storage-collection.test.ts), [exact metadata roots](../packages/ui-server/tests/storage-metadata.test.ts), and [bounded index evidence](../packages/core/tests/storage-references.test.ts). + +SM: [mounted maintenance authority and body limits](../packages/ui-server/tests/storage-maintenance-routes.test.ts), +[durable identity, real process death, cancellation and owner reports](../packages/ui-server/tests/storage-maintenance.test.ts), +[real credential-free workers, retained pins and output termination](../packages/ui-server/tests/storage-maintenance-owners.test.ts), +and [real Git history preservation and a full private tmpfs](../packages/core/tests/storage-pack.test.ts). +Existing [inbox unwind](../packages/ui-server/tests/inbox-unwind.test.ts) and +[resolution](../packages/ui-server/tests/inbox-resolve.test.ts) tests retain live +settlement and unacknowledged staging protections. diff --git a/docs/http-api.md b/docs/http-api.md index 92e530ea0b..fdb35eff2a 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 141 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 145 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -26,6 +26,10 @@ client code has a gap. Source owners are listed after the table. | Method | Path | Class | Purpose | Consumers, existing promise and reason | | --- | --- | --- | --- | --- | +| GET | `/api/storage/maintenance` | S | Read approved maintenance rows and latest receipts | Current human-owner authority; three fixed owners only. | +| POST | `/api/storage/maintenance` | S | Admit one identified maintenance operation | Durable owner-bound identity and credential-free worker execution. | +| GET | `/api/storage/maintenance/:id` | S | Read the original operation receipt | Readback never replays an effect. | +| DELETE | `/api/storage/maintenance/:id` | S | Request cancellation of remaining work | Existing committed effects remain in the receipt. | | GET | `/api/storage` | S | Read current bounded inventory state | Human-owner Storage inspection; no delegated or worker access. | | POST | `/api/storage/scans` | S | Refresh configured stores | Human-owner Storage inspection; no delegated or worker access. | | DELETE | `/api/storage/scans/:id` | S | Cancel an active scan | Human-owner Storage inspection; no delegated or worker access. | @@ -174,7 +178,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 141 declared endpoints are mounted regardless of backend, renderer or +All 145 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1301,5 +1305,65 @@ Index timestamps do not prove current source freshness. Unavailable index suppor returns unavailable evidence; an empty observed list never proves no references. No file body is read for inventory, type filtering or this index inspection. -No endpoint authorizes deletion, expiry, maintenance or recovery. The complete +Inventory endpoints do not authorize deletion, expiry, maintenance or recovery. The complete Storage pane is delivered separately with its approved removal journey. + +## Storage maintenance (additive, #2011) + +These four operations require the same current human-owner authority as Storage +inventory. GETs and cancellation additionally bind receipts to their initiating +principal. Authorization is checked at admission, after waiting for owner locks, +before worker launch and before output publication. Responses are private, +no-store; origin and JSON protections remain in force. + +`GET /api/storage/maintenance` returns `{operations, limits}`. Each row has a fixed +`id`, label, owner, scope, effect, availability/reason, headroom disclosure and +nullable latest receipt. The only operation IDs are `pack-git`, `prune-scratch` +and `cleanup-staging`. These delegate respectively to core's existing +non-destructive packing, pin-refreshed scratch pruning, and the inbox's settled +staging cleanup. They do not run the general inbox drain or retention tick. + +`POST /api/storage/maintenance` accepts only `{id, operation}` in at most 4096 +bytes. IDs contain 16–96 ASCII letters, digits, `_` or `-`. It returns 202 with +the durable receipt. Repeating the same identity/principal/operation reads that +receipt; mismatched reuse is 409. Only one admitted Storage operation per brain +runs across database connections. Another identity while one is active returns +409. Requests cannot choose roots, executables, commands or retention policy. + +`GET /api/storage/maintenance/:id` returns the original receipt; missing or +other-principal receipts return 404. `DELETE` on the same path returns 202 with +that receipt and requests cancellation. Cancellation stops further admission and +interrupts the bounded worker; it does not promise rollback of committed effects. +Current inbox item acknowledgement can finish before later items stop. + +SDK `StorageMaintenanceReceipt` contains identity, operation/owner/scope, +request/start/completion timestamps, state, reason, headroom, nullable before/after +observations and owner outcome. States are `waiting`, `running`, `complete`, +`partial`, `refused`, `cancelled` and `unknown`. Interrupted started work remains +unknown unless the owner supplies a durable item outcome; readback never blindly +re-dispatches it. Failed refresh suspends scratch deletion; retained pins and +pinned overage survive. Live, unacknowledged or recovery-gated staging remains +with its existing owner. There is no new cleanup/expiry schedule. + +Before/after observations separately retain affected stores and capacities, +coverage and basis. They explicitly permit concurrent writers. File counts and +packed-object counts are never converted into reclaimed bytes. Unknown headroom +is null; old/new Git packs may coexist and required temporary bytes are workload +dependent. Existing worker admission controls its temporary space. A known zero +capacity refuses packing, and receipt admission failure starts no owner action. +No other recovery store is purged to make room. + +The worker has a five-minute deadline and a combined 1 MiB output ceiling; +termination awaits its existing cleanup. Kernel uninterruptible I/O cannot carry +a hard completion guarantee. Each staging pass considers at most 200 eligible +rows. Receipts reserve at most 64 KiB each, with bounded item detail and explicit +truncation; the per-brain admission ceiling is 1024 receipts. Reaching it refuses +new IDs with 503 and retains all existing receipts. This is a resource admission +bound, not a new retention expiry. Lost completion persistence retains intent and +readback describes uncertainty. Invalid input is 400, oversized bodies 413, +non-JSON input 415, and unavailable owners/admission 503 or a refused receipt. + +Packing alone requires content core 0.44.0 or newer. Older installations produce +an explicit refusal; the ordinary server CLI minimum remains 0.42.0. Hosted CLI +operations use only the qualified credential-free maintenance worker; unsupported +hosts do not fall back to executing repository code on the host. diff --git a/docs/inbox-actions.md b/docs/inbox-actions.md index fdfd742b93..fa2d196fe1 100644 --- a/docs/inbox-actions.md +++ b/docs/inbox-actions.md @@ -95,7 +95,7 @@ Dismissing that Action terminates its source and journals cleanup. Cleanup failu retains the failed compensation record for inspection/recovery. `cleanup_pending` is server-owned work, not a model effect. Its compensation -(`createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:14-84`) counts no model +(`createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:18-89`) counts no model operation and takes no spend reservation. Compensation claims it before I/O, removes the server-selected staging directory and partial directory idempotently, then acknowledges `done` in a separate transaction. A crash diff --git a/docs/integration-contract/cli.md b/docs/integration-contract/cli.md index 1d2dfb0d95..fec122d07c 100644 --- a/docs/integration-contract/cli.md +++ b/docs/integration-contract/cli.md @@ -3,6 +3,27 @@ Authoritative component of the [integration contract](../integration-contract.md). Its [shared scope and versioning policy](../integration-contract.md) apply to every section below. +## Storage packing (additive, 0.44.0, #2011) + +`brain storage pack --json` runs only the existing non-destructive Git +loose-object, incremental-repack and pack-refs tasks, including the second +loose-object pass. It preserves history, refs and reflogs. Unlike metadata +inventory, packing loads the normal CLI context and obeys its write/read-only +guards; a governed host invokes it only inside its credential-free maintenance +boundary. There are no arbitrary roots, commands or expiry options. + +The JSON result is `{schema_version: 1, action: "pack", state, packed, reason, +headroom}`. `state` is `complete`, `refused` or `partial`; exit code is 0 for +complete and 2 otherwise. `packed` counts loose objects packed by the owner or +is null when no complete count is known. It is not reclaimed bytes. `reason` +is null, `no_space`, `not_a_git_worktree` or `git_maintenance_unavailable`. +`headroom` is null before capacity inspection, or `{availableBytes, requiredBytes: +null, reason}`. Availability is a nullable live statfs observation of the Git +common store. Required temporary bytes depend on the workload; old/new packs +can coexist, and this command reserves no space. Known zero availability +refuses before packing. A failure after maintenance started is partial and +requires remeasurement; it is never reported as a rollback or success. + ## Storage inventory (additive, #2009) `BRAIN_ROOT=/absolute/brain brain storage inventory --json` emits one bounded diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index dd762a0825..55f2860750 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3240,3 +3240,15 @@ a qualified charge and never a reclaimability guarantee. Bun/types/default exports resolve source/declarations/build from the same release. Older core installations without this entry produce unavailable inventory, with no CLI or internal-import fallback. + +### Storage maintenance protocol (additive, 0.44.0, #2011) + +`@schlessera/brain-ui-sdk/protocol` exports `StorageMaintenanceKind`, +`StorageMaintenanceRequest`, `StorageMaintenanceItem`, +`StorageMaintenanceObservation`, `StorageMaintenanceReceipt` and +`StorageMaintenanceRow`. These describe the three fixed host-owner operations, +caller-owned durable identity, qualified state/outcome, headroom disclosure and +independent store/capacity measurements in the +[maintenance HTTP contract](../http-api.md#storage-maintenance-additive-2011). +They do not add an agent tool or generic executor. Existing automatic retention +and owner APIs retain their previous defaults. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index e36aad82a5..d7991ff58f 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -226,7 +226,7 @@ Stated before the requirements because five of them derive from it. that is the shape to copy, including its `close()` lifecycle. - R19. **The cron backstop has independent authorization before the general guard.** Mount it on the existing listener before - (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:749`). + (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:751`). Authorize a boot-minted ephemeral token, rotated each boot and stored in a 0600 runtime file, with the actual socket address as an additional check. Proxy headers cannot authorize it. The poke succeeds in every auth mode diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 1cbcfe5f7b..1b82629178 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -131,7 +131,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. owns manual triggers/history; it does not tick the inbox. - **Two connections:** (`export function createUiDb(`, `packages/ui-server/src/db/client.ts:25-37`) sets WAL, foreign keys and a 5-second busy timeout. Claims are immediate transactions. -- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:749`) follows public routes; +- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:751`) follows public routes; (`export function authGuard(`, `packages/ui-server/src/middleware/auth.ts:189-249`) binds principals in each auth mode. An internal poke needs independent token authorization before this guard. - **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:271-365`). Timeout unwind is diff --git a/docs/plans/navigation-recovery.md b/docs/plans/navigation-recovery.md index 6819f745c2..ec32eccaf9 100644 --- a/docs/plans/navigation-recovery.md +++ b/docs/plans/navigation-recovery.md @@ -397,7 +397,7 @@ implementation. Any PR moving these lines owns their citation repairs. - Activity resolution: (`.get("/activity/runs", (c) => {`, `packages/ui-server/src/routes/activity.ts:142-275`). -- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:749`). +- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:751`). - Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:878-922`). diff --git a/docs/plans/storage-accounting.md b/docs/plans/storage-accounting.md index c3edf3ed5e..a4c0718c27 100644 --- a/docs/plans/storage-accounting.md +++ b/docs/plans/storage-accounting.md @@ -62,22 +62,22 @@ Source anchors, checked against the reviewed commit: - A: root and index resolution (`initContext`, `packages/core/src/lib/context.ts:36-50`); existing corpus exclusions (`corpusSize`, `packages/core/src/lib/stats.ts:158-209`), available capacity (`freeSpaceBytes`, `packages/core/src/lib/stats.ts:213-224`). -- B: loose report (`looseObjects`, `packages/core/src/lib/git-storage.ts:55-64`) - and safe packing (`packRepository`, `packages/core/src/lib/git-storage.ts:76-84`). +- B: loose report (`looseObjects`, `packages/core/src/lib/git-storage.ts:57-66`) + and safe packing (`packRepository`, `packages/core/src/lib/git-storage.ts:78-86`). The completed #94 and #398 provide existing stats/maintenance foundations. - C: snapshot fields (`snapshotOf`, `packages/core/src/lib/stats-history.ts:132-172`), thinning (`thin`, `packages/core/src/lib/stats-history.ts:232-244`). -- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:255-290`), +- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:257-292`), WAL store (`createUiDb`, `packages/ui-server/src/db/client.ts:25-37`), and private locations (`statePaths`, `packages/ui-server/src/brain/operational-state.ts:107-113`). - E: live/archive paths (`transcriptPaths`, `packages/ui-backend-claude/src/transcript-archive.ts:29-38`). - F: session location (`piSessionDirectory`, `packages/ui-backend-pi/src/session-storage.ts:8-28`). - G: staging location (`shareStagingRoot`, `packages/ui-server/src/share/staging.ts:143-145`) - and settlement-aware cleanup (`createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:14-38`). + and settlement-aware cleanup (`createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:18-42`). [#807](https://github.com/schlessera/brain-kit/issues/807) supplies live-work and settlement protection for staging cleanup. - H: scratch defaults (`SCRATCH_DIR`, `packages/core/src/lib/scratch.ts:52-61`), - scheduled prune (`startScratchPrune`, `packages/ui-server/src/cron/scratch-prune.ts:37-45`), + scheduled prune (`startScratchPrune`, `packages/ui-server/src/cron/scratch-prune.ts:41-49`), and the [writable-area decision](../decisions/worker-writable-areas.md). [#1656](https://github.com/schlessera/brain-kit/issues/1656) supplies retained scratch conversation pins. diff --git a/docs/plans/storage-lifecycle.md b/docs/plans/storage-lifecycle.md index 4724c34395..2e0a8fc0b1 100644 --- a/docs/plans/storage-lifecycle.md +++ b/docs/plans/storage-lifecycle.md @@ -31,7 +31,7 @@ Use maintenance workers for repository code, the CLI and Git. Host routes may perform narrowly validated descriptor operations, as the journal/revert do; they must not evaluate brain configuration, hooks, filters or executables. The existing brain client supplies boundary execution -(`async function execBrain`, `packages/ui-server/src/brain/client.ts:215-260`). +(`async function execBrain`, `packages/ui-server/src/brain/client.ts:225-277`). Do not turn the new surface into a host command runner or an extension seam. Reuse existing authorization. A requested reversible, fully journaled edit @@ -70,11 +70,11 @@ by this spike. Source foundations for the matrix: - Packing runs two passes and deletes only loose duplicates already held in - packs (`export function packRepository`, `packages/core/src/lib/git-storage.ts:76-84`). + packs (`export function packRepository`, `packages/core/src/lib/git-storage.ts:78-86`). The upstream [maintenance manual](https://git-scm.com/docs/git-maintenance) explains these tasks; broad `gc` has different expiry behavior. - Staging eligibility, removal and acknowledgment are separate - (`export function createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:14-94`). + (`export function createInboxCleanup`, `packages/ui-server/src/inbox/cleanup.ts:18-102`). - Scratch pin loading and per-entry revalidation sit inside pruning (`export function pruneScratch`, `packages/core/src/lib/scratch.ts:626-707`); explicit clean is different diff --git a/packages/core/README.md b/packages/core/README.md index 731800f617..a5556c393d 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -75,3 +75,8 @@ Host-shell authors can call `readStorageInventory` from Git, repository configuration or the brain CLI. Enforce owner authorization and choose explicit roots at the host boundary. The [host reader contract](../../docs/integration-contract/package-api.md#host-storage-reader-additive-2010) defines resource bounds, Git marker support, unknown evidence and cancellation. + +`brain storage pack --json` runs the existing non-destructive Git packing tasks +without expiring history or refs. Its [qualified result](../../docs/integration-contract/cli.md#storage-packing-additive-0440-2011) +reports refusal/partial outcomes and workload-dependent temporary headroom. +Governed hosts invoke it through their credential-free maintenance worker. diff --git a/packages/core/src/cli/brain.ts b/packages/core/src/cli/brain.ts index 48f7679cd5..ffb8813c4b 100755 --- a/packages/core/src/cli/brain.ts +++ b/packages/core/src/cli/brain.ts @@ -50,7 +50,7 @@ const MUTATING_COMMANDS = new Set([ // render writes the rendered file into the repo. "render", // scratch removes files from the brain's scratch area. - "scratch", + "scratch", "storage", ]); /** @@ -61,6 +61,7 @@ const MUTATING_COMMANDS = new Set([ const MUTATING_SUBCOMMAND: Record = { skills: "sync", graph: "compute", + storage: "pack", }; /** @@ -127,7 +128,7 @@ async function main(): Promise { const { command, wantsHelp } = scanCliArgs(argv); // Storage observes host-selected roots without executing repository config. - if (command === "storage") { + if (command === "storage" && argv[1] !== "pack") { const { storageCommand, runStorageInventory } = await import("./commands/storage.js"); if (wantsHelp) { console.log(storageCommand.helpBlock); return 0; } await runStorageInventory(argv.slice(1), json); diff --git a/packages/core/src/cli/commands/storage.ts b/packages/core/src/cli/commands/storage.ts index 467d5db88f..5c4bfe0555 100644 --- a/packages/core/src/cli/commands/storage.ts +++ b/packages/core/src/cli/commands/storage.ts @@ -1,3 +1,4 @@ +import { gitPackingHeadroom, isGitWorkTree, packRepository } from "../../lib/git-storage.js"; import { resolveEnv } from "../../config/env.js"; import { resolve } from "node:path"; import { collectStorageProcess } from "../../lib/storage-process.js"; @@ -5,8 +6,9 @@ import { UsageError } from "../io.js"; import type { CoreCommand } from "../types.js"; export const storageCommand: CoreCommand = { - summary: "Bounded read-only content and Git storage inventory", + summary: "Bounded storage inventory and non-destructive Git packing", helpBlock: `brain storage inventory --json +brain storage pack --json Collect metadata for BRAIN_ROOT without evaluating brain configuration, opening a database or reading attachment bytes. Linux descriptor traversal is required. @@ -14,8 +16,13 @@ The finite snapshot includes hierarchy pages (200 entries), observation coverage logical/object-union bytes and qualified filesystem capacity. Page cursors belong to this snapshot only and expire after 60 seconds. A new invocation is a new scan. Trusted hosts may set BRAIN_STORAGE_GIT_ROOTS to a JSON array of allowed external -Git directories. No request flag accepts or expands those roots.`, - run: (args, cli) => runStorageInventory(args, cli.json), +Git directories. No request flag accepts or expands those roots. + +pack runs the existing loose-object/incremental-pack/ref-packing maintenance. +It preserves refs, reflogs and history. Old and new packs can coexist; temporary +headroom is workload-dependent and is not reserved. Exit 2 retains a qualified +refusal/partial report; packing never runs broad gc or expires history.`, + run: async (args, cli) => args[0] === "pack" ? runStoragePack(args, cli.brain.root, cli.json) : runStorageInventory(args, cli.json), }; export async function runStorageInventory(args: string[], json: boolean): Promise { @@ -37,3 +44,24 @@ export async function runStorageInventory(args: string[], json: boolean): Promis else console.log(`${result.status}: ${result.totals.entries} entries, ${result.totals.objectLogicalBytes} observed logical bytes`); } finally { process.off("SIGTERM", cancel); process.off("SIGINT", cancel); } } + +export function runStoragePack(args: string[], root: string, json: boolean): number { + if (args[0] !== "pack" || args.slice(1).some(arg => arg !== "--json")) throw new UsageError("Usage: brain storage pack --json"); + let started = false; + const result: { schema_version: 1; action: "pack"; state: "complete" | "refused" | "partial"; packed: number | null; reason: string | null; headroom: ReturnType | null } = + { schema_version: 1, action: "pack", state: "refused", packed: null, reason: null, headroom: null }; + try { + if (!isGitWorkTree(root)) result.reason = "not_a_git_worktree"; + else { + result.headroom = gitPackingHeadroom(root); + if (result.headroom.availableBytes === 0) result.reason = "no_space"; + else { started = true; result.packed = packRepository(root).packed; result.state = "complete"; } + } + } catch (error) { + result.state = started ? "partial" : "refused"; + result.reason = /ENOSPC|No space left/i.test(String(error)) ? "no_space" : "git_maintenance_unavailable"; + } + if (json) console.log(JSON.stringify(result)); + else console.log(result.state === "complete" ? `Packed ${result.packed} loose objects; history and refs retained.` : `Packing ${result.state}: ${result.reason}. Re-measure storage before trying again.`); + return result.state === "complete" ? 0 : 2; +} diff --git a/packages/core/src/lib/git-storage.ts b/packages/core/src/lib/git-storage.ts index 890f431f6c..034d5f382a 100644 --- a/packages/core/src/lib/git-storage.ts +++ b/packages/core/src/lib/git-storage.ts @@ -1,3 +1,5 @@ +import { statfsSync } from "node:fs"; + /** * Git storage upkeep for a brain repository: the non-destructive packing * `brain maintain` runs, and the read-only figures `brain doctor` reports. @@ -89,3 +91,15 @@ export function originalRefs(root: string): string[] { if (!out.ok) throw new GitStorageError(firstLine(out.stderr, "git for-each-ref failed")); return out.stdout ? out.stdout.split("\n") : []; } + +/** Capacity of the Git common store, resolved inside the caller's execution boundary. */ +export function gitPackingHeadroom(root: string): { availableBytes: number | null; requiredBytes: null; reason: string } { + const common = git(root, ["rev-parse", "--path-format=absolute", "--git-common-dir"]); + if (!common.ok) throw new GitStorageError("Git common store unavailable"); + let availableBytes: number | null = null; + try { + const info = statfsSync(common.stdout, { bigint: true }), bytes = info.bavail * info.bsize; + if (bytes >= 0n && bytes <= BigInt(Number.MAX_SAFE_INTEGER)) availableBytes = Number(bytes); + } catch { /* An unknown capacity is not zero. Git still reports write refusals. */ } + return { availableBytes, requiredBytes: null, reason: "Old and new packs coexist; required temporary bytes depend on the workload. No space is reserved." }; +} diff --git a/packages/core/tests/storage-pack.test.ts b/packages/core/tests/storage-pack.test.ts new file mode 100644 index 0000000000..8cbf8dafaf --- /dev/null +++ b/packages/core/tests/storage-pack.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { makeTempBrain, cleanup, runCli } from "./cli-harness"; +test("storage pack uses existing non-destructive maintenance and preserves history and refs", async () => { + const root = makeTempBrain(); + const git = (...args: string[]) => { const p = Bun.spawnSync(["git", "-C", root, ...args], { stdout: "pipe", stderr: "pipe", env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" } }); expect(p.exitCode, p.stderr.toString()).toBe(0); return p.stdout.toString(); }; + try { + git("init"); git("add", "."); git("-c", "user.name=Odysseus", "-c", "user.email=odysseus@example.invalid", "commit", "-m", "Ithaca"); + writeFileSync(join(root, "chart.bin"), Buffer.alloc(8192, 7)); git("add", "chart.bin"); git("-c", "user.name=Odysseus", "-c", "user.email=odysseus@example.invalid", "commit", "-m", "Voyage"); + git("tag", "departure", "HEAD~1"); const refs = git("show-ref"), history = git("log", "--all", "--format=%H"), reflog = git("reflog", "--all", "--format=%H"); + const result = await runCli(root, ["storage", "pack", "--json"]); + expect(result.code, "STORAGE_PACK_NARROW_COMMAND").toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ schema_version: 1, action: "pack", state: "complete", headroom: { requiredBytes: null } }); + expect(git("show-ref"), "STORAGE_PACK_REFS_PRESERVED").toBe(refs); expect(git("log", "--all", "--format=%H")).toBe(history); expect(git("reflog", "--all", "--format=%H")).toBe(reflog); + expect(git("show", "HEAD:chart.bin")).toHaveLength(8192); + } finally { cleanup(root); } +}); + +test("packing on a genuinely full private tmpfs refuses before maintenance and preserves recovery bytes", async () => { + const { mkdtempSync, rmSync } = await import("node:fs"), { tmpdir } = await import("node:os"); + const root = mkdtempSync(join(tmpdir(), "odysseus-packing-full-")); + const cli = join(import.meta.dir, "../src/cli/brain.ts"); + const source = `import {writeFileSync,openSync,writeSync,closeSync,existsSync,statfsSync} from "node:fs"; +const root=${JSON.stringify(root)}; +const env={...process.env,GIT_CONFIG_GLOBAL:"/dev/null",GIT_CONFIG_NOSYSTEM:"1",GIT_OPTIONAL_LOCKS:"0"}; +function git(...args){const p=Bun.spawnSync(["git","-C",root,...args],{env,stdout:"pipe",stderr:"pipe"});if(p.exitCode)throw Error(p.stderr.toString());return p.stdout.toString();} +writeFileSync(root+"/brain.config.ts","export default {};\\n");writeFileSync(root+"/recovery.txt","Odysseus retains the chart.");git("init");git("add",".");git("-c","user.name=Odysseus","-c","user.email=odysseus@example.invalid","commit","-m","Ithaca");const before=git("show-ref"); +const fd=openSync(root+"/occupied","wx");try{for(let i=0;i<4096;i++)writeSync(fd,Buffer.alloc(4096));}catch(error){if(error.code!=="ENOSPC")throw error;}finally{closeSync(fd);} +if(statfsSync(root).bavail!==0)throw Error("volume did not fill"); +const p=Bun.spawnSync([process.execPath,${JSON.stringify(cli)},"storage","pack","--json"],{env:{...env,BRAIN_ROOT:root},stdout:"pipe",stderr:"pipe"}); +console.log(JSON.stringify({code:p.exitCode,out:p.stdout.toString(),err:p.stderr.toString(),sameRefs:before===git("show-ref"),retained:existsSync(root+"/recovery.txt")}));`; + try { + const child = Bun.spawn(["bwrap", "--unshare-user", "--unshare-net", "--bind", "/", "/", "--size", "1048576", "--tmpfs", root, "--proc", "/proc", "--dev", "/dev", process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { stdout: "pipe", stderr: "pipe" }); + const [out, err, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + expect(code, err).toBe(0); const result = JSON.parse(out) as {code: number; out: string; err: string; sameRefs: boolean; retained: boolean}; + expect(result.code, result.err).toBe(2); + expect(JSON.parse(result.out), "STORAGE_PACK_ZERO_SPACE_REFUSAL").toMatchObject({ state: "refused", reason: "no_space", packed: null, headroom: { availableBytes: 0 } }); + expect(result.sameRefs).toBe(true); expect(result.retained).toBe(true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index fd19abfaf5..e5a677feeb 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3646,4 +3646,4 @@ export interface ServerScopeDecisionReceipt { } export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-journal.js"; -export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; +export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageMaintenanceKind, StorageMaintenanceRequest, StorageMaintenanceItem, StorageMaintenanceObservation, StorageMaintenanceReceipt, StorageMaintenanceRow, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; diff --git a/packages/ui-sdk/src/server/maintenance-worker.ts b/packages/ui-sdk/src/server/maintenance-worker.ts index 5baee24a40..e4d0ac9e13 100644 --- a/packages/ui-sdk/src/server/maintenance-worker.ts +++ b/packages/ui-sdk/src/server/maintenance-worker.ts @@ -26,6 +26,8 @@ export const CREDENTIAL_FREE_ENV = ["PATH", "BRAIN_PATH", "BRAIN_ROOT", "NODE_EN "BRAIN_RERANK_MODE", "XDG_BIN_HOME", "GIT_DISCOVERY_ACROSS_FILESYSTEM"] as const; export interface MaintenanceCommand { + /** Combined stdout/stderr ceiling for a bounded host operation. */ + outputBytes?: number; /** Trusted, resolved host limits; separate from agent turn temp and output staging. */ tempLimits?: WorkerStateLimits; /** Existing host operational-state identity; absent standalone callers use their private domain. */ @@ -227,10 +229,6 @@ function killWorker(child: { pid: number; kill(signal?: NodeJS.Signals): void }, if (!reached) child.kill(signal); } -async function collect(stream: ReadableStream): Promise { - return new Response(stream).text(); -} - /** Runs one maintenance command to completion. An abort or the deadline kills the whole worker. */ export async function runMaintenanceWorker(input: MaintenanceCommand & { stdin?: string; @@ -238,16 +236,32 @@ export async function runMaintenanceWorker(input: MaintenanceCommand & { timeoutMs?: number; }): Promise { input.signal?.throwIfAborted(); + if (input.outputBytes !== undefined && (!Number.isSafeInteger(input.outputBytes) || input.outputBytes < 1 || input.outputBytes > 64 * 1024 * 1024)) throw Error("Invalid maintenance output limit"); const worker = launchMaintenanceWorker(input); let killed = false; const kill = () => { killed = true; worker.kill("SIGKILL"); }; input.signal?.addEventListener("abort", kill, { once: true }); const timer = input.timeoutMs ? setTimeout(kill, input.timeoutMs) : undefined; + let bytes = 0; + const collect = async (stream: ReadableStream) => { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) { + bytes += chunk.byteLength; + if (bytes > (input.outputBytes ?? Infinity)) { kill(); continue; } + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("utf8"); + }; + const stdoutRead = collect(worker.stdout), stderrRead = collect(worker.stderr); + const inputWrite = worker.input(input.stdin ?? ""); try { - void worker.input(input.stdin ?? ""); - const [stdout, stderr, exitCode] = await Promise.all([collect(worker.stdout), collect(worker.stderr), worker.exited]); + const [stdout, stderr, exitCode] = await Promise.all([stdoutRead, stderrRead, worker.exited, inputWrite]); input.signal?.throwIfAborted(); return { stdout, stderr, exitCode, killed }; + } catch (error) { + kill(); + await Promise.allSettled([stdoutRead, stderrRead, worker.exited, inputWrite]); + throw error; } finally { if (timer) clearTimeout(timer); input.signal?.removeEventListener("abort", kill); diff --git a/packages/ui-sdk/src/storage.ts b/packages/ui-sdk/src/storage.ts index 1a950ede1b..27f4f1d497 100644 --- a/packages/ui-sdk/src/storage.ts +++ b/packages/ui-sdk/src/storage.ts @@ -69,3 +69,31 @@ export interface StorageTreePage { breadcrumbs: { id: string | null; name: string }[]; nodes: StorageTreeNode[]; totals: StorageBytes; matched: number; nextCursor: string | null; stale: boolean; } + +export type StorageMaintenanceKind = "pack-git" | "prune-scratch" | "cleanup-staging"; +export interface StorageMaintenanceRequest { id: string; operation: StorageMaintenanceKind } +export interface StorageMaintenanceItem { id: string; state: "running" | "cleaned" | "failed"; reason: string | null } +export interface StorageMaintenanceObservation { + observedAt: number; coverage: StorageCoverage; basis: StorageSnapshot["basis"]; + stores: StorageRoot[]; + capacities: StorageCapacityView[]; + /** Availability deltas include other writers and are not attributed savings. */ + attribution: "concurrent-writers-possible"; +} +export interface StorageMaintenanceReceipt { + id: string; operation: StorageMaintenanceKind; owner: "core" | "inbox"; + scope: StorageRoot["category"][]; + requestedAt: number; startedAt: number | null; completedAt: number | null; + state: "waiting" | "running" | "complete" | "partial" | "refused" | "cancelled" | "unknown"; + reason: string | null; + headroom: { requiredBytes: null; reservation: false; note: string }; + before: StorageMaintenanceObservation | null; after: StorageMaintenanceObservation | null; + outcome: { items: StorageMaintenanceItem[]; truncated: boolean; removed: number | null; failed: number | null; deferred: number | null; packed: number | null; pinnedFiles: number | null; pinnedBytes: number | null; overageBytes: number | null; suspended: boolean } | null; +} +export interface StorageMaintenanceRow { + id: StorageMaintenanceKind; label: string; owner: "core" | "inbox"; + scope: StorageRoot["category"][]; effect: string; + available: boolean; reason: string | null; + headroom: StorageMaintenanceReceipt["headroom"]; + latest: StorageMaintenanceReceipt | null; +} diff --git a/packages/ui-server/README.md b/packages/ui-server/README.md index 3c5e29822d..e9a6b4a192 100644 --- a/packages/ui-server/README.md +++ b/packages/ui-server/README.md @@ -145,8 +145,9 @@ requirements and is refused; injection remains supported without them, and can still use a verified `brainCli` minimum. The server refuses to boot against a content CLI older than 0.42.0, the first -release with every command and flag it invokes, including scratch pin input and -hygiene review. +release with its ordinary commands and flags, including scratch pin input and +hygiene review. Optional Storage packing separately probes core 0.44.0 before +dispatch and refuses older/unknown versions without raising the ordinary floor. The CLI always retains this package floor. An explicit `brainCli` minimum makes a failed or unknown version an error and rechecks the selected executable before each client invocation, including streaming sync. Direct @@ -489,3 +490,13 @@ its configured application `SOURCE_COMMIT` (or `dev` when unset). The existing `version` field retains its source-commit meaning. This metadata is diagnostic, not proof of protocol compatibility or a check against the latest npm release. Public `/api/health` deliberately exposes neither release nor commit. + +### Explicit Storage maintenance + +The [owner-only maintenance API](../../docs/http-api.md#storage-maintenance-additive-2011) +exposes existing Git packing, pin-protected scratch pruning and settled inbox +staging cleanup. Stable caller IDs retain restart-readable receipts; retries +read the same operation. Cancellation and crashes preserve uncertainty about +committed effects. Before/after store measurements and capacity observations are +independent and never promise reclaimed bytes. No retention default or automatic +cleanup schedule changes. The initial Storage pane is delivered separately. diff --git a/packages/ui-server/migrations/051_storage_maintenance.sql b/packages/ui-server/migrations/051_storage_maintenance.sql new file mode 100644 index 0000000000..34f643156e --- /dev/null +++ b/packages/ui-server/migrations/051_storage_maintenance.sql @@ -0,0 +1,16 @@ +-- Host-owned durable identities. No automatic receipt expiry or owner-data purge. +CREATE TABLE storage_maintenance_operations ( + id TEXT PRIMARY KEY, + brain TEXT NOT NULL, + principal_id TEXT NOT NULL, + operation TEXT NOT NULL, + state TEXT NOT NULL, + cancel_requested INTEGER NOT NULL DEFAULT 0, + pid INTEGER NOT NULL, + pid_start TEXT NOT NULL, + requested_at INTEGER NOT NULL, + receipt TEXT NOT NULL +); +CREATE UNIQUE INDEX storage_maintenance_active ON storage_maintenance_operations(brain) + WHERE state IN ('waiting','running'); +CREATE INDEX storage_maintenance_recent ON storage_maintenance_operations(brain, principal_id, requested_at); diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index 11b9859a4c..812ac55310 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -1,3 +1,5 @@ +import { createStorageMaintenance } from "./storage/maintenance.js"; +import { createStorageMaintenanceRoutes } from "./routes/storage-maintenance.js"; import { createStorageCoordinator } from "./storage/coordinator.js"; import { createStorageRoutes } from "./routes/storage.js"; import type { StorageStore } from "./storage/collect.js"; @@ -831,6 +833,11 @@ export async function createApp(options: CreateAppOptions = {}): Promise storage.close()); app.route("/api", createStorageRoutes(storage)); + const storageMaintenance = createStorageMaintenance(db, { brainPath: config.brainPath, storage, brain, scratch: scratchPrune, inbox, boundary, + authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, + }); + bootCleanup.push(() => storageMaintenance.close()); + app.route("/api", createStorageMaintenanceRoutes(storageMaintenance)); const reverts = createTurnReverts(db, { brainPath: config.brainPath, journal, gate: hostExecution.gate()!, lock: host.workerLock, isAuthorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, @@ -960,6 +967,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise void | Promise) => { try { await close(); } catch (error) { failures.push(error); } }; + await attempt(() => storageMaintenance.close()); await attempt(() => inbox.close()); await attempt(() => host.close()); await attempt(async () => { await Promise.allSettled([...host.turnTasks]); }); diff --git a/packages/ui-server/src/brain/client.ts b/packages/ui-server/src/brain/client.ts index 8a4446723e..453823d1dd 100644 --- a/packages/ui-server/src/brain/client.ts +++ b/packages/ui-server/src/brain/client.ts @@ -27,6 +27,14 @@ import { brainCommandClass, createHostBoundary, type HostBoundary } from "./host * A `brain sync` that exited non-zero, with its result when stdout held one: * an agent that failed after its runtime reported itself keeps that report. */ +export class BrainMaintenanceUnavailable extends Error { + constructor(message: string, readonly reason: "core_upgrade_or_probe_required" | "owner_admission_refused" = "owner_admission_refused") { super(message); } +} +export class BrainMaintenanceError extends Error { + constructor(message: string, readonly stdout: string, readonly exitCode: number) { super(message); } +} +export interface MaintenanceRequest { signal?: AbortSignal; assertAuthorized?: () => void; bounded?: boolean } +export const STORAGE_PACK_CLI_VERSION = "0.44.0"; export class BrainSyncError extends Error { constructor(message: string, readonly result?: BrainSyncOutput) { super(message); @@ -73,7 +81,9 @@ export interface BrainClient { /** Run `brain skills sync` — re-materialize every agent's skill links. */ skillsSync(): Promise; /** Run `brain scratch prune` — the periodic pass over the scratch area (#310). An aborted signal kills the child. */ - scratchPrune(opts?: { signal?: AbortSignal; pinsFile?: string }): Promise; + scratchPrune(opts?: MaintenanceRequest & { pinsFile?: string }): Promise; + /** Optional narrow packing capability; requires its own CLI version floor. */ + storagePack?(opts?: MaintenanceRequest): Promise; } interface ExecResult { @@ -111,7 +121,7 @@ export function brainCliCommand(brainPath: string): string[] { /** Probe the selected content CLI, keeping unknown versions warning-only by default. */ export async function probeBrainCliVersion(brainPath: string, log: Logger, options: { exec: ExecWrapperConfig; minimumVersion?: string; phase?: string; command?: readonly string[]; signal?: AbortSignal; /** @internal Where the CLI runs: in a maintenance worker while the host execution boundary holds (#1648). */ - boundary?: HostBoundary }): Promise { + boundary?: HostBoundary; outputBytes?: number; assertAuthorized?: () => void }): Promise { const phase = options.phase ?? "startup"; const requirements = [{ owner: "@schlessera/brain-ui-server", kind: "minimum" as const, declaration: MIN_BRAIN_CLI_VERSION }]; if (options.minimumVersion !== undefined) { @@ -133,7 +143,7 @@ export async function probeBrainCliVersion(brainPath: string, log: Logger, optio // The CLI is the brain's own code: answered from a maintenance worker. const answer = await boundary.run({ argv: [...command, "--version"], env: subprocessEnv("brainCli", { NO_COLOR: "1" }), credentials: false, gate: "shared", purpose: "brain CLI version", ...(options.signal ? { signal: options.signal } : {}), - timeoutMs: VERSION_PROBE_TIMEOUT_MS }); + timeoutMs: VERSION_PROBE_TIMEOUT_MS, ...(options.outputBytes ? { outputBytes: options.outputBytes } : {}), ...(options.assertAuthorized ? { assertAuthorized: options.assertAuthorized } : {}) }); result = { stdout: answer.stdout, stderr: answer.stderr, exitCode: answer.timedOut ? null : answer.exitCode, timedOut: answer.timedOut === true, cleanupWarnings: [] }; } else { @@ -212,11 +222,14 @@ export function createBrainClient(opts: BrainClientOptions): BrainClient { throw new Error("searchTimeoutMs must be a positive finite number"); } - async function execBrain(args: string[], signal?: AbortSignal, input?: string, lock?: import("@schlessera/brain-ui-sdk/server").KeyedLock): Promise { + async function execBrain(args: string[], signal?: AbortSignal, input?: string, lock?: import("@schlessera/brain-ui-sdk/server").KeyedLock, maintenance?: MaintenanceRequest & { minimumVersion?: string }): Promise { signal?.throwIfAborted(); + if (maintenance?.bounded && !boundary.isolated()) throw new BrainMaintenanceUnavailable("Storage maintenance requires a qualified worker before probing repository code."); const command = brainCliCommand(brainPath); - if (minimumVersion !== undefined) { - await probeBrainCliVersion(brainPath, log, { exec, minimumVersion, phase: "invocation", command, signal, boundary }); + const requiredVersion = maintenance?.minimumVersion ?? minimumVersion; + if (requiredVersion !== undefined) { + try { await probeBrainCliVersion(brainPath, log, { exec, minimumVersion: requiredVersion, phase: "invocation", command, signal, boundary, ...(maintenance?.bounded ? { outputBytes: 1024 * 1024 } : {}), ...(maintenance?.assertAuthorized ? { assertAuthorized: maintenance.assertAuthorized } : {}) }); } + catch (error) { signal?.throwIfAborted(); if (maintenance?.minimumVersion) throw new BrainMaintenanceUnavailable("Storage packing requires a qualified core CLI at " + maintenance.minimumVersion + " or newer.", "core_upgrade_or_probe_required"); throw error; } signal?.throwIfAborted(); } // Every brain CLI launch goes through the host boundary, not only the @@ -227,7 +240,8 @@ export function createBrainClient(opts: BrainClientOptions): BrainClient { // with the operator's exec wrapper inside it; otherwise in a host process // through the wrapper, as before. const { credentials, gate } = brainCommandClass(args); - return boundary.run({ + let entered = false; + try { return await boundary.run({ argv: [...command, ...args], // Force JSON output when not a TTY env: subprocessEnv("brainCli", { NO_COLOR: "1" }), @@ -236,11 +250,16 @@ export function createBrainClient(opts: BrainClientOptions): BrainClient { purpose: `brain ${args.slice(0, 2).join(" ")}`, ...(args[0] === "sync" ? { sync: true } : {}), ...(lock ? { lock: { keyed: lock, key: "brain-docs" } } : {}), + ...(maintenance?.assertAuthorized ? { assertAuthorized: maintenance.assertAuthorized } : {}), + ...(maintenance?.bounded ? { outputBytes: 1024 * 1024, timeoutMs: 5 * 60_000, onExecute: () => { entered = true; } } : {}), ...(input !== undefined ? { stdin: input } : {}), // Only read-only search passes a signal. Kill even a CLI that ignores // SIGTERM; abandoning a request must not leave the local process alive. ...(signal ? { signal, killSignal: "SIGKILL" as const } : {}), - }); + }); } catch (error) { + if (maintenance?.bounded && !entered) throw new BrainMaintenanceUnavailable("The owner refused maintenance before its command started."); + throw error; + } } function parseJsonOutput(result: ExecResult): T { @@ -430,12 +449,17 @@ export function createBrainClient(opts: BrainClientOptions): BrainClient { return result.stdout; }, + async storagePack(opts) { + const result = await execBrain(["storage", "pack", "--json"], opts?.signal, undefined, undefined, { ...opts, bounded: true, minimumVersion: STORAGE_PACK_CLI_VERSION }); + if (result.exitCode !== 0 && result.exitCode !== 2) throw new BrainMaintenanceError("Git packing failed", result.stdout, result.exitCode); + return JSON.parse(result.stdout); + }, async scratchPrune(opts) { - const args = ["scratch", "prune", ...(opts?.pinsFile ? ["--pins-file", opts.pinsFile, "--json"] : [])]; - const result = await execBrain(args, opts?.signal); + const args = ["scratch", "prune", ...(opts?.pinsFile ? ["--pins-file", opts.pinsFile, "--json"] : opts?.bounded ? ["--json"] : [])]; + const result = await execBrain(args, opts?.signal, undefined, undefined, opts); if (result.exitCode !== 0) { - throw new Error( - `brain scratch prune failed (exit ${result.exitCode}): ${result.stderr || result.stdout}` + throw new BrainMaintenanceError( + `brain scratch prune failed (exit ${result.exitCode}): ${result.stderr || result.stdout}`, result.stdout, result.exitCode ); } return result.stdout; diff --git a/packages/ui-server/src/brain/host-boundary.ts b/packages/ui-server/src/brain/host-boundary.ts index 6da0a09285..f300e162d8 100644 --- a/packages/ui-server/src/brain/host-boundary.ts +++ b/packages/ui-server/src/brain/host-boundary.ts @@ -63,6 +63,12 @@ export interface BrainExecution { /** Host-side Bun options for the old unwrapped path, such as SIGKILL on abort. */ killSignal?: NodeJS.Signals; timeoutMs?: number; + /** @internal Storage owner operations recheck authority after admission/preparation. */ + assertAuthorized?: () => void; + /** @internal Distinguish pre-launch owner refusal from an interrupted effect. */ + onExecute?: () => void; + /** @internal Combined output bound for an explicitly bounded operation. */ + outputBytes?: number; /** Existing host files the command may append to (a cron span sink). */ writeFiles?: readonly string[]; /** A complete hosted sync: its only credentialed part is the host mirror transport. */ @@ -110,6 +116,7 @@ export function brainCommandClass(args: readonly string[]): { credentials: boole // Automatic plain index is keyword-only. Provider/flagged operations keep // the conservative credentialed exclusive envelope. case "index": return args.length === 1 ? { credentials: false, gate: "shared" } : { credentials: true, gate: "exclusive" }; + case "storage": return args.length === 3 && args[1] === "pack" && args[2] === "--json" ? { credentials: false, gate: "shared" } : { credentials: true, gate: "exclusive" }; case "search": return { credentials: true, gate: "shared" }; // A module action runs the module's own code, which may call out. case "module": return args.includes("--action") ? { credentials: true, gate: "exclusive" } : { credentials: false, gate: "shared" }; @@ -331,6 +338,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { brainPath: brain, ...(startupView ? { startupView } : {}), tempLimits: maintenanceTemp, + ...("outputBytes" in execution && execution.outputBytes ? { outputBytes: execution.outputBytes } : {}), ...(tempDomain ? { tempDomain } : {}), // The operator's exec wrapper runs inside the worker, as it does for agent workers. command: wrapCommand(argv, exec.wrapper), @@ -419,7 +427,10 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { withGate, async run(execution) { execution.signal?.throwIfAborted(); + execution.assertAuthorized?.(); if (!isolated()) { + if (execution.outputBytes !== undefined) throw Error("Bounded Storage maintenance requires the qualified worker boundary"); + execution.onExecute?.(); const proc = hostSpawn(execution); try { const [stdout, stderr, exitCode] = await Promise.all([ @@ -435,6 +446,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { if (execution.sync) return outputLock.withLock(brain, () => runMirrorSync({ nativeManager: options.nativeManager, command: command(execution), transportEnv: execution.env, mirror: mirror ??= syncMirrorPath(brain, options.dbPath), ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}) }), { signal: execution.signal }); const run = async () => { + execution.assertAuthorized?.(); const view = await prepareInput(execution.signal); let outputs: Awaited> | undefined; try { @@ -442,9 +454,12 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { outputs = await prepareMaintenanceOutput(brain, { startupView: view, signal: execution.signal }); const launch = command(execution); const git = await prepareMaintenanceGit(launch, outputs, execution.signal); + execution.signal?.throwIfAborted(); execution.assertAuthorized?.(); + execution.onExecute?.(); const result = await runMaintenanceWorker({ ...launch, startupView: outputs.view, output: outputs.output, ...(git ? { git } : {}), stdin: execution.stdin, - ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}) }); + ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}), ...(execution.outputBytes ? { outputBytes: execution.outputBytes } : {}) }); if (!result.killed) { + execution.assertAuthorized?.(); if (execution.lock) await outputs.publishLocked(execution.lock.keyed); else await outputs.publish(); } diff --git a/packages/ui-server/src/cron/scratch-prune.ts b/packages/ui-server/src/cron/scratch-prune.ts index 7100ca5aef..54b6e7f7ce 100644 --- a/packages/ui-server/src/cron/scratch-prune.ts +++ b/packages/ui-server/src/cron/scratch-prune.ts @@ -1,7 +1,7 @@ import type { Database } from "bun:sqlite"; import { recordGovernanceEvent } from "../governance/events.js"; import type { Logger } from "@opentelemetry/api-logs"; -import type { BrainClient } from "../brain/client.js"; +import type { BrainClient, MaintenanceRequest } from "../brain/client.js"; /** * The periodic pass over the brain's scratch area (#310): `brain scratch @@ -21,12 +21,16 @@ import type { BrainClient } from "../brain/client.js"; * `inFlight` and switch pruning off for good. `close()` kills a pass still * running and resolves once its child has exited. */ +export class ScratchPruneSuspended extends Error {} + export const SCRATCH_PRUNE_INTERVAL_MS = 60 * 60 * 1000; export const SCRATCH_PRUNE_DEADLINE_MS = 5 * 60 * 1000; export interface ScratchPruner { /** Run one pass now. Resolves once it finished; never rejects. */ tick(): Promise; + /** Explicit owner pass after existing work settles; preserves the raw owner report. */ + run(request: MaintenanceRequest): Promise; /** Stop the timer, kill a pass in flight, and resolve once it has exited. */ close(): Promise; } @@ -45,26 +49,35 @@ export function startScratchPrune(options: { }): ScratchPruner { const { brain, log } = options; const deadlineMs = options.deadlineMs ?? SCRATCH_PRUNE_DEADLINE_MS; - let inFlight: Promise | null = null; + let inFlight: Promise | null = null; let active: AbortController | null = null; let closed = false; + let tickFlight: Promise | null = null; - const tick = (): Promise => { + const run = (request?: MaintenanceRequest): Promise => { // A pass still running when the next is due is left to finish; the // timer does not stack a second CLI process on top of it. - if (inFlight) return inFlight; - if (closed) return Promise.resolve(); + if (inFlight) return request ? inFlight.catch(() => "").then(() => run(request)) : inFlight; + if (closed) return Promise.reject(new Error("scratch prune is closed")); + request?.signal?.throwIfAborted(); request?.assertAuthorized?.(); const controller = new AbortController(); const deadline = setTimeout( () => controller.abort(new Error(`scratch prune exceeded ${deadlineMs}ms and was killed`)), deadlineMs ); if (typeof deadline === "object" && "unref" in deadline) deadline.unref(); + const abort = () => controller.abort(request?.signal?.reason); + request?.signal?.addEventListener("abort", abort, { once: true }); + if (request?.signal?.aborted) abort(); active = controller; inFlight = (async () => { - if (options.pins) await options.pins.refresh(controller.signal); + if (options.pins) { + try { await options.pins.refresh(controller.signal); } + catch (error) { controller.signal.throwIfAborted(); throw new ScratchPruneSuspended(error instanceof Error ? error.message : String(error)); } + } controller.signal.throwIfAborted(); - return brain.scratchPrune({ signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) }); + request?.assertAuthorized?.(); + return brain.scratchPrune({ ...(request ?? {}), signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) }); })() .then((output) => { try { @@ -76,30 +89,35 @@ export function startScratchPrune(options: { recordGovernanceEvent(options.db, { kind: "scratch_overage", actor: "server", sessionId: null, turnId: null, model: null, profile: null, payload: { bytes: report.bytes, pinnedFiles: report.pinned.files, pinnedBytes: report.pinned.bytes, overageBytes: report.overageBytes } }); - return; + return output; } catch (error) { eventError = error; } } log?.emit({ severityText: "WARN", body: "scratch remains over its size cap; retained conversation pins were preserved", attributes: { "scratch.overage_bytes": report.overageBytes, "scratch.pinned_bytes": report.pinned?.bytes ?? 0, ...(eventError ? { "governance.error": eventError instanceof Error ? eventError.message : String(eventError) } : {}) } }); } - } catch { /* Legacy human-output CLI; failures are handled below. */ } - }) - .catch((error: unknown) => { - log?.emit({ - severityText: "WARN", - body: "scratch prune failed; the next pass is in an hour", - attributes: { error: error instanceof Error ? error.message : String(error) }, - }); + } catch { /* Legacy human-output CLI remains an unknown structured report. */ } + return output; }) .finally(() => { clearTimeout(deadline); + request?.signal?.removeEventListener("abort", abort); active = null; inFlight = null; }); return inFlight; }; + const tick = (): Promise => { + if (closed) return Promise.resolve(); + if (tickFlight) return tickFlight; + tickFlight = run().then(() => undefined).catch((error: unknown) => { + log?.emit({ severityText: "WARN", body: "scratch prune failed; the next pass is in an hour", + attributes: { error: error instanceof Error ? error.message : String(error) } }); + }).finally(() => { tickFlight = null; }); + return tickFlight; + }; + const every: Every = options.every ?? ((check, ms) => { @@ -113,11 +131,12 @@ export function startScratchPrune(options: { return { tick, + run, close: () => { closed = true; cancel(); active?.abort(new Error("the server is shutting down")); - return inFlight ?? Promise.resolve(); + return inFlight?.then(() => undefined, () => undefined) ?? Promise.resolve(); }, }; } diff --git a/packages/ui-server/src/inbox/cleanup.ts b/packages/ui-server/src/inbox/cleanup.ts index a56aa5b9cc..eb6c34ccff 100644 --- a/packages/ui-server/src/inbox/cleanup.ts +++ b/packages/ui-server/src/inbox/cleanup.ts @@ -9,13 +9,17 @@ import { safeResolve } from "../files/walker.js"; import { assertInboxRecoveryReady } from "./recovery-gate.js"; import { inboxThreadAwaitingSettlement, reapInboxThreadLifetimes } from "./lifetime.js"; +export interface CleanupItemOutcome { id: string; state: "running" | "cleaned" | "failed"; reason: string | null } +export interface CleanupReport { cleaned: number; failed: number; deferred: number; cancelled: boolean; truncated: boolean; items: CleanupItemOutcome[] } +export interface CleanupRequest { signal?: AbortSignal; assertAuthorized?: () => void; onItem?: (item: CleanupItemOutcome) => void } + /** Recoverable compensation, never called from a SQLite transaction. Each * directory removal is idempotent; its durable lease is acknowledged afterwards. */ export function createInboxCleanup(db: Database, brainRoot: string, options: { now?: () => number; /** Internal crash fixture between removal and acknowledgement; never wired by createApp. */ afterRemove?: () => Promise } = {}) { const now = options.now ?? Date.now, store = createInboxStore(db, { now }); - let active: Promise | null = null, closed = false; + let active: Promise | null = null, closed = false; function claim(id: string): InboxQueueItem | null { return db.transaction(() => { assertInboxRecoveryReady(db); @@ -51,14 +55,19 @@ export function createInboxCleanup(db: Database, brainRoot: string, options: { n for (const name of [stagingId, `.${stagingId}.partial`]) await rm(join(root, name), { recursive: true, force: true }); } - async function reconcile(): Promise { - const rows = db.query("SELECT id FROM inbox_items WHERE deleted_at IS NULL AND queue = 'queue' AND type = 'cleanup_pending' AND status IN ('ready','claimed') ORDER BY id") - .all() as { id: string }[]; - let cleaned = 0; + async function reconcile(request?: CleanupRequest): Promise { + request?.signal?.throwIfAborted(); request?.assertAuthorized?.(); assertInboxRecoveryReady(db); + const rows = db.query("SELECT id FROM inbox_items WHERE deleted_at IS NULL AND queue = 'queue' AND type = 'cleanup_pending' AND status IN ('ready','claimed') ORDER BY id LIMIT ?") + .all(request ? 200 : -1) as { id: string }[]; + const report: CleanupReport = { cleaned: 0, failed: 0, deferred: 0, cancelled: false, truncated: request ? rows.length === 200 : false, items: [] }; for (const { id } of rows) { - if (closed) break; + if (closed || request?.signal?.aborted) { report.cancelled = true; break; } + request?.assertAuthorized?.(); const item = claim(id); - if (!item || item.type !== "cleanup_pending") continue; + if (!item || item.type !== "cleanup_pending") { report.deferred++; continue; } + request?.assertAuthorized?.(); request?.signal?.throwIfAborted(); + request?.onItem?.({ id, state: "running", reason: null }); + let outcome: CleanupItemOutcome; try { await remove(item.payload.stagingId); await options.afterRemove?.(); @@ -66,22 +75,28 @@ export function createInboxCleanup(db: Database, brainRoot: string, options: { n // A process killed here leaves a claimed row; restart safely repeats // rm, which succeeds even when both directories are already absent. store.commit([{ kind: "transition", itemId: id, expectedVersion: item.version, to: "done" }]); - cleaned++; + report.cleaned++; + outcome = { id, state: "cleaned", reason: null }; } catch { const current = store.getItem(id); if (current?.queue === "queue" && current.status === "claimed" && current.version === item.version) failInboxWork(db, id, item.version, now()); + report.failed++; + outcome = { id, state: "failed", reason: "cleanup_incomplete" }; } + if (request) report.items.push(outcome); request?.onItem?.(outcome); } - return cleaned; + return report; + } + function run(request?: CleanupRequest): Promise { + if (active) return request ? active.then(() => run(request)) : active; + if (closed) return Promise.resolve({ cleaned: 0, failed: 0, deferred: 0, cancelled: true, truncated: false, items: [] }); + active = reconcile(request).finally(() => { active = null; }); + return active; } return { - sweep(): Promise { - if (closed) return Promise.resolve(0); - if (active) return active; - active = reconcile().finally(() => { active = null; }); - return active; - }, + sweep: () => run().then(result => result.cleaned), + run: (request: CleanupRequest) => run(request), async close() { closed = true; await active; }, }; } diff --git a/packages/ui-server/src/inbox/runtime.ts b/packages/ui-server/src/inbox/runtime.ts index cc8de6c129..3c87ff4603 100644 --- a/packages/ui-server/src/inbox/runtime.ts +++ b/packages/ui-server/src/inbox/runtime.ts @@ -176,6 +176,8 @@ export function createInboxRuntime(db: Database, deps: { arm(); return { ready, + /** Concrete cleanup owner only; does not route or dispatch inbox work. */ + cleanup: cleanup ? (request: Parameters[0]) => cleanup.run(request) : undefined, tick, async poke() { if (closed) return { ...result(), rearmed: false, closed: true }; diff --git a/packages/ui-server/src/routes/storage-maintenance.ts b/packages/ui-server/src/routes/storage-maintenance.ts new file mode 100644 index 0000000000..4c0ca096c0 --- /dev/null +++ b/packages/ui-server/src/routes/storage-maintenance.ts @@ -0,0 +1,31 @@ +import { Hono } from "hono"; +import { z } from "zod"; +import type { AppEnv } from "../app-env.js"; +import { requireJson } from "../middleware/origin.js"; +import { readJsonBody } from "../middleware/body-limit.js"; +import { StorageUnavailable } from "../storage/coordinator.js"; +import type { StorageMaintenance } from "../storage/maintenance.js"; +const request = z.object({ id: z.string().regex(/^[a-zA-Z0-9_-]{16,96}$/), operation: z.enum(["pack-git", "prune-scratch", "cleanup-staging"]) }).strict(); +export function createStorageMaintenanceRoutes(maintenance: StorageMaintenance): Hono { + const routes = new Hono(); + routes.onError((error, c) => c.json({ error: error instanceof StorageUnavailable ? error.message : "Maintenance unavailable; read the original receipt before considering another operation." }, error instanceof StorageUnavailable ? error.status : 503)); + routes.use("/storage/maintenance*", async (c, next) => { + c.header("Cache-Control", "private, no-store"); + const principal = c.get("principal"); + if (!principal) return c.json({ error: "Authentication required." }, 401); + if (!["owner", "ambient"].includes(principal.kind)) return c.json({ error: "Maintenance requires human-owner access." }, 403); + await next(); + }); + routes.get("/storage/maintenance", c => c.json(maintenance.list(c.get("principal")!.id))); + routes.get("/storage/maintenance/:id", c => c.json(maintenance.receipt(c.get("principal")!.id, c.req.param("id")))); + routes.post("/storage/maintenance", requireJson(), async c => { + let body: unknown; + try { body = await readJsonBody(c, 4096); } catch { return c.json({ error: "Invalid maintenance JSON." }, 400); } + if (body instanceof Response) return body; + const parsed = request.safeParse(body); + if (!parsed.success) return c.json({ error: "Choose one approved operation and a new request identity; roots and commands are host-owned." }, 400); + return c.json(maintenance.start(c.get("principal")!.id, parsed.data), 202); + }); + routes.delete("/storage/maintenance/:id", c => c.json(maintenance.cancel(c.get("principal")!.id, c.req.param("id")), 202)); + return routes; +} diff --git a/packages/ui-server/src/storage/coordinator.ts b/packages/ui-server/src/storage/coordinator.ts index 6ce296e6df..923c44af91 100644 --- a/packages/ui-server/src/storage/coordinator.ts +++ b/packages/ui-server/src/storage/coordinator.ts @@ -1,7 +1,7 @@ import { projectStorageTree, sortStorageRows } from "./tree.js"; import { storageReferences } from "./process.js"; import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; -import type { StorageFilter, StoragePage, StorageStatus, StorageReferenceEvidence, StorageTreeFilter, StorageTreePage } from "@schlessera/brain-ui-sdk/protocol"; +import type { StorageFilter, StoragePage, StorageStatus, StorageReferenceEvidence, StorageTreeFilter, StorageTreePage, StorageSnapshot } from "@schlessera/brain-ui-sdk/protocol"; import { collectStorage, storageTotals, STORAGE_SERVER_LIMITS, type StorageCollection, type StorageCollectOptions } from "./collect.js"; export class StorageUnavailable extends Error { constructor(readonly status: 400 | 403 | 404 | 409 | 429 | 503, message: string) { super(message); } @@ -30,13 +30,13 @@ export function createStorageCoordinator(options: Omit { + signal.throwIfAborted(); authorized(principal); + while (active) { await active.promise; signal.throwIfAborted(); } + const id = api.start(principal, true).activeScanId; + const abort = () => { if (active?.id === id) active.controller.abort(); }; + signal.addEventListener("abort", abort, { once: true }); if (signal.aborted) abort(); + try { await api.idle(); signal.throwIfAborted(); authorized(principal); return [current, previous].find(value => value?.summary.id === id)?.summary ?? null; } + finally { signal.removeEventListener("abort", abort); } + }, async idle() { await active?.promise; }, async close() { closed = true; active?.controller.abort(); referenceJob?.controller.abort(); await Promise.allSettled([active?.promise, referenceJob?.promise]); current = null; previous = null; }, }; + return api; } export type StorageCoordinator = ReturnType; diff --git a/packages/ui-server/src/storage/maintenance.ts b/packages/ui-server/src/storage/maintenance.ts new file mode 100644 index 0000000000..f1e77c704d --- /dev/null +++ b/packages/ui-server/src/storage/maintenance.ts @@ -0,0 +1,170 @@ +/** Explicit host-owned housekeeping; durable identities never imply exactly-once filesystem effects. */ +import type { Database } from "bun:sqlite"; +import { realpathSync } from "node:fs"; +import type { StorageMaintenanceKind, StorageMaintenanceItem, StorageMaintenanceObservation, StorageMaintenanceReceipt, StorageMaintenanceRequest, StorageMaintenanceRow, StorageSnapshot } from "@schlessera/brain-ui-sdk/protocol"; +import { BrainMaintenanceError, BrainMaintenanceUnavailable, type BrainClient } from "../brain/client.js"; +import type { HostBoundary } from "../brain/host-boundary.js"; +import { processAlive, SELF } from "../brain/writer-gate.js"; +import { ScratchPruneSuspended, type ScratchPruner } from "../cron/scratch-prune.js"; +import type { createInboxRuntime } from "../inbox/runtime.js"; +import { StorageUnavailable, type StorageCoordinator } from "./coordinator.js"; + +export const STORAGE_MAINTENANCE_LIMITS = Object.freeze({ receipts: 1024, receiptBytes: 65_536, items: 200, milliseconds: 300_000 }); +const runningHere = new Set(); +const descriptors = [ + { id: "pack-git", label: "Pack Git objects", owner: "core", scope: ["git"], effect: "Pack local objects and refs; history and reflogs remain." }, + { id: "prune-scratch", label: "Prune eligible scratch", owner: "core", scope: ["scratch"], effect: "Apply existing age/size policy; retained answer pins remain." }, + { id: "cleanup-staging", label: "Settled share staging cleanup", owner: "inbox", scope: ["staging"], effect: "Clean only owner-approved settled staging; live work and recovery remain protected." }, +] as const; +type Row = { id: string; principal_id: string; operation: StorageMaintenanceKind; state: StorageMaintenanceReceipt["state"]; pid: number; pid_start: string; receipt: string }; +const emptyOutcome = (): NonNullable => ({ items: [], truncated: false, removed: null, failed: null, deferred: null, packed: null, pinnedFiles: null, pinnedBytes: null, overageBytes: null, suspended: false }); +const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0; +const path = (value: unknown): value is string => typeof value === "string" && value.length <= 16384 && value.startsWith(".brain/scratch/") && !value.split("/").some(part => !part || part === "..") && !value.includes("\0"); +function scratchOutcome(raw: string): NonNullable { + const value = JSON.parse(raw); + if (value.action !== "prune" || !Array.isArray(value.removed) || !Array.isArray(value.failed) || !count(value.bytes) || !count(value.files) + || !count(value.pinned?.files) || !count(value.pinned?.bytes) || !count(value.overageBytes) + || value.removed.some((item: {path?: unknown}) => !path(item.path)) || value.failed.some((item: {path?: unknown}) => !path(item.path))) throw Error("scratch_report_unavailable"); + const items: StorageMaintenanceItem[] = [...value.removed.map((item: {path: string}) => ({ id: item.path, state: "cleaned" as const, reason: null })), ...value.failed.map((item: {path: string}) => ({ id: item.path, state: "failed" as const, reason: "owner_removal_failed" }))]; + return { ...emptyOutcome(), removed: value.removed.length, failed: value.failed.length, pinnedFiles: value.pinned.files, pinnedBytes: value.pinned.bytes, overageBytes: value.overageBytes, suspended: !!value.pinError, items: items.slice(0, 200), truncated: items.length > 200 }; +} +function observation(snapshot: StorageSnapshot | null, scope: readonly string[]): StorageMaintenanceObservation | null { + if (!snapshot) return null; + const stores = snapshot.roots.filter(root => scope.includes(root.category)).map(root => ({ ...root, label: root.label.slice(0, 80), reason: root.reason?.slice(0, 160) ?? null })); + const ids = new Set(stores.map(root => root.id)); + return { observedAt: snapshot.observedAt, coverage: stores.length ? stores.every(root => root.coverage === "complete") ? "complete" : "partial" : "unknown", basis: snapshot.basis, + stores, capacities: snapshot.capacities.filter(capacity => capacity.rootIds.some(id => ids.has(id))).map(capacity => ({ ...capacity, reason: capacity.reason?.slice(0, 160) ?? null })), attribution: "concurrent-writers-possible" }; +} +export function createStorageMaintenance(db: Database, options: { + brainPath: string; storage: StorageCoordinator; brain: Pick; scratch: ScratchPruner; + inbox: Pick, "cleanup">; boundary: Pick; + authorized(principal: string): boolean; +}) { + const brain = realpathSync(options.brainPath), active = new Map }>(); + let closed = false; + const authorize = (principal: string) => { if (!options.authorized(principal)) throw new StorageUnavailable(403, "Storage maintenance requires current human-owner access."); }; + const available = (kind: StorageMaintenanceKind) => kind === "cleanup-staging" ? !!options.inbox.cleanup : options.boundary.status().hostExecution === "worker" && options.boundary.status().available && (kind !== "pack-git" || !!options.brain.storagePack); + const headroom = (kind: StorageMaintenanceKind): StorageMaintenanceReceipt["headroom"] => ({ requiredBytes: null, reservation: false, + note: kind === "cleanup-staging" ? "Cleanup receipts and the operational database need writable capacity. Recovery is never discarded to make room." : "Temporary worker/output space is admitted by its existing owner. Packing may hold old and new packs together; required bytes depend on the workload and are not reserved." }); + const select = (id: string) => db.query("SELECT * FROM storage_maintenance_operations WHERE id=? AND brain=?").get(id, brain) as Row | null; + function encoded(receipt: StorageMaintenanceReceipt, strict = false) { + let text = JSON.stringify(receipt); + if (strict && Buffer.byteLength(text) > STORAGE_MAINTENANCE_LIMITS.receiptBytes) throw Error("receipt_item_budget"); + while (Buffer.byteLength(text) > STORAGE_MAINTENANCE_LIMITS.receiptBytes && receipt.outcome?.items.length) { receipt.outcome.items.pop(); receipt.outcome.truncated = true; text = JSON.stringify(receipt); } + if (Buffer.byteLength(text) > STORAGE_MAINTENANCE_LIMITS.receiptBytes) throw Error("receipt_budget"); + return text + " ".repeat(STORAGE_MAINTENANCE_LIMITS.receiptBytes - Buffer.byteLength(text)); + } + function write(receipt: StorageMaintenanceReceipt, strict = false) { + db.query("UPDATE storage_maintenance_operations SET state=?,receipt=? WHERE id=? AND brain=?").run(receipt.state, encoded(receipt, strict), receipt.id, brain); + } + function reconciled(row: Row): StorageMaintenanceReceipt { + const receipt = JSON.parse(row.receipt) as StorageMaintenanceReceipt; + if (["waiting", "running"].includes(row.state) && (!processAlive(row.pid, row.pid_start) || row.pid === SELF.pid && row.pid_start === SELF.start && !runningHere.has(`${brain}:${row.id}`))) { + receipt.state = row.state === "waiting" ? "refused" : "unknown"; + receipt.reason = row.state === "waiting" ? "interrupted_before_owner" : "interrupted_owner_outcome_unknown"; receipt.completedAt = Date.now(); + // Inbox owns replay of its existing durable cleanup work. This request never re-dispatches it. + if (row.operation === "cleanup-staging") for (const item of receipt.outcome?.items ?? []) { + const owner = db.query("SELECT status FROM inbox_items WHERE id=?").get(item.id) as {status: string} | null; + if (item.state === "running" && owner?.status === "done") item.state = "cleaned"; + } + try { write(receipt); } catch { /* Existing durable intent remains; readback still describes uncertainty. */ } + } + return receipt; + } + function recover() { for (const row of db.query("SELECT * FROM storage_maintenance_operations WHERE brain=? AND state IN ('waiting','running')").all(brain) as Row[]) reconciled(row); } + function receipt(principal: string, id: string) { + authorize(principal); const row = select(id); + if (!row || row.principal_id !== principal) throw new StorageUnavailable(404, "Maintenance receipt unavailable."); + return reconciled(row); + } + async function execute(principal: string, result: StorageMaintenanceReceipt, controller: AbortController) { + let ownerStarted = false; + const assertAuthorized = () => { controller.signal.throwIfAborted(); authorize(principal); }; + const timer = setTimeout(() => controller.abort(), STORAGE_MAINTENANCE_LIMITS.milliseconds); + const revocation = setInterval(() => { try { if (!options.authorized(principal) || (db.query("SELECT cancel_requested FROM storage_maintenance_operations WHERE id=?").get(result.id) as {cancel_requested: number} | null)?.cancel_requested) controller.abort(); } catch { controller.abort(); } }, 100); + try { + assertAuthorized(); + if (!available(result.operation)) { result.state = "refused"; result.reason = "owner_unavailable"; return; } + result.before = observation(await options.storage.measure(principal, controller.signal), result.scope); + assertAuthorized(); + if (result.operation === "pack-git" && result.before?.capacities.some(value => value.available === 0)) { result.state = "refused"; result.reason = "no_space"; return; } + result.state = "running"; result.startedAt = Date.now(); result.outcome = emptyOutcome(); write(result); + ownerStarted = true; + if (result.operation === "pack-git") { + const value = await options.brain.storagePack!({ signal: controller.signal, assertAuthorized, bounded: true }) as { schema_version?: number; action?: string; state?: string; packed?: unknown; reason?: string }; + if (value.schema_version !== 1 || value.action !== "pack" || !["complete", "partial", "refused"].includes(value.state ?? "") || value.packed !== null && !count(value.packed)) throw Error("packing_report_unavailable"); + result.outcome.packed = value.packed as number | null; + result.state = value.state as "complete" | "partial" | "refused"; result.reason = value.reason === "no_space" ? "no_space" : result.state === "complete" ? null : "git_maintenance_incomplete"; + } else if (result.operation === "prune-scratch") { + let raw: string; + try { raw = await options.scratch.run({ signal: controller.signal, assertAuthorized, bounded: true }); } + catch (error) { if (!(error instanceof BrainMaintenanceError) || error.exitCode !== 2) throw error; raw = error.stdout; } + result.outcome = scratchOutcome(raw); + result.state = result.outcome.suspended ? result.outcome.removed ? "partial" : "refused" : result.outcome.failed ? "partial" : "complete"; + result.reason = result.outcome.suspended ? "pins_unavailable" : result.outcome.overageBytes ? "pinned_overage" : result.outcome.failed ? "owner_removal_failed" : null; + } else { + const value = await options.inbox.cleanup!({ signal: controller.signal, assertAuthorized, onItem(item) { + const previous = result.outcome!.items.find(value => value.id === item.id); + if (previous) Object.assign(previous, item); else result.outcome!.items.push(item); + write(result, true); // Item intent is durable before the owner removes its staging. + } }); + result.outcome = { ...emptyOutcome(), items: value.items, removed: value.cleaned, failed: value.failed, deferred: value.deferred, truncated: value.truncated }; + result.state = value.cancelled ? value.cleaned ? "partial" : "cancelled" : value.failed ? "partial" : "complete"; + result.reason = value.cancelled ? "cancelled" : value.failed ? "cleanup_incomplete" : value.deferred ? "live_or_unsettled_items_retained" : null; + } + } catch (error) { + result.state = error instanceof BrainMaintenanceUnavailable || error instanceof ScratchPruneSuspended ? "refused" : ownerStarted ? "unknown" : controller.signal.aborted ? "cancelled" : "refused"; + result.reason = error instanceof ScratchPruneSuspended ? "pins_refresh_incomplete" : error instanceof BrainMaintenanceUnavailable ? error.reason : ownerStarted ? "owner_outcome_unknown" : controller.signal.aborted ? "cancelled_or_revoked" : "owner_unavailable"; + if (error instanceof ScratchPruneSuspended && result.outcome) result.outcome.suspended = true; + } finally { + clearTimeout(timer); clearInterval(revocation); + try { write(ownerStarted ? { ...result, state: "running" } : result); } catch { /* Retain the previously committed intent. */ } + // A cancelled write can still have committed effects. Its fresh observation is independent. + if (result.startedAt && options.authorized(principal) && !closed) { + const after = new AbortController(), deadline = setTimeout(() => after.abort(), 36_000); + try { result.after = observation(await options.storage.measure(principal, after.signal), result.scope); } catch { result.after = null; } + finally { clearTimeout(deadline); } + } + result.completedAt = Date.now(); + try { write(result); } catch { /* Preserve running intent; readback reconciles to unknown, never retries. */ } + } + } + return { + list(principal: string): { operations: StorageMaintenanceRow[]; limits: typeof STORAGE_MAINTENANCE_LIMITS } { + authorize(principal); recover(); + return { limits: STORAGE_MAINTENANCE_LIMITS, operations: descriptors.map(value => { + const latest = db.query("SELECT * FROM storage_maintenance_operations WHERE brain=? AND principal_id=? AND operation=? ORDER BY requested_at DESC,rowid DESC LIMIT 1").get(brain, principal, value.id) as Row | null; + return { ...value, scope: [...value.scope], available: available(value.id), reason: available(value.id) ? null : "owner_unavailable", headroom: headroom(value.id), latest: latest ? reconciled(latest) : null }; + }) }; + }, + start(principal: string, request: StorageMaintenanceRequest): StorageMaintenanceReceipt { + authorize(principal); + if (closed) throw new StorageUnavailable(503, "Maintenance is shutting down."); + if (!/^[a-zA-Z0-9_-]{16,96}$/.test(request.id) || !descriptors.some(value => value.id === request.operation)) throw new StorageUnavailable(400, "Invalid maintenance operation."); + recover(); + const existing = select(request.id); + if (existing) { + if (existing.principal_id !== principal || existing.operation !== request.operation) throw new StorageUnavailable(409, "Operation identity belongs to another request."); + return reconciled(existing); + } + const descriptor = descriptors.find(value => value.id === request.operation)!; + const result: StorageMaintenanceReceipt = { id: request.id, operation: request.operation, owner: descriptor.owner, scope: [...descriptor.scope], requestedAt: Date.now(), startedAt: null, completedAt: null, state: "waiting", reason: null, headroom: headroom(request.operation), before: null, after: null, outcome: null }; + db.transaction(() => { + authorize(principal); + if (db.query("SELECT 1 FROM storage_maintenance_operations WHERE brain=? AND state IN ('waiting','running')").get(brain)) throw new StorageUnavailable(409, "Another Storage operation is active; read its receipt."); + const { total } = db.query("SELECT count(*) AS total FROM storage_maintenance_operations WHERE brain=?").get(brain) as {total: number}; + if (total >= STORAGE_MAINTENANCE_LIMITS.receipts) throw new StorageUnavailable(503, "Maintenance receipt capacity reached; existing receipts are retained."); + db.query("INSERT INTO storage_maintenance_operations (id,brain,principal_id,operation,state,pid,pid_start,requested_at,receipt) VALUES (?,?,?,?,?,?,?,?,?)").run(result.id, brain, principal, result.operation, result.state, SELF.pid, SELF.start, result.requestedAt, encoded(result)); + }).immediate(); + const controller = new AbortController(); runningHere.add(`${brain}:${result.id}`); + const pending = Promise.resolve().then(() => execute(principal, result, controller)).finally(() => { active.delete(result.id); runningHere.delete(`${brain}:${result.id}`); }); + active.set(result.id, { controller, promise: pending }); + return receipt(principal, result.id); + }, + receipt, + cancel(principal: string, id: string) { const result = receipt(principal, id); if (["waiting", "running"].includes(result.state)) db.query("UPDATE storage_maintenance_operations SET cancel_requested=1 WHERE id=? AND brain=? AND principal_id=?").run(id, brain, principal); active.get(id)?.controller.abort(); return result; }, + async idle() { await Promise.allSettled([...active.values()].map(value => value.promise)); }, + async close() { closed = true; for (const value of active.values()) value.controller.abort(); await Promise.allSettled([...active.values()].map(value => value.promise)); }, + }; +} +export type StorageMaintenance = ReturnType; diff --git a/packages/ui-server/tests/cli-inventory.test.ts b/packages/ui-server/tests/cli-inventory.test.ts index 1e5c2e0b75..ec7fce04f7 100644 --- a/packages/ui-server/tests/cli-inventory.test.ts +++ b/packages/ui-server/tests/cli-inventory.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; import { resolve } from "node:path"; import ts from "typescript"; -import { MIN_BRAIN_CLI_VERSION } from "../src/brain/client"; +import { MIN_BRAIN_CLI_VERSION, STORAGE_PACK_CLI_VERSION } from "../src/brain/client"; // Release evidence: baseline argv is present in the 0.33.0 core tag (including // the -- separator). Newer capabilities are recorded in core/CHANGELOG.md: @@ -12,6 +12,7 @@ import { MIN_BRAIN_CLI_VERSION } from "../src/brain/client"; const available: Record = { search: "0.33.0", briefing: "0.33.0", stats: "0.33.0", list: "0.33.0", read: "0.33.0", sync: "0.33.0", add: "0.33.0", index: "0.33.0", validate: "0.33.0", + "storage pack": "0.44.0", // #2011: separately probed optional maintenance capability. "skills sync": "0.33.0", "scratch prune": "0.38.0", "module list": "0.33.0", "module enable": "0.40.0", "module disable": "0.40.0", "module settings": "0.40.0", "hygiene list": "0.39.0", "hygiene next": "0.41.0", "hygiene resolve": "0.41.0", @@ -87,7 +88,7 @@ function inventory() { const first = elements[0]; if (!first || !ts.isExpression(first)) throw new Error("Missing CLI command"); const commands = strings(first).flatMap(command => { - if (["module", "skills", "scratch"].includes(command)) { + if (["module", "skills", "scratch", "storage"].includes(command)) { const sub = elements[1]; if (!sub || !ts.isExpression(sub)) throw new Error(`Missing ${command} subcommand`); return strings(sub).map(value => `${command} ${value}`); @@ -140,6 +141,12 @@ function inventory() { expect(arg.getText()).toBe('["hygiene", "--json", ...args]'); return; } + if (call === "execBrain" && arg.getText() === '["storage", "pack", "--json"]') { + const options = node.arguments[4]; + if (!options || !ts.isObjectLiteralExpression(options)) throw Error("Storage packing needs its explicit capability floor"); + const floor = options.properties.find(property => ts.isPropertyAssignment(property) && property.name.getText() === "minimumVersion"); + expect(floor && ts.isPropertyAssignment(floor) && floor.initializer.getText()).toBe("STORAGE_PACK_CLI_VERSION"); + } inspect(arg, call === "cli" ? "hygiene " : ""); } if (ts.isPropertyAccessExpression(node.expression) && node.expression.name.text === "hygiene") { @@ -180,18 +187,19 @@ function inventory() { return found; } -test("every server CLI command and flag is available at MIN_BRAIN_CLI_VERSION", () => { +test("every server CLI command and flag is available at its enforced baseline or feature floor", () => { const found = inventory(); expect([...found.keys()].filter(command => command !== "--version").sort()).toEqual(Object.keys(available).sort()); const tooNew: string[] = []; for (const [command, flags] of found) { + const floor = command === "storage pack" ? STORAGE_PACK_CLI_VERSION : MIN_BRAIN_CLI_VERSION; const commandVersion = command === "--version" ? globalFlags[command]! : available[command]; expect(commandVersion, `Record release evidence for brain ${command}`).toBeDefined(); - if (Bun.semver.order(commandVersion!, MIN_BRAIN_CLI_VERSION) > 0) tooNew.push(`${command} requires ${commandVersion}`); + if (Bun.semver.order(commandVersion!, floor) > 0) tooNew.push(`${command} requires ${commandVersion}`); for (const flag of flags) { const version = globalFlags[flag] ?? commandFlags[command.split(" ")[0]!]?.[flag]; expect(version, `Record release evidence for brain ${command} ${flag}`).toBeDefined(); - if (Bun.semver.order(version!, MIN_BRAIN_CLI_VERSION) > 0) tooNew.push(`${command} ${flag} requires ${version}`); + if (Bun.semver.order(version!, floor) > 0) tooNew.push(`${command} ${flag} requires ${version}`); } } expect(tooNew, `MIN_BRAIN_CLI_VERSION ${MIN_BRAIN_CLI_VERSION} is below invoked capabilities`).toEqual([]); diff --git a/packages/ui-server/tests/fixtures/storage-maintenance-worker.ts b/packages/ui-server/tests/fixtures/storage-maintenance-worker.ts new file mode 100644 index 0000000000..23c6fc432f --- /dev/null +++ b/packages/ui-server/tests/fixtures/storage-maintenance-worker.ts @@ -0,0 +1,14 @@ +/** Real process-death receipt fixture. Never used by production. */ +import { writeFileSync } from "node:fs"; +import { createUiDb } from "../../src/db/client.js"; +import { createStorageCoordinator } from "../../src/storage/coordinator.js"; +import { createStorageMaintenance } from "../../src/storage/maintenance.js"; +const [dbPath, brainPath, marker, id] = process.argv.slice(2) as [string,string,string,string]; +const db = createUiDb(dbPath), storage = createStorageCoordinator({ brainPath, stores: [], authorized: () => true }); +const operations = createStorageMaintenance(db, { brainPath, storage, authorized: () => true, + boundary: { status: () => ({ hostExecution: "worker", available: true, profile: "strict", protectedPaths: [], identity: "shared_uid", configurationView: "startup" }) }, + brain: { async storagePack() { writeFileSync(marker, "owner entered after durable intent"); await new Promise(() => {}); } }, + scratch: { async tick() {}, async close() {}, async run() { return ""; } }, inbox: { cleanup: undefined }, +}); +operations.start("odysseus", { id, operation: "pack-git" }); +await operations.idle(); diff --git a/packages/ui-server/tests/http-inventory.test.ts b/packages/ui-server/tests/http-inventory.test.ts index 7c9787fbe8..3a89c88b46 100644 --- a/packages/ui-server/tests/http-inventory.test.ts +++ b/packages/ui-server/tests/http-inventory.test.ts @@ -20,7 +20,7 @@ test("real app method/path inventory equals the complete classified HTTP specifi expect(proseCounts).toEqual([declared.length, declared.length]); } const middleware = [...new Set(t.routes.filter((r) => r.method === "ALL").map((r) => r.path))].sort(); - expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*", "/api/storage*"]); + expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*", "/api/storage*", "/api/storage/maintenance*"]); expect(t.routes.filter((r) => r.method === "ALL" && r.path === "/*")).toHaveLength(staticRoot ? 3 : 2); expect(t.routes.some((r) => r.method === "HEAD" || r.method === "OPTIONS")).toBe(false); } finally { await t.close(); } diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index 747b28409b..653571d3d3 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -16,6 +16,10 @@ type Check = { path?: string; init?: RequestInit; status?: number; body?: object // More detailed existing behavior checks are linked by the coverage matrix; // this map is cross-checked with the spec and the actual Hono mounts. const checks: Record = { + "GET /api/storage/maintenance": { verify: async r => { const value = await r.json(); expect(value.operations.map((operation: {id: string}) => operation.id)).toEqual(["pack-git", "prune-scratch", "cleanup-staging"]); } }, + "POST /api/storage/maintenance": { init: json("POST", { id: "odysseus-mounted-001", operation: "pack-git" }), status: 202, body: { id: "odysseus-mounted-001", operation: "pack-git" } }, + "GET /api/storage/maintenance/:id": { path: "/api/storage/maintenance/missing", status: 404, body: { error: "Maintenance receipt unavailable." } }, + "DELETE /api/storage/maintenance/:id": { path: "/api/storage/maintenance/missing", init: { method: "DELETE" }, status: 404, body: { error: "Maintenance receipt unavailable." } }, "GET /api/storage": { body: { version: 1, state: "idle", snapshot: null } }, "POST /api/storage/scans": { init: json("POST", {}), status: 202, body: { version: 1, state: "scanning" } }, "DELETE /api/storage/scans/:id": { path: "/api/storage/scans/missing", init: { method: "DELETE" }, status: 409, body: { error: "That scan is no longer active." } }, diff --git a/packages/ui-server/tests/storage-maintenance-owners.test.ts b/packages/ui-server/tests/storage-maintenance-owners.test.ts new file mode 100644 index 0000000000..3b81578f29 --- /dev/null +++ b/packages/ui-server/tests/storage-maintenance-owners.test.ts @@ -0,0 +1,85 @@ +import { expect, test } from "bun:test"; +import { chmodSync, existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync, utimesSync, readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { createNativeBrainManagerClient } from "@schlessera/brain-ui-sdk/internal"; +import { createNativeAppFixture } from "./helpers/native-app-fixture"; +import { createHostBoundary } from "../src/brain/host-boundary"; +import { createBrainClient } from "../src/brain/client"; +import { createUiDb } from "../src/db/client"; +import { createWriterGate } from "../src/brain/writer-gate"; +import { startScratchPrune } from "../src/cron/scratch-prune"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-maintenance-owner-")), brain = join(root, "brain"); mkdirSync(brain); + writeFileSync(join(brain, "brain.config.ts"), "export default {};\n"); + const core = resolve(import.meta.dir, "../../core/src/cli/brain.ts"), bin = join(brain, "node_modules", ".bin", "brain"); mkdirSync(join(brain, "node_modules", ".bin"), { recursive: true }); + // This unreleased source fixture advertises the new command it actually imports. + writeFileSync(bin, `#!${process.execPath}\nconst args=process.argv.slice(2);if(args[0]==="--version"){console.log("0.44.0");}else{process.argv=[process.execPath,${JSON.stringify(core)},...args];await import(${JSON.stringify(core)});}\n`); chmodSync(bin, 0o700); + const native = createNativeAppFixture(brain), db = createUiDb(join(root, "ui.sqlite")), gate = createWriterGate(db, brain, { pollMs: 5 }); + const boundary = createHostBoundary({ brainPath: brain, dbPath: join(root, "ui.sqlite"), gate: () => gate, history: () => true, nativeManager: createNativeBrainManagerClient(native.nativeBrain) }); + const client = createBrainClient({ brainPath: brain, boundary, exec: {} }); + return { root, brain, client, boundary, gate, async close() { db.close(); await native.close(); rmSync(root, { recursive: true, force: true }); } }; +} + +test("manual scratch pruning refreshes pins and uses the real bounded credential-free CLI worker", async () => { + const f = fixture(), scratch = join(f.brain, ".brain", "scratch"), pins = join(f.brain, ".brain", "scratch-pins", "pins.json"); + mkdirSync(scratch, { recursive: true }); mkdirSync(join(f.brain, ".brain", "scratch-pins")); + writeFileSync(join(f.brain, ".brain", "scratch-pins-required"), "required"); + writeFileSync(pins, JSON.stringify({ version: 1, ready: true, sessions: {} })); + for (const file of ["linked.txt", "old.txt"]) { writeFileSync(join(scratch, file), "Ithaca"); utimesSync(join(scratch, file), 0, 0); } + let ready = false, refreshes = 0; + const pruner = startScratchPrune({ brain: f.client, every: () => () => {}, pins: { file: pins, async refresh() { refreshes++; if (!ready) throw Error("pin enumeration incomplete"); writeFileSync(pins, JSON.stringify({ version: 1, ready: true, sessions: { odysseus: [".brain/scratch/linked.txt"] } })); } } }); + try { + await pruner.tick(); + expect(existsSync(join(scratch, "old.txt")), "STORAGE_FRESH_PINS_BEFORE_DELETE").toBe(true); + await expect(pruner.run({ bounded: true })).rejects.toThrow("pin enumeration incomplete"); + expect(existsSync(join(scratch, "old.txt")), "STORAGE_MANUAL_PINS_REFRESH_BEFORE_DELETE").toBe(true); + ready = true; + const report = JSON.parse(await pruner.run({ bounded: true })); + expect(report.removed.map((value: {path: string}) => value.path)).toContain(".brain/scratch/old.txt"); + expect(existsSync(join(scratch, "linked.txt")), "STORAGE_MANUAL_PIN_PRESERVED").toBe(true); + expect(existsSync(join(scratch, "old.txt"))).toBe(false); expect(refreshes).toBeGreaterThanOrEqual(3); + } finally { await pruner.close(); await f.close(); } +}); + +test("maintenance revalidates authorization after the real cross-process writer gate admits it", async () => { + const f = fixture(); let allowed = true; + const hold = await f.gate.acquire("exclusive", "fixture hold"); + const pending = f.boundary.run({ argv: [process.execPath, "-e", 'await Bun.write(process.env.BRAIN_WORKER_DATA_ROOT+"/started.txt","Ithaca");'], env: { PATH: process.env.PATH, BRAIN_ROOT: f.brain }, credentials: false, gate: "shared", purpose: "Storage guard fixture", outputBytes: 1024, assertAuthorized() { if (!allowed) throw Error("authorization revoked"); } }); + try { + await Bun.sleep(30); allowed = false; hold.release(); + await expect(pending).rejects.toThrow("authorization revoked"); + expect(existsSync(join(f.brain, "started.txt")), "STORAGE_MAINTENANCE_AFTER_GATE_AUTHORITY").toBe(false); + } finally { hold.release(); await pending.catch(() => {}); await f.close(); } +}); + +test("bounded maintenance kills and reaps an actual output-flooding worker", async () => { + const f = fixture(), marker = crypto.randomUUID(); + try { + const result = await f.boundary.run({ argv: [process.execPath, "-e", `const marker=${JSON.stringify(marker)};process.stdout.write("x".repeat(8192));setInterval(()=>process.stdout.write(marker),1);`], env: { PATH: process.env.PATH, BRAIN_ROOT: f.brain }, credentials: false, gate: "shared", purpose: "Storage output budget", outputBytes: 1024, timeoutMs: 5000 }); + expect(result.timedOut, "STORAGE_MAINTENANCE_OUTPUT_CEILING").toBe(true); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(1024); + const alive = readdirSync("/proc").filter(value => /^\d+$/.test(value)).filter(pid => { try { return readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(marker); } catch { return false; } }); + expect(alive, "STORAGE_MAINTENANCE_OUTPUT_REAPED").toEqual([]); + } finally { await f.close(); } +}); + + +test("hosted packing publishes real Git packs while preserving refs and history", async () => { + const f = fixture(); + const git = (...args: string[]) => { + const p = Bun.spawnSync(["git", "-C", f.brain, ...args], { stdout: "pipe", stderr: "pipe", env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" } }); + expect(p.exitCode, p.stderr.toString()).toBe(0); return p.stdout.toString(); + }; + try { + git("init"); git("config", "user.name", "Odysseus"); git("config", "user.email", "odysseus@example.invalid"); + writeFileSync(join(f.brain, "chart.txt"), "Ithaca"); git("add", "chart.txt", "brain.config.ts"); git("commit", "-m", "Voyage"); git("tag", "departure"); + const refs = git("show-ref"), history = git("log", "--all", "--format=%H"), reflog = git("reflog", "--all", "--format=%H"); + const result = await f.client.storagePack!({ bounded: true }); + expect(result, "STORAGE_HOSTED_PACK_OWNER_COMPLETES").toMatchObject({ action: "pack", state: "complete" }); + expect(readdirSync(join(f.brain, ".git", "objects", "pack")).some(name => name.endsWith(".pack")), "STORAGE_HOSTED_PACK_PUBLISHED").toBe(true); + expect(git("show-ref")).toBe(refs); expect(git("log", "--all", "--format=%H")).toBe(history); expect(git("reflog", "--all", "--format=%H")).toBe(reflog); + expect(git("show", "HEAD:chart.txt")).toBe("Ithaca"); + } finally { await f.close(); } +}); diff --git a/packages/ui-server/tests/storage-maintenance-routes.test.ts b/packages/ui-server/tests/storage-maintenance-routes.test.ts new file mode 100644 index 0000000000..6a8e3ca2f3 --- /dev/null +++ b/packages/ui-server/tests/storage-maintenance-routes.test.ts @@ -0,0 +1,43 @@ +import { expect, test } from "bun:test"; +import { httpContractApp } from "./helpers/http-contract-app"; +test("Storage maintenance mounts only the three approved owner operations", async () => { + const f = await httpContractApp(); + try { + const response = await f.fetch("/api/storage/maintenance"); + expect(response.status, "STORAGE_MAINTENANCE_OWNER_ROUTE").toBe(200); + const result = await response.json() as { operations: Array<{ id: string }> }; + expect(result.operations.map(value => value.id)).toEqual(["pack-git", "prune-scratch", "cleanup-staging"]); + } finally { await f.close(); } +}); + +test("mounted maintenance routes enforce owner, origin, body bounds and durable receipt readback", async () => { + const { generateSignedCookie } = await import("hono/cookie"), { createPrincipal } = await import("../src/db/principals"); + const secret = "odysseus-storage-maintenance-secret-0123456789"; + const f = await httpContractApp({ env: { AUTH_MODE: "password", BRAIN_UI_PASSWORD_HASH: "unused-fixture-hash", COOKIE_SECRET: secret, BRAIN_UI_ALLOW_LOOPBACK_ORIGIN: "1" } }); + try { + const owner = createPrincipal(f.app.db, { authMethod: "password", label: "Odysseus", ttlSeconds: 3600 }); + const agent = createPrincipal(f.app.db, { authMethod: "delegated", createdBy: owner.id, label: "Athena", ttlSeconds: 3600 }); + const cookie = async (id: string) => (await generateSignedCookie("brain_ui_session", id, secret)).split(";")[0]!; + const ownerCookie = await cookie(owner.id), agentCookie = await cookie(agent.id); + const input = { id: "odysseus-cleanup-001", operation: "cleanup-staging" }; + for (const [method, path] of [["GET", "/api/storage/maintenance"], ["GET", "/api/storage/maintenance/unknown"], ["POST", "/api/storage/maintenance"], ["DELETE", "/api/storage/maintenance/unknown"]]) { + const init = { method, headers: { cookie: agentCookie, "content-type": "application/json" }, ...(method === "POST" ? { body: JSON.stringify(input) } : {}) }; + expect((await f.fetch(path!, init)).status, "STORAGE_MAINTENANCE_DELEGATED_DENIED").toBe(403); + expect((await f.fetch(path!, { ...init, headers: { "content-type": "application/json" } })).status).toBe(401); + } + const post = (body: unknown, headers: Record = {}) => f.fetch("/api/storage/maintenance", { method: "POST", headers: { cookie: ownerCookie, "content-type": "application/json", ...headers }, body: JSON.stringify(body) }); + expect((await post({ ...input, operation: "scratch-clean" })).status, "STORAGE_MAINTENANCE_APPROVED_OPERATIONS_ONLY").toBe(400); + expect((await post({ ...input, root: "/" })).status).toBe(400); + expect((await post(input, { "content-type": "text/plain" })).status).toBe(415); + expect((await post(input, { origin: "https://troy.example" })).status).toBe(403); + expect((await post({ ...input, extra: "x".repeat(5000) })).status, "STORAGE_MAINTENANCE_BODY_BOUND").toBe(413); + const first = await post(input); expect(first.status).toBe(202); + let result: any; + for (let i = 0; i < 150; i++) { const response = await f.fetch(`/api/storage/maintenance/${input.id}`, { headers: { cookie: ownerCookie } }); expect(response.status).toBe(200); result = await response.json(); if (!["waiting", "running"].includes(result.state)) break; await Bun.sleep(20); } + expect(result.state, JSON.stringify(result)).toBe("complete"); + expect(await (await post(input)).json(), "STORAGE_MAINTENANCE_HTTP_RECEIPT_REPLAY").toEqual(result); + expect((await f.fetch(`/api/storage/maintenance/${input.id}`, { method: "DELETE", headers: { cookie: ownerCookie } })).status).toBe(202); + f.app.db.query("UPDATE principals SET revoked_at=? WHERE id=?").run(Date.now(), owner.id); + expect((await post({ ...input, id: "odysseus-cleanup-002" })).status).toBe(401); + } finally { await f.close(); } +}); diff --git a/packages/ui-server/tests/storage-maintenance.test.ts b/packages/ui-server/tests/storage-maintenance.test.ts new file mode 100644 index 0000000000..6aca62d0d0 --- /dev/null +++ b/packages/ui-server/tests/storage-maintenance.test.ts @@ -0,0 +1,212 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync, unlinkSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createUiDb } from "../src/db/client"; +import { createStorageCoordinator } from "../src/storage/coordinator"; +import { createStorageMaintenance } from "../src/storage/maintenance"; +import { createInboxCleanup } from "../src/inbox/cleanup"; +import { createInboxStore } from "../src/inbox/store"; +import { createBrainClient, STORAGE_PACK_CLI_VERSION } from "../src/brain/client"; +import type { BrainExecution, HostBoundary } from "../src/brain/host-boundary"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-maintenance-")), brain = join(root, "brain"), dbPath = join(root, "ui.sqlite"); + mkdirSync(brain); mkdirSync(join(brain, ".brain", "scratch"), { recursive: true }); writeFileSync(join(brain, ".brain", "scratch", "chart.txt"), "Ithaca map"); + const db = createUiDb(dbPath); let allowed = true, calls = 0; + const storage = createStorageCoordinator({ brainPath: brain, stores: [], authorized: () => allowed }); + const cleanup = createInboxCleanup(db, brain); + const options: Parameters[1] = { + brainPath: brain, storage, authorized: () => allowed, + boundary: { status: () => ({ hostExecution: "worker", available: true, profile: "strict", protectedPaths: [], identity: "shared_uid", configurationView: "startup" }) }, + brain: { async storagePack(request) { request?.assertAuthorized?.(); calls++; return { schema_version: 1, action: "pack", state: "complete", packed: 2, reason: null }; } }, + scratch: { async tick() {}, async close() {}, async run(request) { + request.assertAuthorized?.(); calls++; unlinkSync(join(brain, ".brain", "scratch", "chart.txt")); + return JSON.stringify({ action: "prune", removed: [{ path: ".brain/scratch/chart.txt", bytes: 10, reason: "age" }], failed: [], bytes: 0, files: 0, pinned: { files: 0, bytes: 0 }, overageBytes: 0 }); + } }, + inbox: { cleanup: request => cleanup.run(request) }, + }; + const operations = createStorageMaintenance(db, options); + return { root, brain, dbPath, db, storage, options, operations, get calls() { return calls; }, revoke() { allowed = false; }, allow() { allowed = true; }, + async close() { await operations.close(); await cleanup.close(); await storage.close(); db.close(); rmSync(root, { recursive: true, force: true }); } }; +} + +test("durable request identity serializes effects and exposes independent fresh measurements", async () => { + const f = fixture(), request = { id: "odysseus-prune-0001", operation: "prune-scratch" as const }; + try { + const first = f.operations.start("odysseus", request); + expect(first.state).toBe("waiting"); + let repeatedId: string | undefined; + expect(() => { repeatedId = f.operations.start("odysseus", request).id; }, "STORAGE_MAINTENANCE_IDEMPOTENT").not.toThrow(); + expect(repeatedId).toBe(first.id); + expect(() => f.operations.start("odysseus", { id: "odysseus-prune-0002", operation: "prune-scratch" }), "STORAGE_MAINTENANCE_SERIALIZED").toThrow("active"); + await f.operations.idle(); + const result = f.operations.receipt("odysseus", request.id); + expect(result.state, JSON.stringify(result)).toBe("complete"); expect(f.calls).toBe(1); + expect(result.before?.stores[0]?.logical).toBe(10); expect(result.after?.stores[0]?.logical).toBe(0); + expect(result.after!.observedAt).toBeGreaterThanOrEqual(result.before!.observedAt); + expect(result.after?.attribution).toBe("concurrent-writers-possible"); expect(result.outcome?.removed).toBe(1); + expect(JSON.stringify(result)).not.toContain("reclaimedBytes"); + const reopened = createUiDb(f.dbPath), second = createStorageMaintenance(reopened, f.options); + try { expect(second.start("odysseus", request), "STORAGE_MAINTENANCE_REOPEN_RECEIPT").toEqual(result); expect(f.calls).toBe(1); } + finally { await second.close(); reopened.close(); } + } finally { await f.close(); } +}); + +test("revocation and full receipt storage refuse maintenance before its owner runs", async () => { + const f = fixture(); + try { + f.operations.start("odysseus", { id: "odysseus-revoked-1", operation: "pack-git" }); f.revoke(); await f.operations.idle(); + expect(f.calls, "STORAGE_MAINTENANCE_REVALIDATES_OWNER").toBe(0); + expect(() => f.operations.list("odysseus")).toThrow("human-owner"); f.allow(); + expect(f.operations.receipt("odysseus", "odysseus-revoked-1").state).toBe("refused"); + f.db.exec("CREATE TRIGGER refuse_storage_receipt BEFORE INSERT ON storage_maintenance_operations BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); + expect(() => f.operations.start("odysseus", { id: "odysseus-full-0001", operation: "pack-git" })).toThrow(); + await f.operations.idle(); expect(f.calls, "STORAGE_MAINTENANCE_INTENT_BEFORE_EFFECT").toBe(0); + expect(existsSync(join(f.brain, ".brain", "scratch", "chart.txt"))).toBe(true); + } finally { await f.close(); } +}); + +test("real settled staging cleanup records item intent and refuses recovery-gated work", async () => { + const f = fixture(), stagingId = "12345678-1234-1234-1234-123456789abc", now = Date.now(); + // The established staging path is exported by the protocol. + const { SHARE_STAGING_DIR } = await import("@schlessera/brain-ui-sdk/protocol"); + const target = join(f.brain, SHARE_STAGING_DIR, stagingId); mkdirSync(target, { recursive: true }); writeFileSync(join(target, "chart.txt"), "Ithaca"); + const store = createInboxStore(f.db); + const seed = (threadId: string) => { + const item = store.ingest({ threadId, itemId: `share-${threadId}`, dedupKey: `share-${threadId}`, source: "share", stagingId, stakes: 2, expiresAt: now + 86_400_000 }).item; + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version, to: "claimed", leaseUntil: now + 60_000 }]); + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version + 1, to: "done" }]); + }; + seed("harbor"); + store.commit([{ kind: "item", item: { id: "cleanup-harbor", dedupKey: "cleanup-harbor", threadId: "harbor", queue: "queue", type: "cleanup_pending", status: "ready", attempts: 0, maxAttempts: 3, version: 1, createdAt: now, updatedAt: now, expiresAt: Number.MAX_SAFE_INTEGER, payload: { stagingId } } }]); + try { + f.operations.start("odysseus", { id: "odysseus-staging-1", operation: "cleanup-staging" }); await f.operations.idle(); + const result = f.operations.receipt("odysseus", "odysseus-staging-1"); + expect(result.state, JSON.stringify(result)).toBe("complete"); expect(existsSync(target), "STORAGE_SETTLED_OWNER_CLEANUP").toBe(false); + expect(result.outcome?.items).toEqual([{ id: "cleanup-harbor", state: "cleaned", reason: null }]); + expect(store.getItem("cleanup-harbor")).toMatchObject({ status: "done" }); + mkdirSync(target, { recursive: true }); writeFileSync(join(target, "chart.txt"), "Ithaca"); + seed("retained"); + store.commit([{ kind: "item", item: { id: "cleanup-retained", dedupKey: "cleanup-retained", threadId: "retained", queue: "queue", type: "cleanup_pending", status: "ready", attempts: 0, maxAttempts: 3, version: 1, createdAt: now, updatedAt: now, expiresAt: Number.MAX_SAFE_INTEGER, payload: { stagingId } } }]); + f.db.query("INSERT INTO inbox_recovery_state VALUES (1, ?, ?, 'pending', NULL)").run("a".repeat(64), f.brain); + f.operations.start("odysseus", { id: "odysseus-staging-2", operation: "cleanup-staging" }); await f.operations.idle(); + expect(existsSync(target), "STORAGE_MAINTENANCE_RECOVERY_GATE").toBe(true); + expect(store.getItem("cleanup-retained")).toMatchObject({ status: "ready", attempts: 0 }); + } finally { await f.close(); } +}); + +test("optional packing floor refuses old core before dispatch without raising the ordinary CLI floor", async () => { + const f = fixture(); let version = "0.43.1"; const calls: BrainExecution[] = []; + const boundary = { isolated: () => true, async run(execution: BrainExecution) { + calls.push(execution); execution.assertAuthorized?.(); + return { exitCode: 0, stderr: "", stdout: execution.argv.at(-1) === "--version" ? version : JSON.stringify({ schema_version: 1, action: "pack", state: "complete", packed: 1, reason: null }) }; + } } as HostBoundary; + const client = createBrainClient({ brainPath: f.brain, boundary, exec: {} }); + try { + await expect(client.storagePack!()).rejects.toThrow("0.44.0"); + expect(calls.some(value => value.argv.includes("pack")), "STORAGE_PACK_VERSION_BEFORE_EFFECT").toBe(false); + version = STORAGE_PACK_CLI_VERSION; + await client.storagePack!(); + const command = calls.find(value => value.argv.includes("pack"))!; + expect(command).toMatchObject({ credentials: false, gate: "shared", outputBytes: 1_048_576 }); + expect(command.argv.slice(-3), "STORAGE_PACK_FIXED_ARGV").toEqual(["storage", "pack", "--json"]); + } finally { await f.close(); } +}); + +test("a killed owner leaves a restart-readable unknown receipt and is never automatically retried", async () => { + const f = fixture(), marker = join(f.brain, "owner-effect"), id = "odysseus-crash-001"; + const child = Bun.spawn([process.execPath, join(import.meta.dir, "fixtures/storage-maintenance-worker.ts"), f.dbPath, f.brain, marker, id], { stdout: "pipe", stderr: "pipe" }); + try { + for (let i = 0; i < 500 && !existsSync(marker); i++) await Bun.sleep(10); + expect(existsSync(marker), "STORAGE_MAINTENANCE_CRASH_REACHED_OWNER").toBe(true); + child.kill("SIGKILL"); await child.exited; + const result = f.operations.receipt("odysseus", id); + expect(result.state, "STORAGE_MAINTENANCE_CRASH_IS_UNKNOWN").toBe("unknown"); + expect(result.startedAt).not.toBeNull(); + expect(f.operations.start("odysseus", { id, operation: "pack-git" }).state).toBe("unknown"); + await f.operations.idle(); expect(f.calls, "STORAGE_MAINTENANCE_CRASH_NO_REPLAY").toBe(0); + } finally { child.kill("SIGKILL"); await child.exited; await f.close(); } +}); + +test("another database connection can cancel an admitted operation without claiming its effects rolled back", async () => { + const f = fixture(), secondDb = createUiDb(f.dbPath), second = createStorageMaintenance(secondDb, f.options); + let entered = false; + f.options.brain.storagePack = async request => { entered = true; await new Promise((_, reject) => { + const abort = () => reject(Error("owner interrupted")); request?.signal?.addEventListener("abort", abort, { once: true }); if (request?.signal?.aborted) abort(); + }); }; + try { + const id = "odysseus-cancel-001"; f.operations.start("odysseus", { id, operation: "pack-git" }); + for (let i = 0; i < 500 && !entered; i++) await Bun.sleep(10); + expect(entered).toBe(true); second.cancel("odysseus", id); await f.operations.idle(); + expect(second.receipt("odysseus", id).state, "STORAGE_MAINTENANCE_REMOTE_CANCEL_UNKNOWN").toBe("unknown"); + } finally { await second.close(); secondDb.close(); await f.close(); } +}); + +test("suspended pin refresh and partial scratch reports retain their owner's meaning", async () => { + const { ScratchPruneSuspended } = await import("../src/cron/scratch-prune"), { BrainMaintenanceError } = await import("../src/brain/client"); + const f = fixture(); + try { + f.options.scratch.run = async () => { throw new ScratchPruneSuspended("incomplete"); }; + f.operations.start("odysseus", { id: "odysseus-pins-0001", operation: "prune-scratch" }); await f.operations.idle(); + expect(f.operations.receipt("odysseus", "odysseus-pins-0001"), "STORAGE_PINS_SUSPENSION_RECEIPT").toMatchObject({ state: "refused", reason: "pins_refresh_incomplete", outcome: { suspended: true } }); + f.options.scratch.run = async () => { throw new BrainMaintenanceError("partial", JSON.stringify({ action: "prune", removed: [{ path: ".brain/scratch/removed.txt", bytes: 6 }], failed: [{ path: ".brain/scratch/kept.txt" }], bytes: 6, files: 1, pinned: { files: 1, bytes: 6 }, overageBytes: 2 }), 2); }; + f.operations.start("odysseus", { id: "odysseus-partial-01", operation: "prune-scratch" }); await f.operations.idle(); + expect(f.operations.receipt("odysseus", "odysseus-partial-01"), "STORAGE_PARTIAL_OWNER_REPORT_RETAINED").toMatchObject({ state: "partial", outcome: { removed: 1, failed: 1, pinnedFiles: 1, overageBytes: 2 } }); + } finally { await f.close(); } +}); + + +test("manual staging cancellation finishes its current acknowledgement and preserves the next item", async () => { + const f = fixture(), controller = new AbortController(), now = Date.now(); + const { SHARE_STAGING_DIR } = await import("@schlessera/brain-ui-sdk/protocol"); + const store = createInboxStore(f.db), targets: string[] = []; + for (const suffix of ["a", "b"]) { + const stagingId = `12345678-1234-1234-1234-123456789ab${suffix}`, target = join(f.brain, SHARE_STAGING_DIR, stagingId); + mkdirSync(target, { recursive: true }); writeFileSync(join(target, "chart.txt"), "Ithaca"); targets.push(target); + const item = store.ingest({ threadId: suffix, itemId: `share-${suffix}`, dedupKey: suffix, source: "share", stagingId, stakes: 2, expiresAt: now + 86_400_000 }).item; + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version, to: "claimed", leaseUntil: now + 60_000 }]); + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version + 1, to: "done" }]); + store.commit([{ kind: "item", item: { id: `cleanup-${suffix}`, dedupKey: `cleanup-${suffix}`, threadId: suffix, queue: "queue", type: "cleanup_pending", status: "ready", attempts: 0, maxAttempts: 3, version: 1, createdAt: now, updatedAt: now, expiresAt: Number.MAX_SAFE_INTEGER, payload: { stagingId } } }]); + } + const cleanup = createInboxCleanup(f.db, f.brain, { async afterRemove() { controller.abort(); } }); + try { + const result = await cleanup.run({ signal: controller.signal }); + expect(result).toMatchObject({ cleaned: 1, cancelled: true }); + expect(store.getItem("cleanup-a"), "STORAGE_CANCEL_ACKNOWLEDGES_COMMITTED_ITEM").toMatchObject({ status: "done" }); + expect(existsSync(targets[0]!)).toBe(false); + expect(existsSync(targets[1]!), "STORAGE_CANCEL_STOPS_NEXT_ITEM").toBe(true); + expect(store.getItem("cleanup-b")).toMatchObject({ status: "ready", attempts: 0 }); + } finally { await cleanup.close(); await f.close(); } +}); + +test("a failed durable per-item intent prevents staging removal", async () => { + const f = fixture(), now = Date.now(), stagingId = "12345678-1234-1234-1234-123456789abc"; + const { SHARE_STAGING_DIR } = await import("@schlessera/brain-ui-sdk/protocol"); + const target = join(f.brain, SHARE_STAGING_DIR, stagingId), store = createInboxStore(f.db); + mkdirSync(target, { recursive: true }); writeFileSync(join(target, "chart.txt"), "Ithaca"); + const item = store.ingest({ threadId: "harbor", itemId: "share", dedupKey: "share", source: "share", stagingId, stakes: 2, expiresAt: now + 86_400_000 }).item; + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version, to: "claimed", leaseUntil: now + 60_000 }]); + store.commit([{ kind: "transition", itemId: item.id, expectedVersion: item.version + 1, to: "done" }]); + store.commit([{ kind: "item", item: { id: "cleanup-harbor", dedupKey: "cleanup-harbor", threadId: "harbor", queue: "queue", type: "cleanup_pending", status: "ready", attempts: 0, maxAttempts: 3, version: 1, createdAt: now, updatedAt: now, expiresAt: Number.MAX_SAFE_INTEGER, payload: { stagingId } } }]); + f.db.exec("CREATE TRIGGER refuse_item_intent BEFORE UPDATE ON storage_maintenance_operations WHEN json_array_length(NEW.receipt, '$.outcome.items') > 0 BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); + try { + f.operations.start("odysseus", { id: "odysseus-intent-001", operation: "cleanup-staging" }); await f.operations.idle(); + expect(existsSync(target), "STORAGE_ITEM_INTENT_BEFORE_REMOVE").toBe(true); + expect(f.operations.receipt("odysseus", "odysseus-intent-001").state).toBe("unknown"); + expect(store.getItem("cleanup-harbor")).toMatchObject({ status: "claimed" }); + } finally { await f.close(); } +}); + + +test("bounded packing refuses a host-only boundary before probing repository code", async () => { + const f = fixture(), marker = join(f.root, "host-probe-entered"), bin = join(f.brain, "node_modules", ".bin", "brain"); + const { chmodSync } = await import("node:fs"); mkdirSync(join(f.brain, "node_modules", ".bin"), { recursive: true }); + writeFileSync(bin, `#!${process.execPath}\nawait Bun.write(${JSON.stringify(marker)},"unexpected host evaluation");console.log("0.44.0");\n`); chmodSync(bin, 0o700); + const boundary = { isolated: () => false, async run() { throw Error("worker unavailable"); } } as unknown as HostBoundary; + const client = createBrainClient({ brainPath: f.brain, boundary, exec: {} }); + try { + await expect(client.storagePack!()).rejects.toThrow(); + expect(existsSync(marker), "STORAGE_BOUNDED_PROBE_NEVER_RUNS_ON_HOST").toBe(false); + } finally { await f.close(); } +}); From ca56a84a83dc6f72bfae8baa90de21a068773c69 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 14:16:00 +0200 Subject: [PATCH 10/44] CONTRACT: feat(storage): add recoverable attachment removal --- .changeset/recoverable-storage-attachments.md | 7 + api-report/core.txt | 58 +++++ api-report/ui-sdk.txt | 118 +++++++++ api-report/ui-server.txt | 4 + docs/decisions/claude-code-runtime.md | 4 +- docs/decisions/http-api-boundary.md | 8 +- docs/decisions/scheduled-tasks.md | 2 +- docs/decisions/session-principals.md | 6 +- docs/http-api-coverage.md | 8 +- docs/http-api.md | 102 +++++++- docs/integration-contract/http.md | 9 + docs/integration-contract/package-api.md | 35 ++- .../plans/async-collaboration-requirements.md | 2 +- docs/plans/async-collaboration.md | 2 +- docs/plans/navigation-recovery.md | 2 +- docs/plans/storage-accounting.md | 2 +- docs/plans/storage-lifecycle.md | 2 +- packages/core/README.md | 8 + .../core/src/lib/storage-attachment-facts.ts | 111 ++++++++ packages/core/src/lib/storage-attachments.ts | 159 ++++++++++++ packages/core/src/storage.ts | 7 +- .../tests/storage-attachment-facts.test.ts | 20 ++ .../core/tests/storage-attachments.test.ts | 79 ++++++ packages/ui-sdk/src/protocol.ts | 1 + packages/ui-sdk/src/storage-removal.ts | 36 +++ packages/ui-server/README.md | 11 + .../migrations/052_storage_removal.sql | 9 + packages/ui-server/src/app.ts | 10 + packages/ui-server/src/brain/host-boundary.ts | 16 +- .../ui-server/src/brain/journal-accounting.ts | 5 +- .../ui-server/src/brain/journal-objects.ts | 9 +- .../ui-server/src/brain/maintenance-entry.ts | 7 +- .../ui-server/src/brain/maintenance-output.ts | 2 +- .../src/brain/native-journal-capture.ts | 22 +- packages/ui-server/src/brain/turn-journal.ts | 35 ++- .../ui-server/src/routes/storage-removal.ts | 41 +++ packages/ui-server/src/storage/maintenance.ts | 6 +- .../ui-server/src/storage/removal-facts.ts | 21 ++ packages/ui-server/src/storage/removal.ts | 237 ++++++++++++++++++ .../tests/fixtures/storage-removal-worker.ts | 26 ++ .../tests/helpers/storage-removal-fixture.ts | 32 +++ .../ui-server/tests/http-inventory.test.ts | 2 +- .../tests/http-supported-mounts.test.ts | 6 + .../tests/storage-removal-journal.test.ts | 29 +++ .../tests/storage-removal-recovery.test.ts | 49 ++++ .../tests/storage-removal-routes.test.ts | 51 ++++ .../tests/storage-removal-space.test.ts | 35 +++ .../tests/storage-removal-worker.test.ts | 32 +++ .../ui-server/tests/storage-removal.test.ts | 140 +++++++++++ 49 files changed, 1585 insertions(+), 40 deletions(-) create mode 100644 .changeset/recoverable-storage-attachments.md create mode 100644 packages/core/src/lib/storage-attachment-facts.ts create mode 100644 packages/core/src/lib/storage-attachments.ts create mode 100644 packages/core/tests/storage-attachment-facts.test.ts create mode 100644 packages/core/tests/storage-attachments.test.ts create mode 100644 packages/ui-sdk/src/storage-removal.ts create mode 100644 packages/ui-server/migrations/052_storage_removal.sql create mode 100644 packages/ui-server/src/routes/storage-removal.ts create mode 100644 packages/ui-server/src/storage/removal-facts.ts create mode 100644 packages/ui-server/src/storage/removal.ts create mode 100644 packages/ui-server/tests/fixtures/storage-removal-worker.ts create mode 100644 packages/ui-server/tests/helpers/storage-removal-fixture.ts create mode 100644 packages/ui-server/tests/storage-removal-journal.test.ts create mode 100644 packages/ui-server/tests/storage-removal-recovery.test.ts create mode 100644 packages/ui-server/tests/storage-removal-routes.test.ts create mode 100644 packages/ui-server/tests/storage-removal-space.test.ts create mode 100644 packages/ui-server/tests/storage-removal-worker.test.ts create mode 100644 packages/ui-server/tests/storage-removal.test.ts diff --git a/.changeset/recoverable-storage-attachments.md b/.changeset/recoverable-storage-attachments.md new file mode 100644 index 0000000000..5b142ff62f --- /dev/null +++ b/.changeset/recoverable-storage-attachments.md @@ -0,0 +1,7 @@ +--- +"@schlessera/brain": minor +"@schlessera/brain-ui-sdk": minor +"@schlessera/brain-ui-server": minor +--- + +Add human-owner attachment review, journal-backed removal and version-checked Undo with durable per-item receipts. Originals are verified and retained before effects; changed, protected, ignored and uncovered files are refused. Interrupted requests never replay, replacements are never overwritten, and retention expiry removes the Undo offer. Add the bounded core primitives and protocol contracts used by this application boundary. diff --git a/api-report/core.txt b/api-report/core.txt index 987e2a676a..b80cd01802 100644 --- a/api-report/core.txt +++ b/api-report/core.txt @@ -195,9 +195,19 @@ export "./queries" (./src/queries/index.ts) readVoiceVocabulary export "./storage" (./src/storage.ts) + STORAGE_ATTACHMENT_LIMITS + type StorageAttachmentFacts + StorageAttachmentRefusal + type StorageAttachmentVersion type StorageInventory type StorageReadOptions + inspectStorageAttachment + inspectStorageAttachmentFacts readStorageInventory + removeStorageAttachment + restoreStorageAttachment + storageAttachmentPath + storageAttachmentVacant export "./testing" (./src/testing/index.ts) type AgentRunnerContractHarness @@ -788,6 +798,13 @@ signatures (every public export and the types it is made of) const SCHEMA_VERSION: 15; SCRATCH_DIR (packages/core/src/lib/scratch.ts) const SCRATCH_DIR: ".brain/scratch"; + STORAGE_ATTACHMENT_LIMITS (packages/core/src/lib/storage-attachments.ts) + const STORAGE_ATTACHMENT_LIMITS: Readonly<{ + depth: 32; + fileBytes: number; + milliseconds: 15000; + pathBytes: 4096; + }>; STORAGE_LIMITS (packages/core/src/lib/storage-inventory.ts) const STORAGE_LIMITS: Readonly; ScratchFailure (packages/core/src/lib/scratch.ts) @@ -908,6 +925,35 @@ signatures (every public export and the types it is made of) evaluatedAt: string; verdicts: StatsTrend[]; } + StorageAttachmentFacts (packages/core/src/lib/storage-attachment-facts.ts) + export interface StorageAttachmentFacts { + fingerprint: string; + complete: boolean; + scannedFiles: number; + unchecked: string[]; + files: { + path: string; + git: "tracked" | "untracked" | "ignored" | "no-repository" | "unknown"; + references: { + source: string; + hash: string; + certainty: "resolved" | "possible"; + }[]; + truncated: boolean; + }[]; + } + StorageAttachmentRefusal (packages/core/src/lib/storage-attachments.ts) + export class StorageAttachmentRefusal extends Error { + constructor(readonly reason: "unsupported" | "protected" | "changed" | "too_large" | "occupied" | "unavailable"); + } + StorageAttachmentVersion (packages/core/src/lib/storage-attachments.ts) + export interface StorageAttachmentVersion { + hash: string; + bytes: number; + mode: number; + identity: string; + parents: string[]; + } StorageCapacity (packages/core/src/lib/storage-inventory.ts) export interface StorageCapacity { id: string; @@ -1397,6 +1443,10 @@ signatures (every public export and the types it is made of) export function getMarkdownFiles(root: string, taxonomy: Taxonomy): string[]; inertGeneratedText (packages/core/src/lib/generated-regions.ts) export function inertGeneratedText(text: string): string; + inspectStorageAttachment (packages/core/src/lib/storage-attachments.ts) + export function inspectStorageAttachment(root: string, path: string, signal?: AbortSignal): Promise; + inspectStorageAttachmentFacts (packages/core/src/lib/storage-attachment-facts.ts) + export function inspectStorageAttachmentFacts(root: string, paths: readonly string[], signal?: AbortSignal): StorageAttachmentFacts; isInScratch (packages/core/src/lib/scratch.ts) export function isInScratch(root: string, path: string): boolean; isWriteRefusal (packages/core/src/lib/scratch.ts) @@ -1477,6 +1527,8 @@ signatures (every public export and the types it is made of) export function readVoiceVocabulary(opts: Root & { limit: number; }): QueryResult; + removeStorageAttachment (packages/core/src/lib/storage-attachments.ts) + export function removeStorageAttachment(root: string, path: string, expected: StorageAttachmentVersion, signal?: AbortSignal, assertCurrent?: () => void): Promise; repoRelativePathSchema (packages/core/src/lib/config.ts) const repoRelativePathSchema: z.ZodString; resolveCompletionProvider (packages/core/src/lib/registry.ts) @@ -1485,6 +1537,8 @@ signatures (every public export and the types it is made of) export function resolveReranker(config?: RerankerSettings): Reranker | undefined; resolveWritable (packages/core/src/lib/safe-path.ts) export function resolveWritable(root: string, relOrAbs: string): string | null; + restoreStorageAttachment (packages/core/src/lib/storage-attachments.ts) + export function restoreStorageAttachment(root: string, path: string, expected: StorageAttachmentVersion, copy: (fd: number) => void, assertCurrent?: () => void): void; rewriteGeneratedRegion (packages/core/src/lib/generated-regions.ts) export function rewriteGeneratedRegion(raw: string, name: string, content: string, asOf: string): string | null; runAgentRunnerContract (packages/core/src/testing/agent-runners.ts) @@ -1511,6 +1565,10 @@ signatures (every public export and the types it is made of) frontmatter: string; body: string; }; + storageAttachmentPath (packages/core/src/lib/storage-attachments.ts) + export function storageAttachmentPath(path: string): boolean; + storageAttachmentVacant (packages/core/src/lib/storage-attachments.ts) + export function storageAttachmentVacant(root: string, path: string, expected: StorageAttachmentVersion): boolean; syncSkills (packages/core/src/lib/skills/sync.ts) export function syncSkills(ctx: SkillSources, opts?: SyncOptions): SyncResult; tagsConfigSchema (packages/core/src/lib/config.ts) diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index 28e4c21bd8..8bc960ad4a 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -319,6 +319,14 @@ export "." (./src/index.ts) type StorageOwner type StoragePage type StorageReferenceEvidence + type StorageRemovalCapability + type StorageRemovalFile + type StorageRemovalItem + type StorageRemovalReceipt + type StorageRemovalReview + type StorageRemovalReviewRequest + type StorageRemovalStartRequest + type StorageRemovalUndoRequest type StorageRoot type StorageSnapshot type StorageStatus @@ -786,6 +794,14 @@ export "./client" (./src/client/index.ts) type StorageOwner type StoragePage type StorageReferenceEvidence + type StorageRemovalCapability + type StorageRemovalFile + type StorageRemovalItem + type StorageRemovalReceipt + type StorageRemovalReview + type StorageRemovalReviewRequest + type StorageRemovalStartRequest + type StorageRemovalUndoRequest type StorageRoot type StorageSnapshot type StorageStatus @@ -1408,6 +1424,14 @@ export "./protocol" (./src/protocol.ts) type StorageOwner type StoragePage type StorageReferenceEvidence + type StorageRemovalCapability + type StorageRemovalFile + type StorageRemovalItem + type StorageRemovalReceipt + type StorageRemovalReview + type StorageRemovalReviewRequest + type StorageRemovalStartRequest + type StorageRemovalUndoRequest type StorageRoot type StorageSnapshot type StorageStatus @@ -2101,6 +2125,14 @@ export "./server" (./src/server/index.ts) type StorageOwner type StoragePage type StorageReferenceEvidence + type StorageRemovalCapability + type StorageRemovalFile + type StorageRemovalItem + type StorageRemovalReceipt + type StorageRemovalReview + type StorageRemovalReviewRequest + type StorageRemovalStartRequest + type StorageRemovalUndoRequest type StorageRoot type StorageSnapshot type StorageStatus @@ -9405,6 +9437,92 @@ signatures (every public export and the types it is made of) freshness: "unchecked"; unchecked: string[]; } + StorageRemovalCapability (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalCapability { + latest: StorageRemovalReceipt | null; + available: boolean; + reason: string | null; + limits: { + items: number; + fileBytes: number; + totalBytes: number; + milliseconds: number; + reviewMilliseconds: number; + receiptBytes: number; + receipts: number; + }; + } + StorageRemovalFile (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalFile { + path: string; + bytes: number | null; + hash: string | null; + eligible: boolean; + reason: string | null; + git: "tracked" | "untracked" | "ignored" | "no-repository" | "unknown"; + references: { + source: string; + certainty: "resolved" | "possible"; + }[]; + recovery: "requires-retention" | "retained" | "unavailable"; + } + StorageRemovalItem (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalItem { + path: string; + state: "not-started" | "applying" | "removed" | "restored" | "changed" | "protected" | "failed" | "unknown"; + reason: string | null; + undoAvailable: boolean; + undoReason: string | null; + } + StorageRemovalReceipt (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalReceipt { + review: StorageRemovalReview; + state: "review" | "waiting" | "running" | "complete" | "partial" | "refused" | "cancelled" | "unknown"; + requestedAt: number; + startedAt: number | null; + completedAt: number | null; + reason: string | null; + items: StorageRemovalItem[]; + before: StorageMaintenanceObservation | null; + after: StorageMaintenanceObservation | null; + index: "not-requested" | "complete" | "unavailable"; + commit: "not-requested"; + sync: "not-requested"; + retained: { + journal: "subject-to-retention"; + git: "unchanged-by-operation"; + backups: "not-measured"; + }; + } + StorageRemovalReview (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalReview { + id: string; + action: "remove" | "undo"; + sourceId: string | null; + fingerprint: string; + observedAt: number; + expiresAt: number; + files: StorageRemovalFile[]; + complete: boolean; + unchecked: string[]; + selectedFileBytes: number; + } + StorageRemovalReviewRequest (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalReviewRequest { + id: string; + paths: string[]; + } + StorageRemovalStartRequest (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalStartRequest { + id: string; + fingerprint: string; + } + StorageRemovalUndoRequest (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalUndoRequest { + id: string; + sourceId: string; + paths: string[]; + } StorageRoot (packages/ui-sdk/src/storage.ts) export interface StorageRoot extends StorageBytes { id: string; diff --git a/api-report/ui-server.txt b/api-report/ui-server.txt index 4b29b676bc..eeb1772223 100644 --- a/api-report/ui-server.txt +++ b/api-report/ui-server.txt @@ -1320,6 +1320,7 @@ signatures (every public export and the types it is made of) details(sessionId: string, turnId: string): TurnJournalDetails | null; finish(turnId: string): Promise; finishReserved(turnId: string): void; + hasRetainedVersion(turnId: string, selected: JournalVersion): boolean; idle(): Promise; isHeld(): boolean; observe(): Promise; @@ -1332,7 +1333,9 @@ signatures (every public export and the types it is made of) after(completed: boolean): void; before(): void; }): () => boolean; + removalCoverage(paths: readonly string[], signal?: AbortSignal, absent?: boolean): Promise; reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]): void; + retainBefore(turnId: string, path: string, signal?: AbortSignal): Promise; summary: (turnId: string) => TurnJournalSummary; version(sessionId: string, turnId: string, sequence: number, side: "after" | "before"): null | { bytes: (maxBytes: number) => Buffer; @@ -1342,6 +1345,7 @@ signatures (every public export and the types it is made of) bytes: number; }; }; + withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): T; withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): T; writeOperation(write: () => T): T; }; diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index 0dd62e9cca..d1ac334daa 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -64,7 +64,7 @@ next to it. of this binary. The only version probe in the server is for the `brain` CLI (`Probe the selected content CLI`, `packages/ui-server/src/brain/client.ts:121-187`, called at - `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:386`). + `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:388`). `brain doctor` runs `claude mcp list` from `PATH` (`which("claude")`, `packages/core/src/cli/commands/doctor.ts:495-499`) — the user's own Claude Code on their own machine, to check the MCP registration, not the server's @@ -212,7 +212,7 @@ unnecessary. `node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs`. The boot probe below turns either into a refusal to start, for the same reason a missing backend refuses to boot (`A missing (or unrecognized) agent backend`, - `packages/ui-server/src/app.ts:331-336`). + `packages/ui-server/src/app.ts:333-338`). - **The published range is still a caret, and that bounds what this repo can guarantee.** `@schlessera/brain-backend-claude` depends on `^0.3.241`, and a host resolves it in its own lockfile. A host can bump the SDK — and so the diff --git a/docs/decisions/http-api-boundary.md b/docs/decisions/http-api-boundary.md index d978c25eca..ce161725fa 100644 --- a/docs/decisions/http-api-boundary.md +++ b/docs/decisions/http-api-boundary.md @@ -46,11 +46,11 @@ explicit supported rows, not silently excluded by this distinction. The source audit on 2026-09-30 used `e0875269b21b707482545be9e94901acb897893b`. The public health mount precedes auth -(`createHealthRoutes`, `packages/ui-server/src/app.ts:731`), the API guard follows -public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:751`), and the -WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:910`). +(`createHealthRoutes`, `packages/ui-server/src/app.ts:733`), the API guard follows +public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:753`), and the +WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:919`). Static serving is conditional (`if (options.staticRoot)`, -`packages/ui-server/src/app.ts:930-951`). That order establishes access; +`packages/ui-server/src/app.ts:939-960`). That order establishes access; it does not select compatibility status. A hermetic real-app launch, with temporary brain root, in-memory database, diff --git a/docs/decisions/scheduled-tasks.md b/docs/decisions/scheduled-tasks.md index 208b8ee262..8f17fdf144 100644 --- a/docs/decisions/scheduled-tasks.md +++ b/docs/decisions/scheduled-tasks.md @@ -23,7 +23,7 @@ Source inspected at `b5c30c180df5584584f261b446c4d440380d2441`: (`else if (item.status === "scheduled"`, `packages/ui-server/src/inbox/actions.ts:188-189`). The runtime ticks every minute (`INBOX_TICK_MS`, `packages/ui-server/src/inbox/runtime.ts:11-13`). - Production app wiring supplies recovery and budgets, without a dispatcher - (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:403-419`). + (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:405-421`). A scripted nonempty ready-item test still reports no claim, attempt or Activity root. That proves disabled wiring, not safe execution of a schedule. - Headless plumbing checks backend support, a usable principal, lifetime and diff --git a/docs/decisions/session-principals.md b/docs/decisions/session-principals.md index f9ba7e73f8..bcae7228ba 100644 --- a/docs/decisions/session-principals.md +++ b/docs/decisions/session-principals.md @@ -96,9 +96,9 @@ Both reviewers verified every row below against the source. | Cookie verify | `auth.ts:265-296` | strict `^(\d+)\.(\d+)$`, safe integers, future/expired rejection, `cookieEpoch === sessionsEpoch(db)` | | Global epoch | `auth.ts:302-318` | `settings` row; missing = 0; non-integer **throws** | | Global revoke | `auth.ts:321-330` | `setSetting` (overflow-guarded) + `clients.closeAll(1008, …)` | -| Auth routes | `authRoutes`, `app.ts:737` | mounted **before** the guard at `authGuard(authMode`, `app.ts:751`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | -| Passkey management | `passkeyManagementRoutes`, `app.ts:761` | mounted **after** the guard ("Mount AFTER the auth guard") | -| WS guard | `"/ws"`, `app.ts:910-927` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | +| Auth routes | `authRoutes`, `app.ts:739` | mounted **before** the guard at `authGuard(authMode`, `app.ts:753`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | +| Passkey management | `passkeyManagementRoutes`, `app.ts:763` | mounted **after** the guard ("Mount AFTER the auth guard") | +| WS guard | `"/ws"`, `app.ts:919-936` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | | WS upgrade | `ws/connection.ts:262-264` | `createWsUpgrade` ignores the request context | | WS admission | `ws/connection.ts:84` | `clients.add(ws)` — the socket's identity is unknown | | Turn record | `ws/run-session.ts:134-142` | recorder built from `{turnId, sessionId, billing}`; `RunningTurn` (`ws/turns.ts:55-77`) has no actor | diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index 23cc396b3f..5efc69bfa6 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:122-155`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:128-161`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -28,6 +28,12 @@ row's real mounting check. They do not establish mounting by themselves. | Supported operation | Mounted receipt | Detailed checks | | --- | --- | --- | +| `GET /api/storage/removals` | `mounted supported handler: GET /api/storage/removals` | SM | +| `POST /api/storage/removals/reviews` | `mounted supported handler: POST /api/storage/removals/reviews` | SM | +| `POST /api/storage/removals/undo-reviews` | `mounted supported handler: POST /api/storage/removals/undo-reviews` | SM | +| `POST /api/storage/removals` | `mounted supported handler: POST /api/storage/removals` | SM | +| `GET /api/storage/removals/:id` | `mounted supported handler: GET /api/storage/removals/:id` | SM | +| `DELETE /api/storage/removals/:id` | `mounted supported handler: DELETE /api/storage/removals/:id` | SM | | `GET /api/storage/maintenance` | `mounted supported handler: GET /api/storage/maintenance` | SM | | `POST /api/storage/maintenance` | `mounted supported handler: POST /api/storage/maintenance` | SM | | `GET /api/storage/maintenance/:id` | `mounted supported handler: GET /api/storage/maintenance/:id` | SM | diff --git a/docs/http-api.md b/docs/http-api.md index fdb35eff2a..75a4d3482b 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 145 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 151 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -26,6 +26,12 @@ client code has a gap. Source owners are listed after the table. | Method | Path | Class | Purpose | Consumers, existing promise and reason | | --- | --- | --- | --- | --- | +| GET | `/api/storage/removals` | S | Read removal capability and latest receipt | Current human-owner authority; exact canonical attachments only. | +| POST | `/api/storage/removals/reviews` | S | Review exact attachment paths | Bound versions, consequences and recovery coverage before confirmation. | +| POST | `/api/storage/removals/undo-reviews` | S | Review retained originals for Undo | New identity and current absent-target checks. | +| POST | `/api/storage/removals` | S | Confirm one reviewed selection | Durable intent and verified originals precede unlink. | +| GET | `/api/storage/removals/:id` | S | Read a removal or Undo receipt | Principal-bound readback never replays effects. | +| DELETE | `/api/storage/removals/:id` | S | Cancel remaining selected items | Current acknowledgement finishes; completed effects remain. | | GET | `/api/storage/maintenance` | S | Read approved maintenance rows and latest receipts | Current human-owner authority; three fixed owners only. | | POST | `/api/storage/maintenance` | S | Admit one identified maintenance operation | Durable owner-bound identity and credential-free worker execution. | | GET | `/api/storage/maintenance/:id` | S | Read the original operation receipt | Readback never replays an effect. | @@ -178,7 +184,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 145 declared endpoints are mounted regardless of backend, renderer or +All 151 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1367,3 +1373,95 @@ Packing alone requires content core 0.44.0 or newer. Older installations produce an explicit refusal; the ordinary server CLI minimum remains 0.42.0. Hosted CLI operations use only the qualified credential-free maintenance worker; unsupported hosts do not fall back to executing repository code on the host. + +## Recoverable attachment removal (additive, #2012) + +The six `/api/storage/removals` operations require current owner/ambient human +access and return private, no-store responses. Missing authentication is 401; +delegated callers are 403. Receipts belong to their initiating principal and an +unknown or another principal's identity returns 404. Origin protections apply +to writes; JSON is mandatory for POST. No agent tool gains deletion authority. + +GET returns SDK `StorageRemovalCapability`: `available`, `reason`, fixed limits +and `latest` receipt or null. Unsupported native recovery/worker hosts report +unavailable. POST `/reviews` accepts exactly `{id, paths}` and POST +`/undo-reviews` exactly `{id, sourceId, paths}`. IDs use 16–96 ASCII letters, +digits, `_` or `-`; at most 32 distinct canonical relative paths, 4096 bytes per +path, 32 components, 64 MiB per file and 128 MiB of eligible file bytes are +admitted. Review bodies are bounded to 140,000 bytes. Paths do not choose roots, +executables, commands, sidecars or a wider scope. One review per coordinator +runs at a time, with a 60-second deadline and a ten-minute confirmation window. + +Review returns a durable `StorageRemovalReceipt` in `review` state. Its review +contains the identity, action, optional source identity, opaque fingerprint, +observation/expiry times, selected logical file bytes, unchecked scopes and +per-file eligibility/reason, raw hash/size, Git state, current reference evidence +and recovery state. File bytes are not promised freed space. Originals are +`requires-retention` until execution actually retains them; Undo reviews say +`retained` only for an available original. Excluded targets remain visible. + +Eligibility is limited to ordinary single-link non-executable attachments with +supported media, PDF, office/archive or ebook suffixes. Hidden paths, policies, +scripts, workspaces, configured other-owner stores, symlinks, hard links and +nested mounts remain ineligible. Unknown protection or current recovery coverage +refuses effects. Ordinary journal exclusions are preserved, including ignored +files. Filename eligibility does not establish document validity or safe preview. + +The consequence worker scans at most 10,000 directory entries, 2000 Markdown +files, 8 MiB of Markdown overall and 512 KiB per file; it records up to 200 +reference sources per selected file. Git subprocesses have two-second/1 MiB +bounds; the enclosing credential-free worker has a 30-second/256 KiB output +bound. It does not initialize brain config and discards every output, including +incidental Git writes. Current parsed Markdown references are independent of +index freshness. Retained conversations, external consumers, hidden/symlinked +content, other workspaces and unsupported link syntax remain unchecked. Incomplete +local evidence is ineligible; absence of observed links never proves no consumers. + +POST at `/api/storage/removals` accepts `{id, fingerprint}` in at most 4096 bytes +and returns 202. It confirms exactly that review; mismatched or expired reviews +are 409. Repeating an admitted identity reads its receipt without repeating any +file effect. Removal and Undo serialize across processes through the existing +exclusive writer gate and brain/Git locks. Active turns finish before validation; +nested shared maintenance can still finish while removal waits. Current +principal authority, selected versions, topology, protection, ordinary retention +coverage and consequences are rechecked after admission and before effects. +Unknown external writers outside the managed admission system are not coordinated +by these locks; this is not an arbitrary filesystem cleanup service. + +Verified before-bytes use the existing charged native journal/CAS. The original +and durable item intent precede unlink; admission, quota or ENOSPC refusal leaves +the source intact. No new ignore override, retention policy, Git-history removal, +reference rewriting, sidecar cascade, commit or push is introduced. Undo gets a +new review/identity and uses retained bytes only while their owner still provides +them. It checks the reviewed parent chain and creates only an absent destination; +a replacement is never overwritten. A failed copy removes only its own incomplete +inode, preserving the retained original. + +GET `/:id` returns the original receipt. DELETE `/:id` requests cancellation and +returns 202; later items stop while the current critical acknowledgement finishes. +Receipts distinguish `not-started`, `applying`, `removed`, `restored`, `changed`, +`protected`, `failed` and `unknown` items. Operation states are `review`, `waiting`, +`running`, `complete`, `partial`, `refused`, `cancelled` and `unknown`. A killed +owner or lost acknowledgement remains explicit uncertainty, never inferred +success or a replay. Confirmed successes survive mixed batches. `undoAvailable` +and `undoReason` are recomputed from the original's current retention ownership; +expired or unavailable detail visibly removes the Undo offer. + +Before/after observations separately record current content, journal, Git, +operational database and volume availability, with timestamp, basis, coverage and +concurrent-writer qualification. `index` is separately `not-requested`, `complete` +or `unavailable`; refresh publication is restricted to the fixed disposable +`brain.db` files, so configuration code cannot publish changes to unselected +content through this refresh. Commit/sync remain `not-requested`, Git history unchanged by +this operation and backups unmeasured. Deletion can initially increase allocation. + +The operation's queue and effects have a five-minute cancellation deadline; +index refresh and after-measurement each have an independent 36-second bound. +Individual raw hashes have a 15-second deadline. Kernel uninterruptible I/O cannot +guarantee a hard return time. Public receipt and private version metadata each +reserve 256 KiB through the existing journal allocation owner. The per-brain +admission ceiling is 128 identities; reaching it refuses new reviews with 503 and +retains existing receipts. This is an admission bound, not a new retention expiry. +Invalid requests are 400, oversized bodies 413, wrong content type 415, contention +409 and unavailable evaluation/recovery/admission 503 or an explicit refused/failed +receipt. The Storage pane is delivered by its complete UI issue. diff --git a/docs/integration-contract/http.md b/docs/integration-contract/http.md index 93874f9ee0..15fe646a92 100644 --- a/docs/integration-contract/http.md +++ b/docs/integration-contract/http.md @@ -501,3 +501,12 @@ Hierarchy pages separate per-subtree object-union `totals` from disjoint sibling `area`, preserve alias names and unknown coverage, and bind cursors to the tree operation and its search/order/ownership criteria. Projection uses retained scan evidence only. Entry search/order is additive; absent options preserve ordering. + +### Recoverable attachment removal (additive, #2012) + +The six [attachment removal operations](../http-api.md#recoverable-attachment-removal-additive-2012) +and SDK `StorageRemoval*` review/capability/receipt types are additive contracts. +They bind current human-owner authority, exact versioned selections and durable +identities to charged retained originals. Unknown outcomes never authorize +replay. Undo is separately reviewed, cannot overwrite a replacement, and expires +with actual journal detail. Existing Markdown application semantics are unchanged. diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index 55f2860750..8807e0b541 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3214,7 +3214,7 @@ brain-specific live/archive leaves; Pi supplies its already resolved session roo ### Host Storage reader (additive, #2010) `@schlessera/brain/storage` supports host-shell authors who need bounded live -physical metadata without evaluating a content repository. Its sole operation +physical metadata without evaluating a content repository. The operation `readStorageInventory({brainPath, gitRoots?, signal?})` returns a promise of the version-1 Storage inventory envelope, with the existing 100,000-entry/30-second limits and isolated/reaped filesystem worker. `brainPath` is absolute; @@ -3222,7 +3222,7 @@ limits and isolated/reaped filesystem worker. `brainPath` is absolute; Invalid roots reject before traversal. The host must enforce user authorization; this is not an agent-facing filesystem browser. -This entry executes neither Git nor the brain CLI, and loads no repository +This inventory operation executes neither Git nor the brain CLI, and loads no repository configuration/providers. It reads only fixed Git location markers (`.git`, `commondir`, `objects/info/alternates`) through bounded non-followed descriptors, approving external stores before reading their markers. It supports ordinary @@ -3252,3 +3252,34 @@ independent store/capacity measurements in the [maintenance HTTP contract](../http-api.md#storage-maintenance-additive-2011). They do not add an agent tool or generic executor. Existing automatic retention and owner APIs retain their previous defaults. + +### Selected attachment application (additive, 0.44.0, #2012) + +`@schlessera/brain/storage` additionally exports `STORAGE_ATTACHMENT_LIMITS`, +`StorageAttachmentRefusal`, `storageAttachmentPath`, `inspectStorageAttachment`, +`removeStorageAttachment`, `restoreStorageAttachment`, `storageAttachmentVacant`, +`inspectStorageAttachmentFacts`, and the `StorageAttachmentVersion` and +`StorageAttachmentFacts` types. The inventory reader keeps its existing contract. + +Raw-file operations bind an absolute canonical Linux root, relative path and +exact descriptor-derived version (hash, size, mode, inode/stamp and parent chain). +They refuse symlinks, aliases, nested mounts and executable/protected targets. +The caller must own exclusive writer coordination, current human authorization, +verified durable recovery and intent before calling removal. Its optional final +`assertCurrent` callback revalidates authority immediately before unlink or restore; +restoration uses a caller-leased verified original and never overwrites a present +entry. These primitives neither provide a journal nor change its retention policy. +Raw hashing supports cancellation and bounded file size/time. + +`inspectStorageAttachmentFacts(root, paths, signal?)` evaluates Git and bounded +current Markdown. Hosted consumers must run it inside their qualified maintenance +worker and discard outputs. It is not a privileged host-side Git reader, and its +unchecked scopes prevent interpreting an empty reference list as proof of no +consumers. Core semantics have no dependency on the UI database. Server contracts +and bounds are defined by the [removal HTTP contract](../http-api.md#recoverable-attachment-removal-additive-2012). + +`@schlessera/brain-ui-sdk/protocol` exports `StorageRemovalReviewRequest`, +`StorageRemovalStartRequest`, `StorageRemovalUndoRequest`, `StorageRemovalFile`, +`StorageRemovalReview`, `StorageRemovalItem`, `StorageRemovalReceipt` and +`StorageRemovalCapability`. They contain relative paths and opaque identities, +never private recovery roots, executable choices or credentials. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index d7991ff58f..289093c5cc 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -226,7 +226,7 @@ Stated before the requirements because five of them derive from it. that is the shape to copy, including its `close()` lifecycle. - R19. **The cron backstop has independent authorization before the general guard.** Mount it on the existing listener before - (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:751`). + (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:753`). Authorize a boot-minted ephemeral token, rotated each boot and stored in a 0600 runtime file, with the actual socket address as an additional check. Proxy headers cannot authorize it. The poke succeeds in every auth mode diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 1b82629178..9f14586013 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -131,7 +131,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. owns manual triggers/history; it does not tick the inbox. - **Two connections:** (`export function createUiDb(`, `packages/ui-server/src/db/client.ts:25-37`) sets WAL, foreign keys and a 5-second busy timeout. Claims are immediate transactions. -- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:751`) follows public routes; +- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:753`) follows public routes; (`export function authGuard(`, `packages/ui-server/src/middleware/auth.ts:189-249`) binds principals in each auth mode. An internal poke needs independent token authorization before this guard. - **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:271-365`). Timeout unwind is diff --git a/docs/plans/navigation-recovery.md b/docs/plans/navigation-recovery.md index ec32eccaf9..4ff0492e26 100644 --- a/docs/plans/navigation-recovery.md +++ b/docs/plans/navigation-recovery.md @@ -397,7 +397,7 @@ implementation. Any PR moving these lines owns their citation repairs. - Activity resolution: (`.get("/activity/runs", (c) => {`, `packages/ui-server/src/routes/activity.ts:142-275`). -- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:751`). +- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:753`). - Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:878-922`). diff --git a/docs/plans/storage-accounting.md b/docs/plans/storage-accounting.md index a4c0718c27..6bf477cd6d 100644 --- a/docs/plans/storage-accounting.md +++ b/docs/plans/storage-accounting.md @@ -67,7 +67,7 @@ Source anchors, checked against the reviewed commit: The completed #94 and #398 provide existing stats/maintenance foundations. - C: snapshot fields (`snapshotOf`, `packages/core/src/lib/stats-history.ts:132-172`), thinning (`thin`, `packages/core/src/lib/stats-history.ts:232-244`). -- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:257-292`), +- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:259-294`), WAL store (`createUiDb`, `packages/ui-server/src/db/client.ts:25-37`), and private locations (`statePaths`, `packages/ui-server/src/brain/operational-state.ts:107-113`). - E: live/archive paths (`transcriptPaths`, `packages/ui-backend-claude/src/transcript-archive.ts:29-38`). diff --git a/docs/plans/storage-lifecycle.md b/docs/plans/storage-lifecycle.md index 2e0a8fc0b1..d53e7cab6b 100644 --- a/docs/plans/storage-lifecycle.md +++ b/docs/plans/storage-lifecycle.md @@ -88,7 +88,7 @@ Source foundations for the matrix: Retention is applied at admission (`begin(turnId: string`, `packages/ui-server/src/brain/turn-journal.ts:236-264`). Object copies reserve temporary bytes before writing - (`async putFileAsync`, `packages/ui-server/src/brain/journal-objects.ts:66-118`). + (`async putFileAsync`, `packages/ui-server/src/brain/journal-objects.ts:66-121`). - Turn-temp purge has its own scope and measured result (`export function purgeTurnTemp`, `packages/ui-sdk/src/server/worker-turn-temp.ts:412-494`), and tool validation/recording diff --git a/packages/core/README.md b/packages/core/README.md index a5556c393d..d41cb1a670 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -80,3 +80,11 @@ defines resource bounds, Git marker support, unknown evidence and cancellation. without expiring history or refs. Its [qualified result](../../docs/integration-contract/cli.md#storage-packing-additive-0440-2011) reports refusal/partial outcomes and workload-dependent temporary headroom. Governed hosts invoke it through their credential-free maintenance worker. + +The same Storage entry exports narrow raw attachment inspection/removal/restore +primitives and bounded consequence inspection. Their [caller obligations and +limits](../../docs/integration-contract/package-api.md#selected-attachment-application-additive-0440-2012) +are part of the contract: callers own exclusive coordination, current authority +and durable verified recovery before removal. Consequence inspection evaluates +Git and belongs in a credential-free worker in hosted consumers. It does not +change the metadata-only inventory reader or add an unjournaled delete command. diff --git a/packages/core/src/lib/storage-attachment-facts.ts b/packages/core/src/lib/storage-attachment-facts.ts new file mode 100644 index 0000000000..996eb12c50 --- /dev/null +++ b/packages/core/src/lib/storage-attachment-facts.ts @@ -0,0 +1,111 @@ +/** Bounded consequence inspection. Git evaluation must run inside the caller's maintenance worker. */ +import { closeSync, constants, fstatSync, lstatSync, openSync, opendirSync, readFileSync, readSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { posix } from "node:path"; +import { unified } from "unified"; +import remarkParse from "remark-parse"; +import { extractWikiLinks } from "./indexer/links.js"; +import { storageAttachmentPath } from "./storage-attachments.js"; + +export interface StorageAttachmentFacts { + fingerprint: string; + complete: boolean; + scannedFiles: number; + unchecked: string[]; + files: { path: string; git: "tracked" | "untracked" | "ignored" | "no-repository" | "unknown"; references: { source: string; hash: string; certainty: "resolved" | "possible" }[]; truncated: boolean }[]; +} +const DIRECTORY = constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW; +const at = (fd: number, name: string) => `/proc/self/fd/${fd}/${name}`; +const volume = (fd: number) => readFileSync(`/proc/self/fdinfo/${fd}`, "utf8").match(/^mnt_id:\s*(\d+)$/m)?.[1]; +type Node = { type: string; url?: string; identifier?: string; children?: Node[] }; +export function inspectStorageAttachmentFacts(root: string, paths: readonly string[], signal?: AbortSignal): StorageAttachmentFacts { + if (paths.length < 1 || paths.length > 32 || new Set(paths).size !== paths.length || paths.some(path => !storageAttachmentPath(path))) throw Error("Invalid attachment review selection."); + const files: StorageAttachmentFacts["files"] = paths.map(path => ({ path, git: "unknown", references: [], truncated: false })); + const proof: string[] = [], started = Date.now(); let scannedFiles = 0, bytes = 0, entries = 0, complete = true; + const deadline = () => { signal?.throwIfAborted(); if (Date.now() - started > 10_000) throw Error("Review time bound exceeded."); }; + const git = (...args: string[]) => { + deadline(); const child = Bun.spawnSync(["git", "-C", root, ...args], { stdin: "ignore", stdout: "pipe", stderr: "pipe", timeout: 2000, maxBuffer: 1_048_576 }); + return { code: child.exitCode, text: child.stdout.toString(), error: child.stderr.toString() }; + }; + let repository = false; + try { + const status = git("rev-parse", "--is-inside-work-tree"); + repository = status.code === 0 && status.text.trim() === "true"; + if (!repository && !/not a git repository/i.test(status.error)) throw Error("Git observation unavailable."); + for (const file of files) { + if (!repository) { file.git = "no-repository"; continue; } + const tracked = git("ls-files", "--error-unmatch", "--", file.path); + if (tracked.code === 0) { file.git = "tracked"; continue; } + if (tracked.code !== 1) continue; + const ignored = git("check-ignore", "--quiet", "--", file.path); + file.git = ignored.code === 0 ? "ignored" : ignored.code === 1 ? "untracked" : "unknown"; + } + } catch { complete = false; } + const tree = unified().use(remarkParse); + function references(source: string, content: string, hash: string) { + const ast = tree.parse(content) as Node, definitions = new Map(), urls: string[] = [], pending = [ast], refs: string[] = []; + while (pending.length) { + const node = pending.pop()!; + if (node.type === "definition" && node.identifier && node.url) definitions.set(node.identifier.toLowerCase(), node.url); + else if (["link", "image"].includes(node.type) && node.url) urls.push(node.url); + else if (["linkReference", "imageReference"].includes(node.type) && node.identifier) refs.push(node.identifier.toLowerCase()); + if (node.children) pending.push(...node.children); + } + for (const ref of refs) { const url = definitions.get(ref); if (url) urls.push(url); } + const resolved = new Set(); + for (const url of urls) { + if (/^[a-z][a-z\d+.-]*:/i.test(url) || url.startsWith("//")) continue; + try { const path = decodeURIComponent(url.split(/[?#]/)[0]!); resolved.add(path.startsWith("/") ? posix.normalize(path.slice(1)) : posix.normalize(posix.join(posix.dirname(source), path))); } + catch { complete = false; } + } + const wiki = extractWikiLinks(content); + for (const file of files) { + const direct = resolved.has(file.path) || wiki.includes(file.path), possible = wiki.some(target => target === posix.basename(file.path)); + if (!direct && !possible) continue; + if (file.references.length === 200) { file.truncated = true; complete = false; continue; } + file.references.push({ source, hash, certainty: direct ? "resolved" : "possible" }); + } + } + const rootFd = openSync(root, DIRECTORY), rootVolume = volume(rootFd); + if (!rootVolume) { closeSync(rootFd); throw Error("Review mount identity unavailable."); } + function walk(directory: number, prefix: string, depth: number) { + if (depth > 32 || entries > 10_000) { complete = false; return; } + const names: string[] = [], stream = opendirSync(at(directory, ".")); + try { + for (;;) { + deadline(); const entry = stream.readSync(); if (!entry) break; + if (++entries > 10_000) { complete = false; break; } + names.push(entry.name); + } + } finally { stream.closeSync(); } + for (const name of names.sort()) { + deadline(); + if (name.startsWith(".") || ["node_modules", "workspaces"].includes(name)) continue; + const path = prefix ? `${prefix}/${name}` : name, info = lstatSync(at(directory, name)); + if (info.isDirectory()) { + let child: number | undefined; + try { child = openSync(at(directory, name), DIRECTORY); if (volume(child) !== rootVolume) { complete = false; continue; } walk(child, path, depth + 1); } + catch { complete = false; } finally { if (child !== undefined) closeSync(child); } + } else if (name.endsWith(".md")) { + if (!info.isFile() || info.nlink !== 1 || info.size > 512 * 1024 || scannedFiles === 2000 || bytes + info.size > 8 * 1024 * 1024) { complete = false; continue; } + let fd: number | undefined; + try { + fd = openSync(at(directory, name), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + if (volume(fd) !== rootVolume) { complete = false; continue; } + const before = fstatSync(fd, { bigint: true }); if (!before.isFile() || before.size !== BigInt(info.size)) { complete = false; continue; } + const buffer = Buffer.alloc(info.size + 1); let offset = 0; + while (offset < buffer.length) { deadline(); const length = readSync(fd, buffer, offset, Math.min(64 * 1024, buffer.length - offset), offset); if (!length) break; offset += length; } + const raw = buffer.subarray(0, offset), after = fstatSync(fd, { bigint: true }); + if (raw.length !== info.size || before.ino !== after.ino || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) { complete = false; continue; } + const hash = createHash("sha256").update(raw).digest("hex"); scannedFiles++; bytes += raw.length; + proof.push(`${path}:${hash}`); references(path, new TextDecoder("utf-8", { fatal: true }).decode(raw), hash); + } catch { complete = false; } finally { if (fd !== undefined) closeSync(fd); } + } + } + } + try { walk(rootFd, "", 0); } catch { complete = false; } finally { closeSync(rootFd); } + signal?.throwIfAborted(); + const unchecked = ["retained conversations", "external consumers", "other workspaces", "hidden content", "symlinked content", "unsupported link syntax"]; + const fingerprint = createHash("sha256").update(JSON.stringify({ files, proof, complete, unchecked })).digest("hex"); + return { fingerprint, complete, scannedFiles, unchecked, files }; +} diff --git a/packages/core/src/lib/storage-attachments.ts b/packages/core/src/lib/storage-attachments.ts new file mode 100644 index 0000000000..c3e784d3a2 --- /dev/null +++ b/packages/core/src/lib/storage-attachments.ts @@ -0,0 +1,159 @@ +/** Narrow raw attachment operations. Callers own approval, retained recovery and writer coordination. */ +import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, openSync, readFileSync, readSync, realpathSync, unlinkSync } from "node:fs"; +import * as fs from "node:fs"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; + +export const STORAGE_ATTACHMENT_LIMITS = Object.freeze({ fileBytes: 64 * 1024 * 1024, pathBytes: 4096, depth: 32, milliseconds: 15_000 }); +export interface StorageAttachmentVersion { hash: string; bytes: number; mode: number; identity: string; parents: string[] } +export class StorageAttachmentRefusal extends Error { + constructor(readonly reason: "unsupported" | "protected" | "changed" | "too_large" | "occupied" | "unavailable") { super(reason); } +} +const DIRECTORY = constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW; +const FILE = constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW; +const at = (fd: number, name: string) => `/proc/self/fd/${fd}/${name}`; +const read = promisify(fs.read); +const EXTENSIONS = /\.(?:pdf|png|jpe?g|gif|webp|avif|heic|tiff?|bmp|ico|mp3|m4a|wav|ogg|flac|aac|mp4|webm|mov|mkv|avi|zip|7z|gz|bz2|xz|docx|xlsx|pptx|odt|ods|odp|epub)$/i; + +export function storageAttachmentPath(path: string): boolean { + const parts = path.split("/"), folded = parts.map(part => part.normalize("NFKC").toLowerCase()); + return Buffer.byteLength(path) <= STORAGE_ATTACHMENT_LIMITS.pathBytes && parts.length <= STORAGE_ATTACHMENT_LIMITS.depth + && !/[\\\u0000-\u001f\u007f]/u.test(path) && parts.every(part => !!part) && folded.every(part => !part.startsWith(".")) + && !folded.some(part => ["node_modules", "workspaces", "scripts"].includes(part)) + && !(folded[0] === "context" && folded[1] === "policies") && EXTENSIONS.test(path); +} +function mount(fd: number): string { + const id = readFileSync(`/proc/self/fdinfo/${fd}`, "utf8").match(/^mnt_id:\s*(\d+)$/m)?.[1]; + if (!id) throw new StorageAttachmentRefusal("unsupported"); + return id; +} +const identity = (fd: number) => { const s = fstatSync(fd, { bigint: true }); return [s.dev, s.ino, s.mode, mount(fd)].join(":"); }; +const stamp = (fd: number) => { const s = fstatSync(fd, { bigint: true }); return [identity(fd), s.size, s.mtimeNs, s.ctimeNs, s.nlink].join(":"); }; +function openParent(root: string, path: string) { + if (process.platform !== "linux") throw new StorageAttachmentRefusal("unsupported"); + if (!storageAttachmentPath(path)) throw new StorageAttachmentRefusal("protected"); + if (resolve(root) !== root || realpathSync(root) !== root) throw new StorageAttachmentRefusal("changed"); + const fds: number[] = [], chain: { parent: number; name: string; child: number }[] = []; + try { + const first = openSync(root, DIRECTORY); fds.push(first); + const parts = path.split("/"), name = parts.pop()!, firstMount = mount(first); + let parent = first; + for (const part of parts) { + const child = openSync(at(parent, part), DIRECTORY); fds.push(child); + if (mount(child) !== firstMount) throw new StorageAttachmentRefusal("protected"); + chain.push({ parent, name: part, child }); parent = child; + } + const parents = fds.map(identity); + const verify = () => { + const live = lstatSync(root, { bigint: true }), held = fstatSync(first, { bigint: true }); + if (!live.isDirectory() || live.dev !== held.dev || live.ino !== held.ino) throw new StorageAttachmentRefusal("changed"); + for (const entry of chain) { + const named = lstatSync(at(entry.parent, entry.name), { bigint: true }), opened = fstatSync(entry.child, { bigint: true }); + if (!named.isDirectory() || named.dev !== opened.dev || named.ino !== opened.ino) throw new StorageAttachmentRefusal("changed"); + } + if (JSON.stringify(fds.map(identity)) !== JSON.stringify(parents)) throw new StorageAttachmentRefusal("changed"); + }; + return { parent, name, parents, mount: firstMount, verify, close() { for (const fd of fds.reverse()) closeSync(fd); } }; + } catch (error) { for (const fd of fds.reverse()) closeSync(fd); throw error; } +} +function fileFacts(fd: number, volume: string) { + const info = fstatSync(fd, { bigint: true }); + if (!info.isFile() || info.nlink !== 1n || (info.mode & 0o7111n) !== 0n || mount(fd) !== volume) throw new StorageAttachmentRefusal("protected"); + if (info.size > BigInt(STORAGE_ATTACHMENT_LIMITS.fileBytes)) throw new StorageAttachmentRefusal("too_large"); + return { bytes: Number(info.size), mode: Number(info.mode & 0o7777n) }; +} +async function digest(fd: number, bytes: number, signal?: AbortSignal): Promise { + const started = Date.now(), hash = createHash("sha256"), buffer = Buffer.alloc(64 * 1024); + let offset = 0; + for (;;) { + signal?.throwIfAborted(); + if (Date.now() - started > STORAGE_ATTACHMENT_LIMITS.milliseconds) throw new StorageAttachmentRefusal("unavailable"); + const { bytesRead } = await read(fd, buffer, 0, Math.min(buffer.length, bytes - offset + 1), offset); + if (!bytesRead) break; + if (offset + bytesRead > bytes) throw new StorageAttachmentRefusal("changed"); + hash.update(buffer.subarray(0, bytesRead)); offset += bytesRead; + } + if (offset !== bytes) throw new StorageAttachmentRefusal("changed"); + return hash.digest("hex"); +} +async function opened(root: string, path: string, signal?: AbortSignal) { + const parent = openParent(root, path); let fd: number | undefined; + try { + fd = openSync(at(parent.parent, parent.name), FILE); + const facts = fileFacts(fd, parent.mount), initial = stamp(fd), hash = await digest(fd, facts.bytes, signal); + parent.verify(); + if (stamp(fd) !== initial) throw new StorageAttachmentRefusal("changed"); + const named = lstatSync(at(parent.parent, parent.name), { bigint: true }), held = fstatSync(fd, { bigint: true }); + if (!named.isFile() || named.dev !== held.dev || named.ino !== held.ino) throw new StorageAttachmentRefusal("changed"); + const version: StorageAttachmentVersion = { ...facts, hash, identity: initial, parents: parent.parents }; + return { parent, fd, version, close() { closeSync(fd!); parent.close(); } }; + } catch (error) { if (fd !== undefined) closeSync(fd); parent.close(); throw error; } +} +export async function inspectStorageAttachment(root: string, path: string, signal?: AbortSignal): Promise { + const held = await opened(root, path, signal); + try { return held.version; } finally { held.close(); } +} +function expectedVersion(value: StorageAttachmentVersion): void { + if (!/^[a-f0-9]{64}$/.test(value.hash) || !Number.isSafeInteger(value.bytes) || value.bytes < 0 || value.bytes > STORAGE_ATTACHMENT_LIMITS.fileBytes + || !Number.isSafeInteger(value.mode) || value.mode < 0 || (value.mode & ~0o666) !== 0 || !Array.isArray(value.parents) + || value.parents.length < 1 || value.parents.length > STORAGE_ATTACHMENT_LIMITS.depth || value.parents.some(part => typeof part !== "string" || part.length > 256) + || typeof value.identity !== "string" || value.identity.length > 512) throw new StorageAttachmentRefusal("unsupported"); +} +/** Approval and verified retained bytes must already be durable; hold the exclusive writer gate throughout. */ +export async function removeStorageAttachment(root: string, path: string, expected: StorageAttachmentVersion, signal?: AbortSignal, assertCurrent?: () => void): Promise { + expectedVersion(expected); + const held = await opened(root, path, signal); + try { + signal?.throwIfAborted(); held.parent.verify(); + if (JSON.stringify(held.version) !== JSON.stringify(expected) || stamp(held.fd) !== expected.identity) throw new StorageAttachmentRefusal("changed"); + const named = lstatSync(at(held.parent.parent, held.parent.name), { bigint: true }), info = fstatSync(held.fd, { bigint: true }); + if (!named.isFile() || named.dev !== info.dev || named.ino !== info.ino || named.nlink !== 1n) throw new StorageAttachmentRefusal("changed"); + assertCurrent?.(); signal?.throwIfAborted(); + unlinkSync(at(held.parent.parent, held.parent.name)); fsyncSync(held.parent.parent); + } finally { held.close(); } +} +/** Caller leases and verifies the original through its existing recovery owner. Never replaces a present entry. */ +export function restoreStorageAttachment(root: string, path: string, expected: StorageAttachmentVersion, copy: (fd: number) => void, assertCurrent?: () => void): void { + expectedVersion(expected); + const parent = openParent(root, path); let fd: number | undefined, complete = false; + try { + parent.verify(); + if (JSON.stringify(parent.parents) !== JSON.stringify(expected.parents)) throw new StorageAttachmentRefusal("changed"); + if (lstatSync(at(parent.parent, parent.name), { throwIfNoEntry: false })) throw new StorageAttachmentRefusal("occupied"); + assertCurrent?.(); + fd = openSync(at(parent.parent, parent.name), constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + copy(fd); + const facts = fileFacts(fd, parent.mount), hash = createHash("sha256"), buffer = Buffer.alloc(64 * 1024); + let offset = 0; const started = Date.now(); + for (;;) { + assertCurrent?.(); + if (Date.now() - started > STORAGE_ATTACHMENT_LIMITS.milliseconds) throw new StorageAttachmentRefusal("unavailable"); + const length = readSync(fd, buffer, 0, Math.min(buffer.length, expected.bytes - offset + 1), offset); + if (!length) break; + if (offset + length > expected.bytes) throw new StorageAttachmentRefusal("changed"); + hash.update(buffer.subarray(0, length)); offset += length; + } + if (facts.bytes !== expected.bytes || offset !== expected.bytes || hash.digest("hex") !== expected.hash) throw new StorageAttachmentRefusal("changed"); + parent.verify(); + const live = lstatSync(at(parent.parent, parent.name)), held = fstatSync(fd); + if (!live.isFile() || live.dev !== held.dev || live.ino !== held.ino) throw new StorageAttachmentRefusal("changed"); + fchmodSync(fd, expected.mode); fsyncSync(fd); fsyncSync(parent.parent); complete = true; + } finally { + try { if (fd !== undefined) { + try { + if (!complete) { + const live = lstatSync(at(parent.parent, parent.name), { throwIfNoEntry: false }), held = fstatSync(fd); + if (live?.isFile() && live.dev === held.dev && live.ino === held.ino) { unlinkSync(at(parent.parent, parent.name)); fsyncSync(parent.parent); } + } + } finally { closeSync(fd); } + } } finally { parent.close(); } + } +} + +/** Checks the exact reviewed parent chain and absence, without creating a replacement. */ +export function storageAttachmentVacant(root: string, path: string, expected: StorageAttachmentVersion): boolean { + expectedVersion(expected); const parent = openParent(root, path); + try { parent.verify(); return JSON.stringify(parent.parents) === JSON.stringify(expected.parents) && !lstatSync(at(parent.parent, parent.name), { throwIfNoEntry: false }); } + finally { parent.close(); } +} diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts index bc1c0fb994..7aff849b3a 100644 --- a/packages/core/src/storage.ts +++ b/packages/core/src/storage.ts @@ -1,4 +1,4 @@ -/** Supported metadata-only reader for host shells; never invokes Git or the brain CLI. */ +/** Supported Storage readers and narrow raw-file operations. Fact inspection evaluates Git and requires worker isolation in hosted consumers. */ import { isAbsolute } from "node:path"; import { collectStorageProcess } from "./lib/storage-process.js"; import type { StorageInventory } from "./lib/storage-inventory.js"; @@ -14,3 +14,8 @@ export async function readStorageInventory(options: StorageReadOptions): Promise if (paths.length > 17 || paths.some(path => typeof path !== "string" || !isAbsolute(path) || path.length > 4096 || path.includes("\0"))) throw Error("Storage requires bounded absolute host roots."); return collectStorageProcess(options.brainPath, options.gitRoots ?? [], options.signal ?? new AbortController().signal, true); } + +export { STORAGE_ATTACHMENT_LIMITS, StorageAttachmentRefusal, storageAttachmentPath, inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment, storageAttachmentVacant } from "./lib/storage-attachments.js"; +export type { StorageAttachmentVersion } from "./lib/storage-attachments.js"; +export { inspectStorageAttachmentFacts } from "./lib/storage-attachment-facts.js"; +export type { StorageAttachmentFacts } from "./lib/storage-attachment-facts.js"; diff --git a/packages/core/tests/storage-attachment-facts.test.ts b/packages/core/tests/storage-attachment-facts.test.ts new file mode 100644 index 0000000000..7c9ca92c8c --- /dev/null +++ b/packages/core/tests/storage-attachment-facts.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { inspectStorageAttachmentFacts } from "../src/lib/storage-attachment-facts"; + +test("attachment consequences include unindexed Markdown links and invalidate when references change", () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-attachment-facts-")); + try { + mkdirSync(join(root, "attachments")); mkdirSync(join(root, "notes")); + writeFileSync(join(root, "attachments/chart.pdf"), "%PDF chart"); + writeFileSync(join(root, "notes/voyage.md"), "[Chart](../attachments/chart.pdf)\n![Map][map]\n[map]: /attachments/chart.pdf\n"); + const first = inspectStorageAttachmentFacts(root, ["attachments/chart.pdf"]); + expect(first).toMatchObject({ complete: true, scannedFiles: 1, files: [{ path: "attachments/chart.pdf", git: "no-repository", references: [{ source: "notes/voyage.md", certainty: "resolved" }] }] }); + writeFileSync(join(root, "notes/arrival.md"), "[[chart.pdf]]\n"); + const second = inspectStorageAttachmentFacts(root, ["attachments/chart.pdf"]); + expect(second.fingerprint, "STORAGE_CONSEQUENCES_BOUND_TO_REVIEW").not.toBe(first.fingerprint); + expect(second.files[0]?.references.some(value => value.source === "notes/arrival.md" && value.certainty === "possible")).toBe(true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/core/tests/storage-attachments.test.ts b/packages/core/tests/storage-attachments.test.ts new file mode 100644 index 0000000000..db6a0444c4 --- /dev/null +++ b/packages/core/tests/storage-attachments.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, linkSync, symlinkSync, chmodSync, writeSync, renameSync, truncateSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment } from "../src/lib/storage-attachments"; +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-attachment-")), path = "attachments/chart.pdf", file = join(root, path), bytes = Buffer.from("%PDF-1.7\nIthaca\0chart\xff", "latin1"); + mkdirSync(join(root, "attachments")); writeFileSync(file, bytes); + return { root, path, file, bytes, close() { rmSync(root, { recursive: true, force: true }); } }; +} +test("exact raw attachment removal and original-byte restoration preserve independent content", async () => { + const f = fixture(); + try { + const expected = await inspectStorageAttachment(f.root, f.path); + await removeStorageAttachment(f.root, f.path, expected); + expect(existsSync(f.file)).toBe(false); + restoreStorageAttachment(f.root, f.path, expected, fd => { writeSync(fd, f.bytes); }); + expect(readFileSync(f.file), "STORAGE_UNDO_ORIGINAL_BYTES").toEqual(f.bytes); + } finally { f.close(); } +}); +test("changed attachments and occupied Undo never modify replacements", async () => { + const f = fixture(); + try { + const expected = await inspectStorageAttachment(f.root, f.path); writeFileSync(f.file, "replacement"); + await expect(removeStorageAttachment(f.root, f.path, expected), "STORAGE_REMOVE_STALE_FILE").rejects.toThrow("changed"); + expect(() => restoreStorageAttachment(f.root, f.path, expected, fd => { writeSync(fd, f.bytes); }), "STORAGE_UNDO_NEVER_OVERWRITES").toThrow("occupied"); + expect(readFileSync(f.file, "utf8")).toBe("replacement"); + } finally { f.close(); } +}); +test("aliases, executable attachments and protected paths remain ineligible", async () => { + const f = fixture(); + try { + linkSync(f.file, join(f.root, "attachments", "alias.pdf")); + await expect(inspectStorageAttachment(f.root, f.path)).rejects.toThrow("protected"); + rmSync(join(f.root, "attachments", "alias.pdf")); chmodSync(f.file, 0o700); + await expect(inspectStorageAttachment(f.root, f.path)).rejects.toThrow("protected"); + chmodSync(f.file, 0o600); symlinkSync(f.file, join(f.root, "attachments", "link.pdf")); + await expect(inspectStorageAttachment(f.root, "attachments/link.pdf")).rejects.toThrow(); + for (const path of [".brain/cache.pdf", "workspaces/harbor/chart.pdf", "context/policies/chart.pdf", "../chart.pdf", "attachments/script.ts"]) + await expect(inspectStorageAttachment(f.root, path)).rejects.toThrow("protected"); + expect(readFileSync(f.file)).toEqual(f.bytes); + } finally { f.close(); } +}); + + +test("same-byte replacements, cancellation and oversized sparse attachments cannot cross the reviewed boundary",async()=>{ + const f=fixture(); + try{ + const expected=await inspectStorageAttachment(f.root,f.path); + renameSync(join(f.root,"attachments"),join(f.root,"prior-attachments"));mkdirSync(join(f.root,"attachments"));writeFileSync(f.file,f.bytes); + await expect(removeStorageAttachment(f.root,f.path,expected),"STORAGE_REMOVAL_PARENT_IDENTITY").rejects.toThrow("changed"); + const current=await inspectStorageAttachment(f.root,f.path),controller=new AbortController();controller.abort(Error("cancelled")); + await expect(removeStorageAttachment(f.root,f.path,current,controller.signal),"STORAGE_REMOVAL_ABORT_BEFORE_EFFECT").rejects.toThrow("cancelled"); + expect(readFileSync(f.file)).toEqual(f.bytes); + truncateSync(f.file,64*1024*1024+1); + await expect(inspectStorageAttachment(f.root,f.path),"STORAGE_REMOVAL_FILE_BYTE_CEILING").rejects.toThrow("too_large"); + }finally{f.close();} +}); + +test("failed Undo copies remove only their own partial inode and preserve a concurrent replacement",async()=>{ + const f=fixture(); + try{ + const expected=await inspectStorageAttachment(f.root,f.path);await removeStorageAttachment(f.root,f.path,expected); + expect(()=>restoreStorageAttachment(f.root,f.path,expected,fd=>{writeSync(fd,"wrong bytes");}),"STORAGE_UNDO_VERIFIES_COPIED_BYTES").toThrow("changed"); + expect(existsSync(f.file)).toBe(false); + expect(()=>restoreStorageAttachment(f.root,f.path,expected,fd=>{writeSync(fd,f.bytes);renameSync(f.file,join(f.root,"moved-original.pdf"));writeFileSync(f.file,"Penelope's concurrent replacement");}),"STORAGE_UNDO_REPLACEMENT_DURING_COPY").toThrow("changed"); + expect(readFileSync(f.file,"utf8")).toBe("Penelope's concurrent replacement"); + }finally{f.close();} +}); + +test("a real same-filesystem nested bind mount remains ineligible",async()=>{ + const f=fixture(),source=join(f.root,"mounted-source");mkdirSync(source);writeFileSync(join(source,"chart.pdf"),f.bytes); + const entry=join(import.meta.dir,"../src/lib/storage-attachments.ts"),script=`import {inspectStorageAttachment} from ${JSON.stringify(entry)};let refused=false;try{await inspectStorageAttachment(${JSON.stringify(f.root)},${JSON.stringify(f.path)});}catch(error){if(error.reason!=="protected")throw error;refused=true;}if(!refused)throw Error("STORAGE_BIND_MOUNT_MUST_BE_REFUSED");console.log("protected nested mount");`; + try{ + const child=Bun.spawn(["bwrap","--unshare-user","--unshare-net","--bind","/","/","--bind",source,join(f.root,"attachments"),"--proc","/proc","--dev","/dev",process.execPath,"--config=/dev/null","--no-env-file","-e",script],{stdout:"pipe",stderr:"pipe"}); + const [code,stderr,stdout]=await Promise.all([child.exited,new Response(child.stderr).text(),new Response(child.stdout).text()]); + expect(code,stderr).toBe(0);expect(stdout,"STORAGE_BIND_MOUNT_REAL_CONTROL").toContain("protected nested mount");expect(readFileSync(f.file)).toEqual(f.bytes); + }finally{f.close();} +}); diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index e5a677feeb..943d86dd40 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3647,3 +3647,4 @@ export interface ServerScopeDecisionReceipt { export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-journal.js"; export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageMaintenanceKind, StorageMaintenanceRequest, StorageMaintenanceItem, StorageMaintenanceObservation, StorageMaintenanceReceipt, StorageMaintenanceRow, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; +export type { StorageRemovalReviewRequest, StorageRemovalStartRequest, StorageRemovalUndoRequest, StorageRemovalFile, StorageRemovalReview, StorageRemovalItem, StorageRemovalReceipt, StorageRemovalCapability } from "./storage-removal.js"; diff --git a/packages/ui-sdk/src/storage-removal.ts b/packages/ui-sdk/src/storage-removal.ts new file mode 100644 index 0000000000..224e110351 --- /dev/null +++ b/packages/ui-sdk/src/storage-removal.ts @@ -0,0 +1,36 @@ +import type { StorageMaintenanceObservation } from "./storage.js"; +/** Exact canonical attachment selections. Roots, versions and recovery locations remain server-owned. */ +export interface StorageRemovalReviewRequest { id: string; paths: string[] } +export interface StorageRemovalStartRequest { id: string; fingerprint: string } +export interface StorageRemovalUndoRequest { id: string; sourceId: string; paths: string[] } +export interface StorageRemovalFile { + path: string; bytes: number | null; hash: string | null; eligible: boolean; reason: string | null; + git: "tracked" | "untracked" | "ignored" | "no-repository" | "unknown"; + references: { source: string; certainty: "resolved" | "possible" }[]; + recovery: "requires-retention" | "retained" | "unavailable"; +} +export interface StorageRemovalReview { + id: string; action: "remove" | "undo"; sourceId: string | null; fingerprint: string; + observedAt: number; expiresAt: number; files: StorageRemovalFile[]; + complete: boolean; unchecked: string[]; selectedFileBytes: number; +} +export interface StorageRemovalItem { + path: string; + state: "not-started" | "applying" | "removed" | "restored" | "changed" | "protected" | "failed" | "unknown"; + reason: string | null; undoAvailable: boolean; undoReason: string | null; +} +export interface StorageRemovalReceipt { + review: StorageRemovalReview; + state: "review" | "waiting" | "running" | "complete" | "partial" | "refused" | "cancelled" | "unknown"; + requestedAt: number; startedAt: number | null; completedAt: number | null; reason: string | null; + items: StorageRemovalItem[]; + before: StorageMaintenanceObservation | null; after: StorageMaintenanceObservation | null; + index: "not-requested" | "complete" | "unavailable"; + commit: "not-requested"; sync: "not-requested"; + retained: { journal: "subject-to-retention"; git: "unchanged-by-operation"; backups: "not-measured" }; +} +export interface StorageRemovalCapability { + latest: StorageRemovalReceipt | null; + available: boolean; reason: string | null; + limits: { items: number; fileBytes: number; totalBytes: number; milliseconds: number; reviewMilliseconds: number; receiptBytes: number; receipts: number }; +} diff --git a/packages/ui-server/README.md b/packages/ui-server/README.md index e9a6b4a192..1fe5e95c0a 100644 --- a/packages/ui-server/README.md +++ b/packages/ui-server/README.md @@ -500,3 +500,14 @@ read the same operation. Cancellation and crashes preserve uncertainty about committed effects. Before/after store measurements and capacity observations are independent and never promise reclaimed bytes. No retention default or automatic cleanup schedule changes. The initial Storage pane is delivered separately. + +### Recoverable attachment operations + +The [human-owner removal API](../../docs/http-api.md#recoverable-attachment-removal-additive-2012) +reviews exact canonical attachments, retains and verifies originals through the +existing native journal, and records per-item intent before removal. Interrupted +work remains readable without replay. Undo is a fresh checked operation that +refuses replacements and disappears when retained detail expires. Ignored, +protected, linked, executable or unsupported targets remain ineligible. Review +runs Git inside a disposable worker; the existing Markdown application stays +narrow. The complete Storage pane is a separate UI delivery. diff --git a/packages/ui-server/migrations/052_storage_removal.sql b/packages/ui-server/migrations/052_storage_removal.sql new file mode 100644 index 0000000000..1a35bbcc47 --- /dev/null +++ b/packages/ui-server/migrations/052_storage_removal.sql @@ -0,0 +1,9 @@ +-- Charged by the existing journal allocation owner. Admission is bounded; receipts never trigger replay. +CREATE TABLE turn_journal_storage_operations ( + id TEXT PRIMARY KEY, brain TEXT NOT NULL, principal_id TEXT NOT NULL, + action TEXT NOT NULL, source_id TEXT, selection TEXT NOT NULL, fingerprint TEXT NOT NULL, + journal_turn_id TEXT NOT NULL, state TEXT NOT NULL, pid INTEGER NOT NULL, pid_start TEXT NOT NULL, + cancel_requested INTEGER NOT NULL DEFAULT 0, requested_at INTEGER NOT NULL, + receipt TEXT NOT NULL, private_versions TEXT NOT NULL +); +CREATE UNIQUE INDEX turn_journal_storage_active ON turn_journal_storage_operations(brain) WHERE state IN ('waiting','running'); diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index 812ac55310..b1f26a02e2 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -1,3 +1,5 @@ +import { createStorageRemoval } from "./storage/removal.js"; +import { createStorageRemovalRoutes } from "./routes/storage-removal.js"; import { createStorageMaintenance } from "./storage/maintenance.js"; import { createStorageMaintenanceRoutes } from "./routes/storage-maintenance.js"; import { createStorageCoordinator } from "./storage/coordinator.js"; @@ -839,6 +841,13 @@ export async function createApp(options: CreateAppOptions = {}): Promise storageMaintenance.close()); app.route("/api", createStorageMaintenanceRoutes(storageMaintenance)); + const storageRemoval = createStorageRemoval(db, { brainPath: config.brainPath, boundary, journal, gate: hostExecution.gate()!, lock: host.workerLock, storage, + protectedPaths: [...storageStores.map(store => store.path), ...config.storage?.gitRoots ?? []].filter(path => path === config.brainPath || path.startsWith(config.brainPath + "/")).map(path => path === config.brainPath ? "." : path.slice(config.brainPath.length + 1)), + authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, + }); + bootCleanup.push(() => storageRemoval.close()); + app.route("/api", createStorageRemovalRoutes(storageRemoval)); + const reverts = createTurnReverts(db, { brainPath: config.brainPath, journal, gate: hostExecution.gate()!, lock: host.workerLock, isAuthorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, refresh() { host.governedTurns?.refresh(host.workerLock); } }); @@ -967,6 +976,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise void | Promise) => { try { await close(); } catch (error) { failures.push(error); } }; + await attempt(() => storageRemoval.close()); await attempt(() => storageMaintenance.close()); await attempt(() => inbox.close()); await attempt(() => host.close()); diff --git a/packages/ui-server/src/brain/host-boundary.ts b/packages/ui-server/src/brain/host-boundary.ts index f300e162d8..db74d7e9f8 100644 --- a/packages/ui-server/src/brain/host-boundary.ts +++ b/packages/ui-server/src/brain/host-boundary.ts @@ -69,6 +69,10 @@ export interface BrainExecution { onExecute?: () => void; /** @internal Combined output bound for an explicitly bounded operation. */ outputBytes?: number; + /** @internal A consequence reader discards every worker output, including Git side effects. */ + publish?: false; + /** @internal Fixed host-selected publication areas, never supplied by a request. */ + outputAreas?: readonly string[]; /** Existing host files the command may append to (a cron span sink). */ writeFiles?: readonly string[]; /** A complete hosted sync: its only credentialed part is the host mirror transport. */ @@ -429,7 +433,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { execution.signal?.throwIfAborted(); execution.assertAuthorized?.(); if (!isolated()) { - if (execution.outputBytes !== undefined) throw Error("Bounded Storage maintenance requires the qualified worker boundary"); + if (execution.outputBytes !== undefined || execution.publish === false) throw Error("Bounded Storage maintenance requires the qualified worker boundary"); execution.onExecute?.(); const proc = hostSpawn(execution); try { @@ -451,14 +455,14 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { let outputs: Awaited> | undefined; try { execution.signal?.throwIfAborted(); - outputs = await prepareMaintenanceOutput(brain, { startupView: view, signal: execution.signal }); + outputs = await prepareMaintenanceOutput(brain, { startupView: view, signal: execution.signal, areas: execution.outputAreas }); const launch = command(execution); const git = await prepareMaintenanceGit(launch, outputs, execution.signal); execution.signal?.throwIfAborted(); execution.assertAuthorized?.(); execution.onExecute?.(); const result = await runMaintenanceWorker({ ...launch, startupView: outputs.view, output: outputs.output, ...(git ? { git } : {}), stdin: execution.stdin, ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}), ...(execution.outputBytes ? { outputBytes: execution.outputBytes } : {}) }); - if (!result.killed) { + if (!result.killed && execution.publish !== false) { execution.assertAuthorized?.(); if (execution.lock) await outputs.publishLocked(execution.lock.keyed); else await outputs.publish(); @@ -473,6 +477,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { async launch(execution) { execution.signal?.throwIfAborted(); if (!isolated()) { + if (execution.publish === false) throw Error("A discarded read requires the qualified worker boundary"); const proc = hostSpawn(execution); return { stdout: proc.stdout, stderr: proc.stderr, exited: proc.exited, get signalCode() { return proc.signalCode; }, kill: (signal?: NodeJS.Signals) => killWrapped(proc, exec, signal ?? "SIGTERM") }; @@ -516,7 +521,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { execution.signal?.throwIfAborted(); view = await prepareInput(execution.signal); execution.signal?.throwIfAborted(); - outputs = await prepareMaintenanceOutput(brain, { startupView: view, signal: execution.signal }); + outputs = await prepareMaintenanceOutput(brain, { startupView: view, signal: execution.signal, areas: execution.outputAreas }); const launch = command(execution); const git = await prepareMaintenanceGit(launch, outputs, execution.signal); execution.signal?.throwIfAborted(); @@ -526,7 +531,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { void worker.input(execution.stdin ?? ""); let signalCode: string | null = null; const exited = worker.exited.then(async code => { - if (!signalCode) { + if (!signalCode && execution.publish !== false) { if (execution.lock) await outputs!.publishLocked(execution.lock.keyed); else await outputs!.publish(); } @@ -539,6 +544,7 @@ export function createHostBoundary(options: HostBoundaryOptions): HostBoundary { }, runSync(execution) { if (!isolated()) { + if (execution.publish === false) throw Error("A discarded read requires the qualified worker boundary"); const result = Bun.spawnSync([...execution.argv], { cwd: brain, env: { ...execution.env }, stdout: "pipe", stderr: "pipe", timeout: execution.timeoutMs }); return { stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode ?? 1 }; } diff --git a/packages/ui-server/src/brain/journal-accounting.ts b/packages/ui-server/src/brain/journal-accounting.ts index c139715825..00413d3222 100644 --- a/packages/ui-server/src/brain/journal-accounting.ts +++ b/packages/ui-server/src/brain/journal-accounting.ts @@ -18,7 +18,7 @@ export function createJournalAccounting(db: Database, dbPath: string, options: { } = {}): JournalObjectAccounting & { write(operation: () => T): T; usage(): number; - openRead(hash: string): string; + openRead(hash: string, assertOwner?: () => void): string; releaseRead(id: string): void; recover(objects: { has(hash: string): boolean; verify(hash: string, bytes: number): boolean; discardPending(id: string): void }): void; } { @@ -93,9 +93,10 @@ export function createJournalAccounting(db: Database, dbPath: string, options: { const usage = (): number => measure().bytes; return { write, usage, - openRead(hash) { + openRead(hash, assertOwner) { const id = crypto.randomUUID(); write(() => { + assertOwner?.(); if (!db.query("SELECT 1 FROM turn_journal_objects WHERE hash=? AND state='present'").get(hash)) throw new JournalStorageUnavailable("the retained object is unavailable"); db.query("INSERT INTO turn_journal_reservations (id,bytes,pid,pid_start,created_at,hash,kind) VALUES (?,0,?,?,?,?,'read')").run(id, SELF.pid, SELF.start, Date.now(), hash); }); return id; diff --git a/packages/ui-server/src/brain/journal-objects.ts b/packages/ui-server/src/brain/journal-objects.ts index c23d9a0217..d7043097f9 100644 --- a/packages/ui-server/src/brain/journal-objects.ts +++ b/packages/ui-server/src/brain/journal-objects.ts @@ -33,7 +33,7 @@ export class JournalCapacityError extends Error { /** No brain pathname reaches this store. Only checked source descriptors and sha256 object names do. */ export function createJournalObjects(root: string, accounting: JournalObjectAccounting): { has(hash: string): boolean; - putFileAsync(source: number, size: number): Promise; + putFileAsync(source: number, size: number, signal?: AbortSignal): Promise; read(hash: string, maxBytes: number): Buffer; verify(hash: string, bytes: number): boolean; withVerifiedCopy(hash: string, bytes: number, apply: (copy: (destination: number) => void) => T): T; @@ -63,8 +63,8 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco }; return { has, - async putFileAsync(source, size) { - assertOpen(); + async putFileAsync(source, size, signal) { + assertOpen(); signal?.throwIfAborted(); if (!Number.isSafeInteger(size) || size < 0) throw new Error("Invalid journal object size."); const info = fstatSync(source); if (!info.isFile() || info.size !== size) throw new Error("Journal source changed before its storage reservation."); @@ -75,6 +75,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco const hash = createHash("sha256"), chunk = Buffer.alloc(64 * 1024); let position = 0; for (;;) { + signal?.throwIfAborted(); const { bytesRead } = await readAsync(source, chunk, 0, Math.min(chunk.length, size - position + 1), position); if (!bytesRead) break; if (position + bytesRead > size) throw new Error("Journal source grew during capture."); @@ -82,6 +83,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco if (output !== undefined) { let written = 0; while (written < bytesRead) { + signal?.throwIfAborted(); const { bytesWritten } = await writeAsync(output, chunk, written, bytesRead - written, position + written); if (!bytesWritten) throw new Error("Journal object write made no progress."); written += bytesWritten; @@ -93,6 +95,7 @@ export function createJournalObjects(root: string, accounting: JournalObjectAcco return hash.digest("hex"); } const expectedHash = await digestSource(); + signal?.throwIfAborted(); const reservation = accounting.reserve(size, expectedHash); const name = `pending-${reservation.id ?? crypto.randomUUID()}`; let output: number | undefined, present = false, published = false; diff --git a/packages/ui-server/src/brain/maintenance-entry.ts b/packages/ui-server/src/brain/maintenance-entry.ts index 58ccef04d1..10ceb67d09 100644 --- a/packages/ui-server/src/brain/maintenance-entry.ts +++ b/packages/ui-server/src/brain/maintenance-entry.ts @@ -16,6 +16,7 @@ import { maintenanceDataRoot } from "../config/env.js"; interface PlanRequest { op: "plan-add"; root: string; input: Record } interface IndexRequest { op: "index"; root: string } +interface StorageRequest { op: "storage-facts"; root: string; paths: string[] } const MAX_DOCUMENT_BYTES = 4 * 1024 * 1024; @@ -34,9 +35,13 @@ function readDocument(root: string, path: string): string | null { } async function main(): Promise { - const request = JSON.parse(await Bun.stdin.text()) as PlanRequest | IndexRequest; + const request = JSON.parse(await Bun.stdin.text()) as PlanRequest | IndexRequest | StorageRequest; // Only the trusted launch sets this separate, disposable output root. const root = maintenanceDataRoot() ?? request.root; + if (request.op === "storage-facts") { + const { inspectStorageAttachmentFacts } = await import("@schlessera/brain/storage"); + return { ok: true, facts: inspectStorageAttachmentFacts(root, request.paths) }; + } const core = await import("@schlessera/brain/internal"); const context = await core.initContext({ root }); const db = core.openDatabase(context.dbPath); diff --git a/packages/ui-server/src/brain/maintenance-output.ts b/packages/ui-server/src/brain/maintenance-output.ts index b829ca34fd..eae89d3ab2 100644 --- a/packages/ui-server/src/brain/maintenance-output.ts +++ b/packages/ui-server/src/brain/maintenance-output.ts @@ -1,6 +1,6 @@ /** The native stage's checkpoint is owned until publication and deletion complete. */ import { prepareNativeWorkerOutput, type WorkerStartupView } from "@schlessera/brain-ui-sdk/internal"; -export async function prepareMaintenanceOutput(brain: string, options: { startupView: WorkerStartupView; signal?: AbortSignal }) { +export async function prepareMaintenanceOutput(brain: string, options: { startupView: WorkerStartupView; signal?: AbortSignal; areas?: readonly string[] }) { const prepared = await prepareNativeWorkerOutput(brain, options); return { ...prepared, output: { ...prepared.output, target: "/tmp/brain-maintenance-output" } }; } diff --git a/packages/ui-server/src/brain/native-journal-capture.ts b/packages/ui-server/src/brain/native-journal-capture.ts index 4573a241f1..e86aed0263 100644 --- a/packages/ui-server/src/brain/native-journal-capture.ts +++ b/packages/ui-server/src/brain/native-journal-capture.ts @@ -6,7 +6,7 @@ import { isJournalSteeringPath, nativeJournalSteeringInputs } from "./journal-st import type { JournalSnapshotEntry } from "./journal-snapshot.js"; export type NativeJournalPolicy = Awaited>; -export function createNativeJournalCapture(manager: NativeJournalManager, retain: (source: number, bytes: number) => Promise) { +export function createNativeJournalCapture(manager: NativeJournalManager, retain: (source: number, bytes: number, signal?: AbortSignal) => Promise) { const observer = createNativeJournalObserver(manager); const entry = (metadata: NativeJournalMetadata, policy: NativeJournalPolicy): JournalSnapshotEntry | undefined => { const scope = journalPathScope(metadata.path); @@ -19,6 +19,26 @@ export function createNativeJournalCapture(manager: NativeJournalManager, retain const root = observer.currentRoot(); return freezeNativeJournalPolicy(observer.ignoreEntries(), (path, maximum) => readNativeJournalFile(root.path, path, maximum), covered); }, + allowed(path: string, policy: NativeJournalPolicy): boolean { return !!journalPathScope(path) && !policy.ignored(path); }, + eligible(path: string, policy: NativeJournalPolicy): boolean { + const metadata = observer.get(path); + return !!metadata && metadata.kind === "file" && metadata.bytes <= 64 * 1024 * 1024 && !policy.ignored(path) && !!journalPathScope(path); + }, + /** Exact current sealed before-version; the caller keeps the ordinary policy and storage owner. */ + async retainBefore(path: string, policy: NativeJournalPolicy, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const metadata = observer.get(path), root = observer.currentRoot(); + if (!metadata) return undefined; + if (metadata.bytes > 64 * 1024 * 1024) throw new Error("Selected attachment exceeds the bounded recovery copy size."); + const result = entry(metadata, policy); + if (!result || result.ignored || result.kind !== "file") return result; + const file = await openNativeJournalFile(root.path, path); + try { + const hash = await retain(file.fd, metadata.bytes, signal); + signal?.throwIfAborted(); + return { ...result, hash }; + } finally { await file.close(); } + }, async observe(policies: ReadonlyMap, signal?: AbortSignal) { const observation = await observer.observe(signal); try { diff --git a/packages/ui-server/src/brain/turn-journal.ts b/packages/ui-server/src/brain/turn-journal.ts index a2f2e16a38..ba427a2ef5 100644 --- a/packages/ui-server/src/brain/turn-journal.ts +++ b/packages/ui-server/src/brain/turn-journal.ts @@ -34,8 +34,8 @@ export function createTurnJournal(db: Database, options: { if (!processAlive(row.pid, row.pid_start)) accounting.write(() => db.query("UPDATE turn_journal_turns SET state='partial',ended_at=?,missing=? WHERE turn_id=?").run(now(), JSON.stringify([{ path: ".", reason: "interrupted_observation" }]), row.turn_id)); } const active = new Map void; sequence: number }>(); - const capture = createNativeJournalCapture(options.nativeManager, async (source, bytes) => { - const hash = await retainAsync(() => objects.putFileAsync(source, bytes)); + const capture = createNativeJournalCapture(options.nativeManager, async (source, bytes, signal) => { + const hash = await retainAsync(() => objects.putFileAsync(source, bytes, signal)); // Pin the shared observation before another admission can collect objects. retain(() => accounting.write(() => { for (const turnId of active.keys()) db.query("INSERT OR IGNORE INTO turn_journal_object_refs VALUES (?,?)").run(turnId, hash); @@ -298,6 +298,28 @@ export function createTurnJournal(db: Database, options: { collectObjects(); publish(turnId); } }); }, + /** Ordinary current coverage only; never adds selected paths to ignored content. */ + removalCoverage(paths: readonly string[], signal?: AbortSignal, absent = false): Promise { return held(async () => { + if (paths.length > 32) throw new JournalStorageUnavailable("Too many selected attachments."); + await capture.initialize(signal); await observe(); signal?.throwIfAborted(); + const policy = await capture.policy(); + return paths.map(path => absent ? capture.allowed(path, policy) : capture.eligible(path, policy)); + }); }, + /** Targeted user removal retains its original before effects; no ignore/retention override. */ + retainBefore(turnId: string, path: string, signal?: AbortSignal): Promise { return held(async () => { + const turn = active.get(turnId); + if (!turn || summary(turnId).state !== "recording") throw new JournalStorageUnavailable("A complete active journal is required before removal."); + const before = version(await capture.retainBefore(path, turn.policy, signal)); + if (!before || before.ignored || before.kind !== "file" || !before.hash) return null; + accounting.write(() => pin(turnId, before)); + if (!objects.verify(before.hash, before.bytes)) throw new JournalStorageUnavailable("The retained original could not be verified."); + return before; + }); }, + hasRetainedVersion(turnId: string, selected: JournalVersion): boolean { + if (!selected.hash || selected.ignored || selected.kind !== "file" || ["unknown", "expired"].includes(summary(turnId).state)) return false; + if (!db.query("SELECT 1 FROM turn_journal_object_refs WHERE turn_id=? AND hash=?").get(turnId, selected.hash)) return false; + try { return objects.verify(selected.hash, selected.bytes); } catch { return false; } + }, /** All operation metadata uses the journal's existing physical allocation owner. */ writeOperation(write: () => T): T { return accounting.write(write); }, reserveRevert(turnId: string, sourceTurnId: string, versions: readonly (JournalVersion | null)[]) { @@ -316,6 +338,15 @@ export function createTurnJournal(db: Database, options: { db.query("DELETE FROM turn_journal_object_refs WHERE turn_id=?").run(turnId); }); collectObjects(); }, + /** Acquire the read lease while the selected source still owns its original. */ + withRetainedRemoval(turnId: string, selected: JournalVersion, apply: (copy: (fd: number) => void) => T): T { + if (!selected.hash || selected.ignored || selected.kind !== "file") throw new JournalStorageUnavailable("retained original is unavailable"); + const lease = accounting.openRead(selected.hash, () => { + if (!db.query("SELECT 1 FROM turn_journal_object_refs r JOIN turn_journal_turns t ON t.turn_id=r.turn_id WHERE r.turn_id=? AND r.hash=? AND t.state IN ('complete','partial') AND t.ended_at IS NOT NULL").get(turnId, selected.hash!)) throw new JournalStorageUnavailable("original retention ownership expired before admission"); + }); + try { return objects.withVerifiedCopy(selected.hash, selected.bytes, apply); } + finally { try { accounting.releaseRead(lease); } catch { pendingReadReleases.add(lease); } } + }, withRetainedVersion(selected: JournalVersion | null, apply: (copy: (fd: number) => void) => T): T { if (!selected || selected.kind === "symlink") return apply(() => {}); if (!selected.hash || selected.ignored || selected.kind !== "file") throw new JournalStorageUnavailable("retained version is unavailable"); diff --git a/packages/ui-server/src/routes/storage-removal.ts b/packages/ui-server/src/routes/storage-removal.ts new file mode 100644 index 0000000000..abd429e9e0 --- /dev/null +++ b/packages/ui-server/src/routes/storage-removal.ts @@ -0,0 +1,41 @@ +import { Hono } from "hono"; +import { z } from "zod"; +import type { AppEnv } from "../app-env.js"; +import { requireJson } from "../middleware/origin.js"; +import { readJsonBody } from "../middleware/body-limit.js"; +import { StorageUnavailable } from "../storage/coordinator.js"; +import type { StorageRemoval } from "../storage/removal.js"; +const id = z.string().regex(/^[a-zA-Z0-9_-]{16,96}$/), paths = z.array(z.string().min(1).max(4096)).min(1).max(32); +const review = z.object({ id, paths }).strict(), undo = z.object({ id, sourceId: id, paths }).strict(), start = z.object({ id, fingerprint: z.string().regex(/^[a-f0-9]{64}$/) }).strict(); +export function createStorageRemovalRoutes(removal: StorageRemoval): Hono { + const routes = new Hono(); + routes.onError((error, c) => c.json({ error: error instanceof StorageUnavailable ? error.message : "Attachment operation unavailable; read its original receipt before starting another operation." }, error instanceof StorageUnavailable ? error.status : 503)); + routes.use("/storage/removals*", async (c, next) => { + c.header("Cache-Control", "private, no-store"); const principal = c.get("principal"); + if (!principal) return c.json({ error: "Authentication required." }, 401); + if (!["owner", "ambient"].includes(principal.kind)) return c.json({ error: "Attachment removal requires human-owner access." }, 403); + await next(); + }); + routes.get("/storage/removals", c => c.json(removal.capability(c.get("principal")!.id))); + routes.get("/storage/removals/:id", c => c.json(removal.receipt(c.get("principal")!.id, c.req.param("id")))); + routes.post("/storage/removals/reviews", requireJson(), async c => { + let body: unknown; try { body = await readJsonBody(c, 140_000); } catch { return c.json({ error: "Invalid attachment JSON." }, 400); } + if (body instanceof Response) return body; const parsed = review.safeParse(body); + if (!parsed.success) return c.json({ error: "Select bounded attachment paths and a new review identity." }, 400); + return c.json(await removal.review(c.get("principal")!.id, parsed.data.id, parsed.data.paths)); + }); + routes.post("/storage/removals/undo-reviews", requireJson(), async c => { + let body: unknown; try { body = await readJsonBody(c, 140_000); } catch { return c.json({ error: "Invalid attachment JSON." }, 400); } + if (body instanceof Response) return body; const parsed = undo.safeParse(body); + if (!parsed.success) return c.json({ error: "Select retained originals from one removal and a new Undo identity." }, 400); + return c.json(await removal.review(c.get("principal")!.id, parsed.data.id, parsed.data.paths, parsed.data.sourceId)); + }); + routes.post("/storage/removals", requireJson(), async c => { + let body: unknown; try { body = await readJsonBody(c, 4096); } catch { return c.json({ error: "Invalid attachment JSON." }, 400); } + if (body instanceof Response) return body; const parsed = start.safeParse(body); + if (!parsed.success) return c.json({ error: "Confirm the exact current review identity." }, 400); + return c.json(removal.start(c.get("principal")!.id, parsed.data.id, parsed.data.fingerprint), 202); + }); + routes.delete("/storage/removals/:id", c => c.json(removal.cancel(c.get("principal")!.id, c.req.param("id")), 202)); + return routes; +} diff --git a/packages/ui-server/src/storage/maintenance.ts b/packages/ui-server/src/storage/maintenance.ts index f1e77c704d..0b972a0c20 100644 --- a/packages/ui-server/src/storage/maintenance.ts +++ b/packages/ui-server/src/storage/maintenance.ts @@ -28,7 +28,7 @@ function scratchOutcome(raw: string): NonNullable ({ id: item.path, state: "cleaned" as const, reason: null })), ...value.failed.map((item: {path: string}) => ({ id: item.path, state: "failed" as const, reason: "owner_removal_failed" }))]; return { ...emptyOutcome(), removed: value.removed.length, failed: value.failed.length, pinnedFiles: value.pinned.files, pinnedBytes: value.pinned.bytes, overageBytes: value.overageBytes, suspended: !!value.pinError, items: items.slice(0, 200), truncated: items.length > 200 }; } -function observation(snapshot: StorageSnapshot | null, scope: readonly string[]): StorageMaintenanceObservation | null { +export function storageObservation(snapshot: StorageSnapshot | null, scope: readonly string[]): StorageMaintenanceObservation | null { if (!snapshot) return null; const stores = snapshot.roots.filter(root => scope.includes(root.category)).map(root => ({ ...root, label: root.label.slice(0, 80), reason: root.reason?.slice(0, 160) ?? null })); const ids = new Set(stores.map(root => root.id)); @@ -85,7 +85,7 @@ export function createStorageMaintenance(db: Database, options: { try { assertAuthorized(); if (!available(result.operation)) { result.state = "refused"; result.reason = "owner_unavailable"; return; } - result.before = observation(await options.storage.measure(principal, controller.signal), result.scope); + result.before = storageObservation(await options.storage.measure(principal, controller.signal), result.scope); assertAuthorized(); if (result.operation === "pack-git" && result.before?.capacities.some(value => value.available === 0)) { result.state = "refused"; result.reason = "no_space"; return; } result.state = "running"; result.startedAt = Date.now(); result.outcome = emptyOutcome(); write(result); @@ -122,7 +122,7 @@ export function createStorageMaintenance(db: Database, options: { // A cancelled write can still have committed effects. Its fresh observation is independent. if (result.startedAt && options.authorized(principal) && !closed) { const after = new AbortController(), deadline = setTimeout(() => after.abort(), 36_000); - try { result.after = observation(await options.storage.measure(principal, after.signal), result.scope); } catch { result.after = null; } + try { result.after = storageObservation(await options.storage.measure(principal, after.signal), result.scope); } catch { result.after = null; } finally { clearTimeout(deadline); } } result.completedAt = Date.now(); diff --git a/packages/ui-server/src/storage/removal-facts.ts b/packages/ui-server/src/storage/removal-facts.ts new file mode 100644 index 0000000000..4bd1ff18a6 --- /dev/null +++ b/packages/ui-server/src/storage/removal-facts.ts @@ -0,0 +1,21 @@ +import { subprocessEnv } from "../config/env.js"; +/** Repository evaluation stays in a disposable worker; even incidental Git writes are discarded. */ +import { z } from "zod"; +import type { StorageAttachmentFacts } from "@schlessera/brain/storage"; +import { MAINTENANCE_ENTRY, type HostBoundary } from "../brain/host-boundary.js"; +const relative = z.string().min(1).max(4096).refine(value => !value.startsWith("/") && !/[\\\u0000-\u001f\u007f]/u.test(value) && value.split("/").every(part => part && part !== "." && part !== "..")); +const hash = z.string().regex(/^[a-f0-9]{64}$/); +const schema = z.object({ fingerprint: hash, complete: z.boolean(), scannedFiles: z.number().int().min(0).max(2000), unchecked: z.array(z.string().max(160)).max(16), + files: z.array(z.object({ path: relative, git: z.enum(["tracked", "untracked", "ignored", "no-repository", "unknown"]), truncated: z.boolean(), + references: z.array(z.object({ source: relative, hash, certainty: z.enum(["resolved", "possible"]) }).strict()).max(200) }).strict()).max(32) }).strict(); +export async function readRemovalFacts(boundary: Pick, root: string, paths: string[], signal: AbortSignal, assertAuthorized: () => void): Promise { + if (!boundary.isolated()) throw Error("Attachment review requires the qualified worker boundary."); + const result = await boundary.run({ argv: [process.execPath, "--config=/dev/null", "--no-env-file", MAINTENANCE_ENTRY], env: subprocessEnv("brainCli", { NO_COLOR: "1" }), + credentials: false, gate: "none", purpose: "storage-removal-review", publish: false, outputBytes: 262144, timeoutMs: 30_000, + stdin: JSON.stringify({ op: "storage-facts", root, paths }), signal, assertAuthorized }); + if (result.exitCode !== 0 || result.timedOut) throw Error("Attachment consequences unavailable."); + const response = JSON.parse(result.stdout) as { ok?: boolean; facts?: unknown }; + const facts = schema.parse(response.facts); + if (!response.ok || facts.files.length !== paths.length || facts.files.some((file, index) => file.path !== paths[index])) throw Error("Attachment selection changed in review."); + return facts; +} diff --git a/packages/ui-server/src/storage/removal.ts b/packages/ui-server/src/storage/removal.ts new file mode 100644 index 0000000000..a2efa850a8 --- /dev/null +++ b/packages/ui-server/src/storage/removal.ts @@ -0,0 +1,237 @@ +import { subprocessEnv } from "../config/env.js"; +/** Human-authorized attachment edits with charged retained originals and durable per-item intent. */ +import type { Database } from "bun:sqlite"; +import { realpathSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment, storageAttachmentPath, storageAttachmentVacant, StorageAttachmentRefusal, type StorageAttachmentVersion, type StorageAttachmentFacts } from "@schlessera/brain/storage"; +import type { JournalVersion, StorageRemovalCapability, StorageRemovalReview, StorageRemovalReceipt, StorageRemovalFile } from "@schlessera/brain-ui-sdk/protocol"; +import { BRAIN_LOCK_KEY, type KeyedLock } from "@schlessera/brain-ui-sdk/server"; +import { GIT_LOCK_KEY } from "@schlessera/brain-ui-sdk/internal"; +import { MAINTENANCE_ENTRY, type HostBoundary } from "../brain/host-boundary.js"; +import type { TurnJournal } from "../brain/turn-journal.js"; +import { SELF, processAlive, type WriterGate } from "../brain/writer-gate.js"; +import { StorageUnavailable, type StorageCoordinator } from "./coordinator.js"; +import { readRemovalFacts } from "./removal-facts.js"; +import { storageObservation } from "./maintenance.js"; + +export const STORAGE_REMOVAL_LIMITS = Object.freeze({ items: 32, fileBytes: 64 * 1024 * 1024, totalBytes: 128 * 1024 * 1024, milliseconds: 300_000, reviewMilliseconds: 60_000, receiptBytes: 262_144, receipts: 128 }); +const REVIEW_TTL = 10 * 60_000, runningHere = new Set(); +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +const prefix = (path: string, root: string) => root === "." || path === root || path.startsWith(`${root}/`); +type PrivateFile = { path: string; version: StorageAttachmentVersion | null; consequences: string; retained: JournalVersion | null }; +type Row = { id: string; principal_id: string; action: "remove" | "undo"; source_id: string | null; selection: string; fingerprint: string; journal_turn_id: string; state: StorageRemovalReceipt["state"]; pid: number; pid_start: string; receipt: string; private_versions: string }; +const factIdentity = (facts: StorageAttachmentFacts, path: string) => digest({ file: facts.files.find(file => file.path === path), complete: facts.complete, unchecked: facts.unchecked }); +const encode = (value: unknown) => { const text = JSON.stringify(value); if (Buffer.byteLength(text) > STORAGE_REMOVAL_LIMITS.receiptBytes) throw new StorageUnavailable(400, "Attachment receipt exceeds its bound."); return text.padEnd(STORAGE_REMOVAL_LIMITS.receiptBytes); }; +const validId = (id: string) => /^[a-zA-Z0-9_-]{16,96}$/.test(id); + +export function createStorageRemoval(db: Database, options: { + brainPath: string; boundary: Pick; journal: TurnJournal; gate: WriterGate; lock: KeyedLock; storage: StorageCoordinator; + authorized(principal: string): boolean; + /** Existing operator-owned stores, represented relative to the canonical brain when applicable. */ + protectedPaths: readonly string[]; +}) { + const brain = realpathSync(options.brainPath), journal = options.journal; + const active = new Map }>(); + const reviews = new Set(), pendingReviews = new Set>(); let closed = false; + const key = (id: string) => `${brain}:${id}`; + const select = (id: string) => db.query("SELECT * FROM turn_journal_storage_operations WHERE id=? AND brain=?").get(id, brain) as Row | null; + const authorize = (principal: string) => { if (!options.authorized(principal)) throw new StorageUnavailable(403, "Attachment removal requires current human-owner access."); }; + function capability(principal: string): StorageRemovalCapability { + authorize(principal); reconcile(); const status = options.boundary.status(); + const latest = db.query("SELECT * FROM turn_journal_storage_operations WHERE brain=? AND principal_id=? ORDER BY requested_at DESC,rowid DESC LIMIT 1").get(brain, principal) as Row | null; + const available = !closed && process.platform === "linux" && options.boundary.isolated() && status.available; + return { available, reason: available ? null : "qualified_native_recovery_unavailable", limits: STORAGE_REMOVAL_LIMITS, latest: latest ? recover(latest) : null }; + } + function protectedReason(path: string): string | null { + if (!storageAttachmentPath(path)) return "unsupported_or_protected_attachment"; + const status = options.boundary.status(); + if (!status.available || status.profileFallback || !status.protectedPaths) return "protection_or_ownership_unknown"; + if ([...options.protectedPaths, ...status.protectedPaths.map(value => value.path)].some(root => prefix(path, root))) return "protected_owner_path"; + return null; + } + const save = (row: Row, result: StorageRemovalReceipt, versions?: PrivateFile[]) => journal.writeOperation(() => { + db.query("UPDATE turn_journal_storage_operations SET state=?,receipt=?,private_versions=? WHERE id=? AND brain=?") + .run(result.state, encode(result), versions ? encode(versions) : row.private_versions, row.id, brain); + row.state = result.state; row.receipt = encode(result); if (versions) row.private_versions = encode(versions); + }); + function recover(row: Row): StorageRemovalReceipt { + const result = JSON.parse(row.receipt) as StorageRemovalReceipt; + if (["waiting", "running"].includes(row.state) && (!processAlive(row.pid, row.pid_start) || row.pid === SELF.pid && row.pid_start === SELF.start && !runningHere.has(key(row.id)))) { + result.state = row.state === "waiting" ? "refused" : "unknown"; result.reason = row.state === "waiting" ? "interrupted_before_effect" : "interrupted_outcome_unknown"; result.completedAt = Date.now(); + for (const item of result.items) if (item.state === "applying") { item.state = "unknown"; item.reason = "interrupted_outcome_unknown"; } + try { save(row, result); } catch { /* Durable pending intent still prevents replay. */ } + } + const versions = JSON.parse(row.private_versions) as PrivateFile[]; + for (const item of result.items) { + const original = versions.find(file => file.path === item.path)?.retained; + item.undoAvailable = row.action === "remove" && item.state === "removed" && !!original && journal.hasRetainedVersion(row.journal_turn_id, original); + item.undoReason = item.undoAvailable ? null : item.state === "removed" ? "retained_detail_expired_or_unavailable" : "no_confirmed_removal"; + } + return result; + } + function reconcile() { for (const row of db.query("SELECT * FROM turn_journal_storage_operations WHERE brain=? AND state IN ('waiting','running')").all(brain) as Row[]) recover(row); } + function receipt(principal: string, id: string) { + authorize(principal); const row = select(id); + if (!row || row.principal_id !== principal) throw new StorageUnavailable(404, "Attachment receipt unavailable."); + return recover(row); + } + async function exclusive(signal: AbortSignal, purpose: string, apply: () => Promise): Promise { + const hold = await options.gate.acquire("exclusive", purpose, { queueTurns: true, signal }); + try { return await options.lock.withLock(BRAIN_LOCK_KEY, () => options.lock.withLock(GIT_LOCK_KEY, apply, { signal }), { signal }); } + finally { hold.release(); } + } + function pathsFor(paths: string[]): string[] { + if (!Array.isArray(paths) || paths.length < 1 || paths.length > STORAGE_REMOVAL_LIMITS.items || paths.some(path => typeof path !== "string" || Buffer.byteLength(path) > 4096 || /[\\\u0000-\u001f\u007f]/u.test(path) || path.split("/").some(part => !part || part === "." || part === "..")) || new Set(paths).size !== paths.length) throw new StorageUnavailable(400, "Select one bounded exact set of attachment paths."); + return [...paths].sort(); + } + async function review(principal: string, id: string, paths: string[], sourceId: string | null = null): Promise { + authorize(principal); paths = pathsFor(paths); + if (!validId(id) || sourceId !== null && !validId(sourceId)) throw new StorageUnavailable(400, "Invalid attachment operation identity."); + const selection = JSON.stringify(paths), action = sourceId ? "undo" : "remove"; + const prior = select(id); + if (prior) { if (prior.principal_id !== principal || prior.selection !== selection || prior.action !== action || prior.source_id !== sourceId) throw new StorageUnavailable(409, "Operation identity belongs to another selection."); return recover(prior); } + if (!capability(principal).available) throw new StorageUnavailable(503, "This host cannot remove attachments yet."); + if (reviews.size || active.size) throw new StorageUnavailable(409, "Another attachment review or operation is active."); + const controller = new AbortController(), timeout = setTimeout(() => controller.abort(), STORAGE_REMOVAL_LIMITS.reviewMilliseconds); reviews.add(controller); + let finish!: () => void; const done = new Promise(resolve => { finish = resolve; }); pendingReviews.add(done); + const check = () => { controller.signal.throwIfAborted(); authorize(principal); }; + try { return await exclusive(controller.signal, `storage-review:${id}`, async () => { + check(); + const supported = paths.filter(storageAttachmentPath); + const facts = supported.length ? await readRemovalFacts(options.boundary, brain, supported, controller.signal, check) : { fingerprint: "", complete: false, scannedFiles: 0, unchecked: ["unsupported selection"], files: [] }; + const covered = await journal.removalCoverage(paths, controller.signal, !!sourceId), source = sourceId ? select(sourceId) : null; + if (sourceId && (!source || source.principal_id !== principal || source.action !== "remove")) throw new StorageUnavailable(404, "Original removal unavailable."); + const sourceReceipt = source ? recover(source) : null, sourceVersions = source ? JSON.parse(source.private_versions) as PrivateFile[] : []; + const files: StorageRemovalFile[] = [], versions: PrivateFile[] = []; let selectedFileBytes = 0; + for (const [index, path] of paths.entries()) { + check(); const fact = facts.files.find(value => value.path === path); let reason = protectedReason(path), version: StorageAttachmentVersion | null = null, retained: JournalVersion | null = null; + try { + if (source) { + const original = sourceVersions.find(value => value.path === path), removed = sourceReceipt!.items.find(value => value.path === path); + version = original?.version ?? null; retained = original?.retained ?? null; + if (!version || !retained || removed?.state !== "removed" || !journal.hasRetainedVersion(source.journal_turn_id, retained)) reason ??= "retained_detail_expired_or_unavailable"; + else if (!storageAttachmentVacant(brain, path, version)) reason ??= "replacement_or_parent_changed"; + } else if (!reason) { version = await inspectStorageAttachment(brain, path, controller.signal); } + } catch (error) { reason ??= error instanceof StorageAttachmentRefusal ? error.reason : "version_unavailable"; } + // A missing undo target has no current file metadata; its ordinary ignore rules still apply at admission. + if (!covered[index]) reason ??= "journal_coverage_unavailable"; + if (!facts.complete || !fact || fact.truncated || fact.git === "unknown") reason ??= "consequences_incomplete"; + if (fact?.git === "ignored") reason ??= "ignored_content_not_covered"; + if (!reason && version && selectedFileBytes + version.bytes > STORAGE_REMOVAL_LIMITS.totalBytes) reason = "batch_byte_bound"; + if (!reason && version) selectedFileBytes += version.bytes; + files.push({ path, bytes: version?.bytes ?? null, hash: version?.hash ?? null, eligible: !reason && !!version, reason, git: fact?.git ?? "unknown", references: fact?.references.map(({ source, certainty }) => ({ source, certainty })) ?? [], recovery: reason ? "unavailable" : source ? "retained" : "requires-retention" }); + versions.push({ path, version, retained, consequences: factIdentity(facts, path) }); + } + check(); const now = Date.now(), proposed = { id, action, sourceId, observedAt: now, expiresAt: now + REVIEW_TTL, files, complete: facts.complete, unchecked: facts.unchecked, selectedFileBytes } as Omit; + const reviewed: StorageRemovalReview = { ...proposed, fingerprint: digest({ proposed, versions }) }; + const result: StorageRemovalReceipt = { review: reviewed, state: "review", requestedAt: now, startedAt: null, completedAt: null, reason: null, + items: files.map(file => ({ path: file.path, state: file.eligible ? "not-started" : "protected", reason: file.reason, undoAvailable: false, undoReason: "no_confirmed_removal" })), + before: null, after: null, index: "not-requested", commit: "not-requested", sync: "not-requested", retained: { journal: "subject-to-retention", git: "unchanged-by-operation", backups: "not-measured" } }; + journal.writeOperation(() => { + if ((db.query("SELECT count(*) AS total FROM turn_journal_storage_operations WHERE brain=?").get(brain) as {total: number}).total >= STORAGE_REMOVAL_LIMITS.receipts) throw new StorageUnavailable(503, "Attachment receipt capacity reached; existing receipts remain readable."); + db.query("INSERT INTO turn_journal_storage_operations (id,brain,principal_id,action,source_id,selection,fingerprint,journal_turn_id,state,pid,pid_start,requested_at,receipt,private_versions) VALUES (?,?,?,?,?,?,?,?,'review',?,?,?,?,?)") + .run(id, brain, principal, action, sourceId, selection, reviewed.fingerprint, `storage_${id}`, SELF.pid, SELF.start, now, encode(result), encode(versions)); + }); + return result; + }); } finally { clearTimeout(timeout); reviews.delete(controller); pendingReviews.delete(done); finish(); } + } + async function execute(row: Row, controller: AbortController) { + const result = JSON.parse(row.receipt) as StorageRemovalReceipt, versions = JSON.parse(row.private_versions) as PrivateFile[]; + const timeout = setTimeout(() => controller.abort(), STORAGE_REMOVAL_LIMITS.milliseconds); + const check = () => { controller.signal.throwIfAborted(); authorize(row.principal_id); if ((db.query("SELECT cancel_requested FROM turn_journal_storage_operations WHERE id=?").get(row.id) as {cancel_requested: number}).cancel_requested) throw Error("cancelled"); }; + const revoked = setInterval(() => { try { check(); } catch { controller.abort(); } }, 100); + let journalStarted = false, effects = false, unresolved = false; + try { + check(); result.before = storageObservation(await options.storage.measure(row.principal_id, controller.signal), ["content", "git", "journal", "database"]); + await exclusive(controller.signal, `storage-remove:${row.id}`, async () => { + check(); if (Date.now() >= result.review.expiresAt) throw Error("review_expired"); + const eligible = result.review.files.filter(file => file.eligible).map(file => file.path); + const facts = await readRemovalFacts(options.boundary, brain, eligible, controller.signal, check); + const covered = await journal.removalCoverage(eligible, controller.signal, row.action === "undo"); + check(); + await journal.begin(row.journal_turn_id, { sessionId: null, actor: "user", model: null, profile: "strict", policyHash: null }, () => controller.abort(), controller.signal); journalStarted = true; + result.state = "running"; result.startedAt = Date.now(); save(row, result); + for (const item of result.items) { + if (item.state !== "not-started") continue; + check(); const privateFile = versions.find(file => file.path === item.path)!; + const reason = protectedReason(item.path); + if (reason) { item.state = "protected"; item.reason = reason; save(row, result); continue; } + if (privateFile.consequences !== factIdentity(facts, item.path)) { item.state = "changed"; item.reason = "consequences_changed"; save(row, result); continue; } + try { + if (!covered[eligible.indexOf(item.path)]) throw new StorageAttachmentRefusal("protected"); + if (row.action === "remove") { + const current = await inspectStorageAttachment(brain, item.path, controller.signal); + if (JSON.stringify(current) !== JSON.stringify(privateFile.version)) throw new StorageAttachmentRefusal("changed"); + privateFile.retained = await journal.retainBefore(row.journal_turn_id, item.path, controller.signal); + if (!privateFile.retained || privateFile.retained.hash !== current.hash || privateFile.retained.bytes !== current.bytes || privateFile.retained.mode !== current.mode || !journal.hasRetainedVersion(row.journal_turn_id, privateFile.retained)) throw Error("retained_before_unavailable"); + } else { + const source = select(row.source_id!); + if (!source || !privateFile.retained || !journal.hasRetainedVersion(source.journal_turn_id, privateFile.retained)) throw Error("retained_detail_expired_or_unavailable"); + if (!storageAttachmentVacant(brain, item.path, privateFile.version!)) throw new StorageAttachmentRefusal("occupied"); + } + check(); if (protectedReason(item.path)) throw new StorageAttachmentRefusal("protected"); + item.state = "applying"; save(row, result, versions); // Durable retained identity and item intent precede the first file effect. + if (row.action === "remove") await removeStorageAttachment(brain, item.path, privateFile.version!, controller.signal, check); + else journal.withRetainedRemoval(select(row.source_id!)!.journal_turn_id, privateFile.retained!, copy => restoreStorageAttachment(brain, item.path, privateFile.version!, copy, check)); + effects = true; item.state = row.action === "remove" ? "removed" : "restored"; item.reason = null; + save(row, result, versions); // An acknowledgement failure cannot authorize another file. + await journal.observe(); + } catch (error) { + if (item.state === "applying" || item.state === "removed" || item.state === "restored") { unresolved = true; throw error; } + item.state = error instanceof StorageAttachmentRefusal && ["changed", "occupied"].includes(error.reason) ? "changed" : error instanceof StorageAttachmentRefusal && error.reason === "protected" ? "protected" : "failed"; + item.reason = error instanceof StorageAttachmentRefusal ? error.reason : (error as {reason?: string}).reason === "journal_capacity" ? "journal_capacity" : ["ENOSPC", "EDQUOT"].includes((error as {code?: string}).code ?? "") ? "recovery_no_space" : "recovery_or_file_unavailable"; + save(row, result, versions); + } + } + await journal.finish(row.journal_turn_id); journalStarted = false; + if (journal.summary(row.journal_turn_id).state !== "complete") unresolved = true; + }); + result.state = unresolved ? "unknown" : result.items.every(item => ["removed", "restored"].includes(item.state)) ? "complete" : "partial"; + } catch { + result.state = unresolved || result.items.some(item => item.state === "applying") ? "unknown" : effects ? "partial" : controller.signal.aborted ? "cancelled" : "refused"; + result.reason = result.state === "unknown" ? "outcome_requires_reconciliation" : controller.signal.aborted ? "cancelled_or_revoked" : "operation_unavailable"; + } finally { + clearTimeout(timeout); clearInterval(revoked); + if (journalStarted) { try { await journal.finish(row.journal_turn_id); } catch { result.state = "unknown"; result.reason = "journal_reconciliation_unavailable"; } } + if (effects && !closed && options.authorized(row.principal_id)) { + const after = new AbortController(), deadline = setTimeout(() => after.abort(), 36_000); + try { + const refreshed = await options.boundary.run({ argv: [process.execPath, "--config=/dev/null", "--no-env-file", MAINTENANCE_ENTRY], env: subprocessEnv("brainCli", { NO_COLOR: "1" }), credentials: false, gate: "shared", purpose: "storage-index-refresh", outputAreas: ["brain.db", "brain.db-wal", "brain.db-shm", "brain.db-journal"], stdin: JSON.stringify({ op: "index", root: brain }), timeoutMs: 30_000, outputBytes: 262144, signal: after.signal, assertAuthorized: () => authorize(row.principal_id) }); + result.index = refreshed.exitCode === 0 && !refreshed.timedOut && JSON.parse(refreshed.stdout).ok === true ? "complete" : "unavailable"; + } catch { result.index = "unavailable"; } finally { clearTimeout(deadline); } + } + if (result.startedAt && !closed && options.authorized(row.principal_id)) { + const after = new AbortController(), deadline = setTimeout(() => after.abort(), 36_000); + try { result.after = storageObservation(await options.storage.measure(row.principal_id, after.signal), ["content", "git", "journal", "database"]); } catch { result.after = null; } finally { clearTimeout(deadline); } + } + for (const item of result.items) if (item.state === "applying") { item.state = "unknown"; item.reason = "outcome_requires_reconciliation"; } + result.completedAt = Date.now(); + try { save(row, result, versions); } catch { /* Original per-item intent survives; a later read returns unknown without replay. */ } + } + } + return { + capability, review, receipt, + start(principal: string, id: string, fingerprint: string) { + authorize(principal); const row = select(id); + if (!row || row.principal_id !== principal) throw new StorageUnavailable(404, "Attachment review unavailable."); + if (fingerprint !== row.fingerprint) throw new StorageUnavailable(409, "Review identity changed."); + const result = recover(row); + if (row.state !== "review") return result; + if (!capability(principal).available || Date.now() >= result.review.expiresAt || !result.review.files.some(file => file.eligible)) throw new StorageUnavailable(409, "Refresh the attachment review before removal."); + journal.writeOperation(() => { + if (db.query("SELECT 1 FROM turn_journal_storage_operations WHERE brain=? AND state IN ('waiting','running')").get(brain)) throw new StorageUnavailable(409, "Another attachment operation is active."); + result.state = "waiting"; + const changed = db.query("UPDATE turn_journal_storage_operations SET state='waiting',pid=?,pid_start=?,receipt=? WHERE id=? AND state='review'").run(SELF.pid, SELF.start, encode(result), id); + if (changed.changes !== 1) throw new StorageUnavailable(409, "Attachment operation already admitted."); + }); + const controller = new AbortController(); runningHere.add(key(id)); + const pending = Promise.resolve().then(() => execute(select(id)!, controller)).finally(() => { active.delete(id); runningHere.delete(key(id)); }); + active.set(id, { controller, promise: pending }); + return result; + }, + cancel(principal: string, id: string) { const result = receipt(principal, id); if (["waiting", "running"].includes(result.state)) journal.writeOperation(() => db.query("UPDATE turn_journal_storage_operations SET cancel_requested=1 WHERE id=?").run(id)); active.get(id)?.controller.abort(); return result; }, + async idle() { await Promise.allSettled([...active.values()].map(value => value.promise)); }, + async close() { closed = true; for (const controller of reviews) controller.abort(); for (const value of active.values()) value.controller.abort(); await Promise.allSettled([...pendingReviews, ...[...active.values()].map(value => value.promise)]); }, + }; +} +export type StorageRemoval = ReturnType; diff --git a/packages/ui-server/tests/fixtures/storage-removal-worker.ts b/packages/ui-server/tests/fixtures/storage-removal-worker.ts new file mode 100644 index 0000000000..863ea36271 --- /dev/null +++ b/packages/ui-server/tests/fixtures/storage-removal-worker.ts @@ -0,0 +1,26 @@ +import { existsSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { storageRemovalFixture } from "../helpers/storage-removal-fixture.js"; +import { createUiDb } from "../../src/db/client.js"; +import { createWriterGate } from "../../src/brain/writer-gate.js"; +const [root,phase,id,marker]=process.argv.slice(2) as [string,string,string,string]; +if(phase==="writer") { + const db=createUiDb(join(root,"ui.db")),gate=createWriterGate(db,join(root,"brain"),{pollMs:1}),hold=await gate.acquire("shared","turn:athena",{turn:true}); + try { writeFileSync(marker,"holding");while(!existsSync(marker+"-release"))await Bun.sleep(5);writeFileSync(join(root,"brain/attachments/chart.pdf"),"Athena's concurrent revision"); } + finally{hold.release();db.close();} +} else { + const f=storageRemovalFixture(root),target=join(f.brain,"attachments/chart.pdf"); + const pause=()=>{writeFileSync(marker,phase);Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);}; + const retain=f.journal.retainBefore.bind(f.journal),write=f.journal.writeOperation.bind(f.journal); + if(phase==="after-retain")f.journal.retainBefore=async(...args)=>{const result=await retain(...args);pause();return result;}; + f.journal.writeOperation=(apply:()=>T):T=>{ + const state=()=>{const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(id) as {receipt:string}|null;return row?JSON.parse(row.receipt).items[0].state:null;}; + if(phase==="after-unlink"&&state()==="applying"&&!existsSync(target))pause(); + const result=write(apply); + if(phase==="after-intent"&&state()==="applying")pause(); + if(phase==="after-ack"&&state()==="removed")pause(); + return result; + }; + try{const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();throw Error("Crash boundary was not reached");} + finally{await f.close();} +} diff --git a/packages/ui-server/tests/helpers/storage-removal-fixture.ts b/packages/ui-server/tests/helpers/storage-removal-fixture.ts new file mode 100644 index 0000000000..5d470aa4c3 --- /dev/null +++ b/packages/ui-server/tests/helpers/storage-removal-fixture.ts @@ -0,0 +1,32 @@ +/** Real filesystem/journal/gate harness; the evaluation boundary is replaced at its shared request method. */ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { inspectStorageAttachmentFacts } from "@schlessera/brain/storage"; +import { createKeyedLock } from "@schlessera/brain-ui-sdk/server"; +import { createUiDb } from "../../src/db/client.js"; +import { createTurnJournal } from "../../src/brain/turn-journal.js"; +import { createWriterGate } from "../../src/brain/writer-gate.js"; +import { createStorageCoordinator } from "../../src/storage/coordinator.js"; +import { createStorageRemoval } from "../../src/storage/removal.js"; +import type { BrainExecution, HostBoundary } from "../../src/brain/host-boundary.js"; +import { createNativeJournalFixture } from "./native-journal-fixture.js"; + +export function storageRemovalFixture(existingRoot?: string, journalOptions: { capBytes?: number; now?: () => number } = {}) { + const root = existingRoot ?? mkdtempSync(join(tmpdir(), "odysseus-removal-")), brain = join(root, "brain"), dbPath = join(root, "ui.db"); + mkdirSync(join(brain, "attachments"), { recursive: true }); + const original = Buffer.from([37, 80, 68, 70, 0, 255, 13, 10]); + if (!existingRoot) { writeFileSync(join(brain, "attachments/chart.pdf"), original); writeFileSync(join(brain, "attachments/score.pdf"), original); writeFileSync(join(brain, "voyage.md"), "[Harbor](attachments/chart.pdf)\n"); } + const db = createUiDb(dbPath), native = createNativeJournalFixture(brain, root), journal = createTurnJournal(db, { brainPath: brain, dbPath, objectsRoot: join(root, "objects"), nativeManager: native.manager, ...journalOptions }), gate = createWriterGate(db, brain, { pollMs: 1 }); + let allowed = true; const authorized = (principal: string) => allowed && principal === "odysseus"; + const calls: BrainExecution[] = []; + const boundary = { isolated: () => true, status: () => ({ hostExecution: "worker", available: true, profile: "strict", protectedPaths: [], identity: "shared_uid", configurationView: "startup" }), async run(execution: BrainExecution) { + calls.push(execution); execution.assertAuthorized?.(); execution.signal?.throwIfAborted(); + const request = JSON.parse(execution.stdin!) as {op: string; paths?: string[]}; + return { exitCode: 0, stderr: "", stdout: JSON.stringify(request.op === "storage-facts" ? { ok: true, facts: inspectStorageAttachmentFacts(brain, request.paths!, execution.signal) } : { ok: true }) }; + } } satisfies Pick; + const storage = createStorageCoordinator({ brainPath: brain, stores: [{path:join(root,"objects"),label:"Retained history",category:"journal",owner:"app",recursive:true}], authorized }); + const options = { brainPath: brain, boundary, journal, gate, lock: createKeyedLock(), storage, authorized, protectedPaths: [] as string[] }; + const removal = createStorageRemoval(db, options); + return { root, brain, dbPath, original, db, native, journal, gate, storage, options, removal, calls, revoke() { allowed = false; }, allow() { allowed = true; }, async close() { await removal.close(); await storage.close(); await journal.close(); await native.close(); db.close(); if (!existingRoot) rmSync(root, {recursive:true,force:true}); } }; +} diff --git a/packages/ui-server/tests/http-inventory.test.ts b/packages/ui-server/tests/http-inventory.test.ts index 3a89c88b46..6f4f2cf85c 100644 --- a/packages/ui-server/tests/http-inventory.test.ts +++ b/packages/ui-server/tests/http-inventory.test.ts @@ -20,7 +20,7 @@ test("real app method/path inventory equals the complete classified HTTP specifi expect(proseCounts).toEqual([declared.length, declared.length]); } const middleware = [...new Set(t.routes.filter((r) => r.method === "ALL").map((r) => r.path))].sort(); - expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*", "/api/storage*", "/api/storage/maintenance*"]); + expect(middleware).toEqual(["/*", "/api/*", "/api/auth/passkey/*", "/api/auth/principals/*", "/api/internal/inbox/poke", "/api/sessions/:sessionId/turns/:turnId/revert*", "/api/storage*", "/api/storage/maintenance*", "/api/storage/removals*"]); expect(t.routes.filter((r) => r.method === "ALL" && r.path === "/*")).toHaveLength(staticRoot ? 3 : 2); expect(t.routes.some((r) => r.method === "HEAD" || r.method === "OPTIONS")).toBe(false); } finally { await t.close(); } diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index 653571d3d3..5910fc8495 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -16,6 +16,12 @@ type Check = { path?: string; init?: RequestInit; status?: number; body?: object // More detailed existing behavior checks are linked by the coverage matrix; // this map is cross-checked with the spec and the actual Hono mounts. const checks: Record = { + "GET /api/storage/removals": { verify: async r => { const value = await r.json(); expect(value).toHaveProperty("available"); expect(value.limits.items).toBe(32); } }, + "POST /api/storage/removals/reviews": { init: json("POST", { id: "odysseus-mounted-review", paths: [] }), status: 400, body: {error:"Select bounded attachment paths and a new review identity."} }, + "POST /api/storage/removals/undo-reviews": { init: json("POST", {}), status: 400, body: {error:"Select retained originals from one removal and a new Undo identity."} }, + "POST /api/storage/removals": { init: json("POST", {id:"odysseus-mounted-missing",fingerprint:"a".repeat(64)}), status:404, body:{error:"Attachment review unavailable."} }, + "GET /api/storage/removals/:id": {path:"/api/storage/removals/missing",status:404,body:{error:"Attachment receipt unavailable."}}, + "DELETE /api/storage/removals/:id": {path:"/api/storage/removals/missing",init:{method:"DELETE"},status:404,body:{error:"Attachment receipt unavailable."}}, "GET /api/storage/maintenance": { verify: async r => { const value = await r.json(); expect(value.operations.map((operation: {id: string}) => operation.id)).toEqual(["pack-git", "prune-scratch", "cleanup-staging"]); } }, "POST /api/storage/maintenance": { init: json("POST", { id: "odysseus-mounted-001", operation: "pack-git" }), status: 202, body: { id: "odysseus-mounted-001", operation: "pack-git" } }, "GET /api/storage/maintenance/:id": { path: "/api/storage/maintenance/missing", status: 404, body: { error: "Maintenance receipt unavailable." } }, diff --git a/packages/ui-server/tests/storage-removal-journal.test.ts b/packages/ui-server/tests/storage-removal-journal.test.ts new file mode 100644 index 0000000000..41f4d1b063 --- /dev/null +++ b/packages/ui-server/tests/storage-removal-journal.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from "bun:test"; +import { createNativeJournalFixture } from "./helpers/native-journal-fixture"; +import { createTurnJournal } from "../src/brain/turn-journal"; +import { createUiDb } from "../src/db/client"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, openSync, closeSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +test("the charged journal retains and verifies selected original bytes before removal without widening ignore rules", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-journal-")), brain = join(root, "brain"), dbPath = join(root, "ui.sqlite"); + mkdirSync(brain); mkdirSync(join(brain, "attachments")); + const original = Buffer.from([37, 80, 68, 70, 0, 255, 13]); writeFileSync(join(brain, "attachments/chart.pdf"), original); + writeFileSync(join(brain, "attachments/ignored.pdf"), "ignored content"); writeFileSync(join(brain, ".gitignore"), "attachments/ignored.pdf\n"); + const db = createUiDb(dbPath), native = createNativeJournalFixture(brain, root), journal = createTurnJournal(db, { brainPath: brain, dbPath, objectsRoot: join(root, "objects"), nativeManager: native.manager }); + try { + await journal.begin("storage-odysseus", { sessionId: "storage", actor: "user", model: null, profile: "strict", policyHash: null }, () => {}); + const before = await journal.retainBefore("storage-odysseus", "attachments/chart.pdf"); + expect(before?.hash, "STORAGE_RETAINED_BEFORE_UNLINK").toHaveLength(64); + expect(readFileSync(join(brain, "attachments/chart.pdf"))).toEqual(original); + expect(journal.hasRetainedVersion("storage-odysseus", before!)).toBe(true); + const fd = openSync(join(root, "independent-copy"), "wx"); + try { journal.withRetainedVersion(before, copy => copy(fd)); } finally { closeSync(fd); } + expect(readFileSync(join(root, "independent-copy")), "STORAGE_RETAINED_ORIGINAL_CONTROL").toEqual(original); + expect(await journal.retainBefore("storage-odysseus", "attachments/ignored.pdf"), "STORAGE_NO_IGNORE_EXPANSION").toBeNull(); + rmSync(join(brain, "attachments/chart.pdf")); await journal.finish("storage-odysseus"); + expect(journal.hasRetainedVersion("storage-odysseus", before!)).toBe(true); + expect(journal.details("storage", "storage-odysseus")?.changes[0]?.before?.hash).toBe(before?.hash); + } finally { await journal.close(); await native.close(); db.close(); rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/ui-server/tests/storage-removal-recovery.test.ts b/packages/ui-server/tests/storage-removal-recovery.test.ts new file mode 100644 index 0000000000..41c35f6779 --- /dev/null +++ b/packages/ui-server/tests/storage-removal-recovery.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { storageRemovalFixture } from "./helpers/storage-removal-fixture.js"; +const entry=join(import.meta.dir,"fixtures/storage-removal-worker.ts"); +async function reached(marker:string,child:ReturnType) { + for(let index=0;index<1000&&!existsSync(marker)&&child.exitCode===null;index++)await Bun.sleep(5); + expect(existsSync(marker),"STORAGE_REMOVAL_REAL_CHILD_BOUNDARY").toBe(true); +} +for(const phase of ["after-retain","after-intent","after-unlink","after-ack"])test(`SIGKILL ${phase} preserves the original identity and never replays a removal`,async()=>{ + const f=storageRemovalFixture(),id=`odysseus-crash-${phase}`,marker=join(f.root,`reached-${phase}`),target=join(f.brain,"attachments/chart.pdf"); + const child=Bun.spawn([process.execPath,entry,f.root,phase,id,marker],{stdout:"pipe",stderr:"pipe"}); + try { + await reached(marker,child);child.kill("SIGKILL");await child.exited; + expect(child.signalCode).toBe("SIGKILL"); + const recovered=storageRemovalFixture(f.root); + try { + const result=recovered.removal.receipt("odysseus",id); + expect(result.state,"STORAGE_REMOVAL_CRASH_UNKNOWN").toBe("unknown"); + expect(existsSync(target)).toBe(["after-retain","after-intent"].includes(phase)); + expect(result.items[0]?.state).toBe(phase==="after-ack"?"removed":phase==="after-retain"?"not-started":"unknown"); + expect(result.items[0]?.undoAvailable).toBe(phase==="after-ack"); + if(phase==="after-ack"){ + const undo=await recovered.removal.review("odysseus","odysseus-crash-undo",["attachments/chart.pdf"],id); + recovered.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await recovered.removal.idle(); + expect(readFileSync(target),"STORAGE_CRASH_RETAINED_ORIGINAL_UNDO").toEqual(f.original); + } + writeFileSync(target,"Penelope's later replacement"); + expect(recovered.removal.start("odysseus",id,result.review.fingerprint).state).toBe("unknown");await recovered.removal.idle(); + expect(readFileSync(target,"utf8"),"STORAGE_CRASH_NO_REPLAY_ON_REPLACEMENT").toBe("Penelope's later replacement"); + } finally{await recovered.close();} + }finally{child.kill("SIGKILL");await child.exited;await f.close();} +}); + +test("an actual other-process writer finishes before removal validation while nested shared work remains admissible",async()=>{ + const f=storageRemovalFixture(),id="odysseus-writer-001",marker=join(f.root,"writer-holding"); + const reviewed=await f.removal.review("odysseus",id,["attachments/chart.pdf"]); + const child=Bun.spawn([process.execPath,entry,f.root,"writer",id,marker],{stdout:"pipe",stderr:"pipe"}); + try { + await reached(marker,child);f.removal.start("odysseus",id,reviewed.review.fingerprint); + for(let i=0;i<500&&!(f.db.query("SELECT count(*) AS n FROM writer_gate_turn_waiters").get() as {n:number}).n;i++)await Bun.sleep(5); + expect((f.db.query("SELECT count(*) AS n FROM writer_gate_turn_waiters").get() as {n:number}).n,"STORAGE_REMOVAL_QUEUED_EXCLUSIVE").toBe(1); + expect(readFileSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_REMOVAL_WAITS_FOR_LIVE_WRITER").toEqual(f.original); + const nested=await f.gate.acquire("shared","nested-index",{signal:AbortSignal.timeout(2000)});nested.release(); + writeFileSync(marker+"-release","finish");expect(await child.exited,await new Response(child.stderr).text()).toBe(0);await f.removal.idle(); + expect(f.removal.receipt("odysseus",id).items[0],"STORAGE_REMOVAL_VALIDATES_AFTER_WRITER").toMatchObject({state:"changed"}); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"),"utf8")).toBe("Athena's concurrent revision"); + }finally{child.kill("SIGKILL");await child.exited;await f.close();} +}); diff --git a/packages/ui-server/tests/storage-removal-routes.test.ts b/packages/ui-server/tests/storage-removal-routes.test.ts new file mode 100644 index 0000000000..d3fe429cf3 --- /dev/null +++ b/packages/ui-server/tests/storage-removal-routes.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { httpContractApp } from "./helpers/http-contract-app"; + +test("attachment removal exposes an owner capability without deleting on read", async () => { + const fixture = await httpContractApp(); + try { + const response = await fixture.fetch("/api/storage/removals"); + expect(response.status, "STORAGE_REMOVAL_OWNER_ROUTE").toBe(200); + expect(await response.json()).toHaveProperty("available"); + } finally { await fixture.close(); } +}); + +test("mounted removal enforces human authority, origin, exact review and durable native-worker readback",async()=>{ + const {generateSignedCookie}=await import("hono/cookie"),{createPrincipal}=await import("../src/db/principals"),{mkdirSync,writeFileSync,readFileSync,existsSync}=await import("node:fs"),{join}=await import("node:path"); + const secret="odysseus-removal-route-secret-0123456789",f=await httpContractApp({env:{AUTH_MODE:"password",BRAIN_UI_PASSWORD_HASH:"unused-fixture-hash",COOKIE_SECRET:secret,BRAIN_UI_ALLOW_LOOPBACK_ORIGIN:"1"}}); + try{ + const owner=createPrincipal(f.app.db,{authMethod:"password",label:"Odysseus",ttlSeconds:3600}),agent=createPrincipal(f.app.db,{authMethod:"delegated",createdBy:owner.id,label:"Athena",ttlSeconds:3600}),other=createPrincipal(f.app.db,{authMethod:"password",label:"Penelope",ttlSeconds:3600}); + const cookie=async(id:string)=>(await generateSignedCookie("brain_ui_session",id,secret)).split(";")[0]!,ownerCookie=await cookie(owner.id),agentCookie=await cookie(agent.id),otherCookie=await cookie(other.id); + const request={id:"odysseus-route-removal",paths:["attachments/chart.pdf"]}; + for(const [method,path] of [["GET","/api/storage/removals"],["GET","/api/storage/removals/missing"],["POST","/api/storage/removals/reviews"],["POST","/api/storage/removals/undo-reviews"],["POST","/api/storage/removals"],["DELETE","/api/storage/removals/missing"]]){ + const init={method,headers:{cookie:agentCookie,"content-type":"application/json"},...(method==="POST"?{body:JSON.stringify(request)}:{})}; + expect((await f.fetch(path!,init)).status,"STORAGE_REMOVAL_AGENT_DENIED").toBe(403); + expect((await f.fetch(path!,{...init,headers:{"content-type":"application/json"}})).status).toBe(401); + } + const post=(path:string,body:unknown,headers:Record={})=>f.fetch(path,{method:"POST",headers:{cookie:ownerCookie,"content-type":"application/json",...headers},body:JSON.stringify(body)}); + expect((await post("/api/storage/removals/reviews",{...request,root:"/"})).status).toBe(400); + expect((await post("/api/storage/removals/reviews",request,{origin:"https://troy.example"})).status).toBe(403); + expect((await post("/api/storage/removals/reviews",request,{"content-type":"text/plain"})).status).toBe(415); + expect((await post("/api/storage/removals/reviews",{...request,extra:"x".repeat(141000)})).status).toBe(413); + mkdirSync(join(f.brainPath,"attachments"));const original=Buffer.from([37,80,68,70,0,255,13]);writeFileSync(join(f.brainPath,request.paths[0]!),original);writeFileSync(join(f.brainPath,"notes/arrival.md"),"[Harbor](../attachments/chart.pdf)\n"); + writeFileSync(join(f.brainPath,"attachments/keep.pdf"),"Penelope's unselected original"); + writeFileSync(join(f.brainPath,"brain.config.ts"),'import {rmSync} from "node:fs";rmSync(process.env.BRAIN_WORKER_DATA_ROOT+"/attachments/keep.pdf");export default {};\n'); + const reviewed=await post("/api/storage/removals/reviews",request),review=await reviewed.json(); + expect(reviewed.status,JSON.stringify(review)).toBe(200); + expect(review.review.files[0],"STORAGE_HTTP_CURRENT_NATIVE_REFERENCE").toMatchObject({eligible:true,references:[{source:"notes/arrival.md",certainty:"resolved"}]}); + expect((await post("/api/storage/removals",{id:request.id,fingerprint:"a".repeat(64)})).status).toBe(409); + const confirmed={id:request.id,fingerprint:review.review.fingerprint};expect((await post("/api/storage/removals",confirmed)).status).toBe(202); + let result:any; + for(let i=0;i<300;i++){const response=await f.fetch(`/api/storage/removals/${request.id}`,{headers:{cookie:ownerCookie}});expect(response.status).toBe(200);result=await response.json();if(!["waiting","running"].includes(result.state))break;await Bun.sleep(20);} + expect(result,JSON.stringify(result)).toMatchObject({state:"complete",items:[{state:"removed",undoAvailable:true}]}); + expect(existsSync(join(f.brainPath,request.paths[0]!))).toBe(false); + expect(existsSync(join(f.brainPath,"attachments/keep.pdf")),"STORAGE_INDEX_REFRESH_CANNOT_PUBLISH_UNSELECTED_REMOVAL").toBe(true); + expect(readFileSync(join(f.brainPath,"attachments/keep.pdf"),"utf8")).toBe("Penelope's unselected original"); + expect(result.index).toBe("complete");expect(existsSync(join(f.brainPath,"brain.db"))).toBe(true); + expect(readFileSync(join(f.brainPath,"notes/arrival.md"),"utf8")).toContain("attachments/chart.pdf"); + expect(await(await post("/api/storage/removals",confirmed)).json(),"STORAGE_HTTP_LOST_RESPONSE_READBACK").toEqual(result); + expect((await f.fetch(`/api/storage/removals/${request.id}`,{headers:{cookie:otherCookie}})).status).toBe(404); + f.app.db.query("UPDATE principals SET revoked_at=? WHERE id=?").run(Date.now(),owner.id); + expect((await post("/api/storage/removals",confirmed)).status).toBe(401); + }finally{await f.close();} +}); diff --git a/packages/ui-server/tests/storage-removal-space.test.ts b/packages/ui-server/tests/storage-removal-space.test.ts new file mode 100644 index 0000000000..a32a6836a0 --- /dev/null +++ b/packages/ui-server/tests/storage-removal-space.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +for(const volume of ["objects","brain"] as const)test(`a genuinely full ${volume} volume preserves the source or retained Undo original`,async()=>{ + const root=mkdtempSync(join(tmpdir(),"odysseus-removal-space-"));mkdirSync(join(root,volume)); + const entry=join(import.meta.dir,"helpers/storage-removal-fixture.ts"); + const source=` +import {chmodSync,mkdirSync,writeFileSync,readFileSync,openSync,closeSync,writeSync,statfsSync,existsSync} from "node:fs"; +import {storageRemovalFixture} from ${JSON.stringify(entry)}; +const root=${JSON.stringify(root)},volume=${JSON.stringify(volume)},target=root+"/brain/attachments/chart.pdf"; +chmodSync(root+"/"+volume,0o700);mkdirSync(root+"/brain/attachments",{recursive:true});const original=Buffer.from([37,80,68,70,0,255,13,10]);writeFileSync(target,original); +const f=storageRemovalFixture(root); +const full=()=>{const fd=openSync(root+"/"+volume+"/occupied","wx");let refused=false;try{for(let i=0;i<4096;i++)writeSync(fd,Buffer.alloc(4096));}catch(error){if(error.code!=="ENOSPC")throw error;refused=true;}finally{closeSync(fd);}if(!refused||statfsSync(root+"/"+volume).bavail!==0)throw Error("STORAGE_SPACE_CONTROL_NOT_FULL");}; +try{ +if(volume==="objects")full(); +const review=await f.removal.review("odysseus","odysseus-space-0001",["attachments/chart.pdf"]);f.removal.start("odysseus",review.review.id,review.review.fingerprint);await f.removal.idle(); +if(volume==="objects"){ +if(!readFileSync(target).equals(original))throw Error("STORAGE_ENOSPC_ORIGINAL_MUST_SURVIVE"); +if(f.removal.receipt("odysseus",review.review.id).items[0].state!=="failed")throw Error("STORAGE_ENOSPC_MUST_REFUSE"); +}else{ +if(existsSync(target))throw Error("STORAGE_UNDO_SPACE_CONTROL_NOT_REMOVED");full(); +const undo=await f.removal.review("odysseus","odysseus-space-undo",["attachments/chart.pdf"],review.review.id);f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); +if(existsSync(target))throw Error("STORAGE_UNDO_ENOSPC_PARTIAL_MUST_NOT_SURVIVE"); +if(!f.removal.receipt("odysseus",review.review.id).items[0].undoAvailable)throw Error("STORAGE_UNDO_ENOSPC_RETAINED_ORIGINAL_MUST_SURVIVE"); +} +console.log("STORAGE_REAL_ENOSPC_PRESERVED"); +}finally{await f.close();} +`; + try{ + const child=Bun.spawn(["bwrap","--unshare-user","--unshare-net","--bind","/","/","--size","1048576","--tmpfs",join(root,volume),"--proc","/proc","--dev","/dev",process.execPath,"--config=/dev/null","--no-env-file","-e",source],{stdout:"pipe",stderr:"pipe"}); + const [stdout,stderr,code]=await Promise.all([new Response(child.stdout).text(),new Response(child.stderr).text(),child.exited]); + expect(code,stderr).toBe(0);expect(stdout,"STORAGE_REAL_ENOSPC_PROOF").toContain("STORAGE_REAL_ENOSPC_PRESERVED"); + }finally{rmSync(root,{recursive:true,force:true});} +}); diff --git a/packages/ui-server/tests/storage-removal-worker.test.ts b/packages/ui-server/tests/storage-removal-worker.test.ts new file mode 100644 index 0000000000..a706b58e79 --- /dev/null +++ b/packages/ui-server/tests/storage-removal-worker.test.ts @@ -0,0 +1,32 @@ +import { expect, test } from "bun:test"; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync, lstatSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; +import { createNativeBrainManagerClient } from "@schlessera/brain-ui-sdk/internal"; +import { createNativeAppFixture } from "./helpers/native-app-fixture"; +import { createHostBoundary } from "../src/brain/host-boundary"; +import { createUiDb } from "../src/db/client"; +import { createWriterGate } from "../src/brain/writer-gate"; +import { readRemovalFacts } from "../src/storage/removal-facts"; +function files(root: string): string[] { return readdirSync(root).sort().flatMap(name => { const file=join(root,name);return lstatSync(file).isDirectory()?files(file).map(value=>`${name}/${value}`):[`${name}:${createHash("sha256").update(readFileSync(file)).digest("hex")}`]; }); } + +test("real removal review evaluates Git in a disposable native worker without loading brain config or publishing incidental objects", async () => { + const root=mkdtempSync(join(tmpdir(),"odysseus-removal-worker-")),brain=join(root,"brain"),marker=join(root,"config-executed");mkdirSync(brain); + writeFileSync(join(brain,"brain.config.ts"),`await Bun.write(${JSON.stringify(marker)},"unexpected evaluation");export default {};\n`); + mkdirSync(join(brain,"attachments"));writeFileSync(join(brain,"attachments/chart.pdf"),Buffer.from([0,255,13]));writeFileSync(join(brain,"voyage.md"),"[Chart](attachments/chart.pdf)\n"); + const git=(...args:string[])=>{const result=Bun.spawnSync(["git","-C",brain,...args],{env:{...process.env,GIT_CONFIG_GLOBAL:"/dev/null",GIT_CONFIG_NOSYSTEM:"1"},stdout:"pipe",stderr:"pipe"});expect(result.exitCode,result.stderr.toString()).toBe(0);return result.stdout.toString();}; + git("init");git("add","attachments/chart.pdf"); + const native=createNativeAppFixture(brain),dbPath=join(root,"ui.sqlite"),db=createUiDb(dbPath),gate=createWriterGate(db,brain,{pollMs:1}); + const boundary=createHostBoundary({brainPath:brain,dbPath,gate:()=>gate,history:()=>true,nativeManager:createNativeBrainManagerClient(native.nativeBrain)}); + try { + const before=files(join(brain,".git")),hold=await gate.acquire("exclusive","storage-review-proof"); + let facts; + try{facts=await readRemovalFacts(boundary,brain,["attachments/chart.pdf"],new AbortController().signal,()=>{});}finally{hold.release();} + expect(facts.files[0],"STORAGE_REAL_WORKER_REFERENCE_FACTS").toMatchObject({git:"tracked",references:[{source:"voyage.md",certainty:"resolved"}]}); + expect(existsSync(marker),"STORAGE_REVIEW_NO_BRAIN_CONFIG").toBe(false); + expect(files(join(brain,".git")),"STORAGE_REVIEW_NO_INCIDENTAL_GIT_PUBLICATION").toEqual(before); + const output=await boundary.run({argv:[process.execPath,"--config=/dev/null","--no-env-file","-e",'await Bun.write(process.env.BRAIN_WORKER_DATA_ROOT+"/unexpected.pdf","unexpected write");console.log("read complete");'],env:process.env,credentials:false,gate:"shared",purpose:"storage-read-discard-proof",publish:false,outputBytes:1024,timeoutMs:5000}); + expect(output.exitCode,output.stderr).toBe(0);expect(existsSync(join(brain,"unexpected.pdf")),"STORAGE_REVIEW_DISCARDS_WORKER_WRITES").toBe(false); + } finally{db.close();await native.close();rmSync(root,{recursive:true,force:true});} +}); diff --git a/packages/ui-server/tests/storage-removal.test.ts b/packages/ui-server/tests/storage-removal.test.ts new file mode 100644 index 0000000000..e3b045c394 --- /dev/null +++ b/packages/ui-server/tests/storage-removal.test.ts @@ -0,0 +1,140 @@ +import { expect, test } from "bun:test"; +import { readFileSync, writeFileSync, existsSync, linkSync, symlinkSync, chmodSync } from "node:fs"; +import { join } from "node:path"; +import { createTurnJournal } from "../src/brain/turn-journal.js"; +import { createWriterGate } from "../src/brain/writer-gate.js"; +import { createUiDb } from "../src/db/client.js"; +import { createStorageRemoval } from "../src/storage/removal.js"; +import { storageRemovalFixture } from "./helpers/storage-removal-fixture.js"; + +test("exact removal retains original bytes, keeps references and history facts separate, and Undo survives readback", async () => { + const f = storageRemovalFixture(), id = "odysseus-removal-0001"; + try { + const review = await f.removal.review("odysseus", id, ["attachments/chart.pdf"]); + expect(review.review.files[0], "STORAGE_REMOVAL_REVIEW_COVERAGE").toMatchObject({ eligible: true, recovery: "requires-retention", references: [{source:"voyage.md",certainty:"resolved"}] }); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); + expect(f.removal.start("odysseus", id, review.review.fingerprint).state).toBe("waiting"); await f.removal.idle(); + const result = f.removal.receipt("odysseus", id); + expect(result, "STORAGE_REMOVAL_DURABLE_RESULT").toMatchObject({state:"complete",items:[{state:"removed",undoAvailable:true}],commit:"not-requested",sync:"not-requested",index:"complete"}); + expect(existsSync(join(f.brain,"attachments/chart.pdf"))).toBe(false); + expect(readFileSync(join(f.brain,"voyage.md"),"utf8")).toContain("attachments/chart.pdf"); + expect(result.after?.attribution).toBe("concurrent-writers-possible"); + expect(f.calls.filter(call=>call.purpose==="storage-removal-review").every(call=>call.publish===false && call.credentials===false)).toBe(true); + const db = createUiDb(f.dbPath), journal = createTurnJournal(db, {brainPath:f.brain,dbPath:f.dbPath,objectsRoot:join(f.root,"objects"),nativeManager:f.native.manager}), reopened = createStorageRemoval(db, {...f.options,journal,gate:createWriterGate(db,f.brain,{pollMs:1})}); + try { + expect(reopened.start("odysseus", id, review.review.fingerprint).state, "STORAGE_REMOVAL_NO_REPLAY").toBe("complete"); + const undo = await reopened.review("odysseus", "odysseus-undo-0001", ["attachments/chart.pdf"], id); + expect(undo.review.files[0], "STORAGE_UNDO_CURRENT_REVIEW").toMatchObject({eligible:true,recovery:"retained"}); + reopened.start("odysseus",undo.review.id,undo.review.fingerprint); await reopened.idle(); + expect(reopened.receipt("odysseus",undo.review.id), "STORAGE_UNDO_RECEIPT").toMatchObject({state:"complete",items:[{state:"restored"}]}); + expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_UNDO_INDEPENDENT_ORIGINAL_BYTES").toEqual(f.original); + } finally { await reopened.close(); await journal.close(); db.close(); } + } finally { await f.close(); } +}); + +test("changed bytes keep a mixed batch partial, changed consequences refuse, and stale Undo preserves replacements", async () => { + const f = storageRemovalFixture(); + try { + const review = await f.removal.review("odysseus", "odysseus-mixed-0001", ["attachments/chart.pdf","attachments/score.pdf"]); + writeFileSync(join(f.brain,"attachments/score.pdf"),"Athena's revised score"); + f.removal.start("odysseus",review.review.id,review.review.fingerprint); await f.removal.idle(); + expect(f.removal.receipt("odysseus",review.review.id), "STORAGE_REMOVAL_EXACT_REVIEWED_VERSION").toMatchObject({state:"partial",items:[{state:"removed"},{state:"changed"}]}); + expect(readFileSync(join(f.brain,"attachments/score.pdf"),"utf8")).toBe("Athena's revised score"); + const undo = await f.removal.review("odysseus","odysseus-stale-undo",["attachments/chart.pdf"],review.review.id); + writeFileSync(join(f.brain,"attachments/chart.pdf"),"Penelope's replacement"); + f.removal.start("odysseus",undo.review.id,undo.review.fingerprint); await f.removal.idle(); + expect(f.removal.receipt("odysseus",undo.review.id).items[0]?.state, "STORAGE_UNDO_NEVER_OVERWRITES").toBe("changed"); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"),"utf8")).toBe("Penelope's replacement"); + const reference = await f.removal.review("odysseus","odysseus-ref-000001",["attachments/score.pdf"]); + writeFileSync(join(f.brain,"new-voyage.md"),"[New score](attachments/score.pdf)\n"); + f.removal.start("odysseus",reference.review.id,reference.review.fingerprint); await f.removal.idle(); + expect(f.removal.receipt("odysseus",reference.review.id).items[0], "STORAGE_REMOVAL_CONSEQUENCES_RECHECKED").toMatchObject({state:"changed",reason:"consequences_changed"}); + expect(existsSync(join(f.brain,"attachments/score.pdf"))).toBe(true); + } finally { await f.close(); } +}); + +test("ignored, protected, linked, executable and uncovered attachments never become eligible", async () => { + const f = storageRemovalFixture(); + try { + writeFileSync(join(f.brain,".gitignore"),"attachments/chart.pdf\n"); + linkSync(join(f.brain,"attachments/score.pdf"),join(f.brain,"attachments/alias.pdf")); + symlinkSync("score.pdf",join(f.brain,"attachments/link.pdf")); + writeFileSync(join(f.brain,"attachments/exec.pdf"),"executable"); chmodSync(join(f.brain,"attachments/exec.pdf"),0o700); + f.options.protectedPaths.push("attachments/private.pdf"); writeFileSync(join(f.brain,"attachments/private.pdf"),"protected"); + const review = await f.removal.review("odysseus","odysseus-protected-1",["attachments/chart.pdf","attachments/score.pdf","attachments/link.pdf","attachments/exec.pdf","attachments/private.pdf"]); + expect(review.review.files.every(file=>!file.eligible), "STORAGE_REMOVAL_PROTECTED_SELECTION").toBe(true); + expect(()=>f.removal.start("odysseus",review.review.id,review.review.fingerprint)).toThrow("Refresh"); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); + } finally { await f.close(); } +}); + +test("authority, item-intent storage and missing retained bytes each stop before unlink", async () => { + const f = storageRemovalFixture(); + try { + const review = await f.removal.review("odysseus","odysseus-revoked-01",["attachments/chart.pdf"]); + const hold = await f.gate.acquire("shared","turn:athena",{turn:true}); + f.removal.start("odysseus",review.review.id,review.review.fingerprint); f.revoke(); hold.release(); await f.removal.idle(); f.allow(); + expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_CURRENT_AUTHORITY").toEqual(f.original); + const blocked = await f.removal.review("odysseus","odysseus-intent-001",["attachments/chart.pdf"]); + f.db.exec("CREATE TRIGGER refuse_removal_intent BEFORE UPDATE ON turn_journal_storage_operations WHEN json_extract(NEW.receipt,'$.items[0].state')='applying' BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); + f.removal.start("odysseus",blocked.review.id,blocked.review.fingerprint); await f.removal.idle(); + expect(readFileSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_DURABLE_ITEM_INTENT").toEqual(f.original); + f.db.exec("DROP TRIGGER refuse_removal_intent"); + const unavailable = await f.removal.review("odysseus","odysseus-retain-001",["attachments/chart.pdf"]); + f.db.exec("CREATE TRIGGER refuse_original BEFORE INSERT ON turn_journal_reservations WHEN NEW.kind='copy' BEGIN SELECT RAISE(ABORT,'SQLITE_FULL'); END"); + f.removal.start("odysseus",unavailable.review.id,unavailable.review.fingerprint); await f.removal.idle(); + expect(existsSync(join(f.brain,"attachments/chart.pdf")), "STORAGE_REMOVAL_RETAIN_BEFORE_UNLINK").toBe(true); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); + } finally { await f.close(); } +}); + +test("cancellation completes the current item acknowledgement and leaves later items not started", async () => { + const f=storageRemovalFixture(),id="odysseus-cancel-001"; + try { + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf","attachments/score.pdf"]); + const write=f.journal.writeOperation.bind(f.journal);let cancelled=false; + f.journal.writeOperation=(apply:()=>T):T=>{const result=write(apply);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(id) as {receipt:string}|null; + if(!cancelled&&row&&JSON.parse(row.receipt).items[0].state==="removed"){cancelled=true;f.removal.cancel("odysseus",id);}return result;}; + f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + expect(f.removal.receipt("odysseus",id),"STORAGE_CANCEL_PRESERVES_ACK_AND_UNSTARTED").toMatchObject({state:"partial",items:[{state:"removed",undoAvailable:true},{state:"not-started"}]}); + expect(readFileSync(join(f.brain,"attachments/score.pdf"))).toEqual(f.original); + }finally{await f.close();} +}); + +test("expired detail hides Undo and changed ignore policy cannot expand recovery coverage",async()=>{ + const f=storageRemovalFixture(),id="odysseus-expiry-001"; + try{ + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + const undo=await f.removal.review("odysseus","odysseus-ignore-undo",["attachments/chart.pdf"],id); + writeFileSync(join(f.brain,".gitignore"),"attachments/chart.pdf\n");f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); + expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_UNDO_CURRENT_IGNORE_POLICY").toBe(false); + expect(f.removal.receipt("odysseus",undo.review.id).items[0]?.state).toBe("protected"); + f.db.query("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?").run(`storage_${id}`); + expect(f.removal.receipt("odysseus",id).items[0],"STORAGE_UNDO_EXPIRY_VISIBLE").toMatchObject({undoAvailable:false,undoReason:"retained_detail_expired_or_unavailable"}); + }finally{await f.close();} +}); + + +test("revocation after retained-original admission still refuses the file effect",async()=>{ + const f=storageRemovalFixture(),id="odysseus-late-revoke"; + try{ + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]); + const retain=f.journal.retainBefore.bind(f.journal); + f.journal.retainBefore=async(...args)=>{const original=await retain(...args);f.revoke();return original;}; + f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle();f.allow(); + expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_REMOVAL_AUTHORITY_BEFORE_EFFECT").toBe(true); + expect(readFileSync(join(f.brain,"attachments/chart.pdf"))).toEqual(f.original); + }finally{await f.close();} +}); + +test("Undo leases only originals still owned at the atomic retention admission boundary",async()=>{ + const f=storageRemovalFixture(),id="odysseus-lease-original"; + try{ + const review=await f.removal.review("odysseus",id,["attachments/chart.pdf"]);f.removal.start("odysseus",id,review.review.fingerprint);await f.removal.idle(); + const undo=await f.removal.review("odysseus","odysseus-lease-undo",["attachments/chart.pdf"],id),write=f.journal.writeOperation.bind(f.journal);let expired=false; + f.journal.writeOperation=(apply:()=>T):T=>{const result=write(apply);const row=f.db.query("SELECT receipt FROM turn_journal_storage_operations WHERE id=?").get(undo.review.id) as {receipt:string}|null; + if(!expired&&row&&JSON.parse(row.receipt).items[0].state==="applying"){expired=true;f.db.query("UPDATE turn_journal_turns SET state='expired' WHERE turn_id=?").run(`storage_${id}`);}return result;}; + f.removal.start("odysseus",undo.review.id,undo.review.fingerprint);await f.removal.idle(); + expect(existsSync(join(f.brain,"attachments/chart.pdf")),"STORAGE_UNDO_RETENTION_CHECK_AT_LEASE").toBe(false); + }finally{await f.close();} +}); From 6ed35a3cecfc25d4512c8d648a6fb259c38027b4 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 15:10:53 +0200 Subject: [PATCH 11/44] CONTRACT: feat(storage): inspect attachments without allocating receipts --- .changeset/recoverable-storage-attachments.md | 2 ++ api-report/ui-sdk.txt | 12 ++++++++++++ docs/http-api-coverage.md | 3 ++- docs/http-api.md | 14 +++++++++++--- docs/integration-contract/package-api.md | 2 +- packages/ui-sdk/src/protocol.ts | 2 +- packages/ui-sdk/src/storage-removal.ts | 2 ++ packages/ui-server/src/routes/storage-removal.ts | 6 ++++++ packages/ui-server/src/storage/removal.ts | 16 ++++++++++++---- .../tests/http-supported-mounts.test.ts | 1 + .../tests/storage-removal-routes.test.ts | 8 +++++++- 11 files changed, 57 insertions(+), 11 deletions(-) diff --git a/.changeset/recoverable-storage-attachments.md b/.changeset/recoverable-storage-attachments.md index 5b142ff62f..db5aeb0b17 100644 --- a/.changeset/recoverable-storage-attachments.md +++ b/.changeset/recoverable-storage-attachments.md @@ -5,3 +5,5 @@ --- Add human-owner attachment review, journal-backed removal and version-checked Undo with durable per-item receipts. Originals are verified and retained before effects; changed, protected, ignored and uncovered files are refused. Interrupted requests never replay, replacements are never overwritten, and retention expiry removes the Undo offer. Add the bounded core primitives and protocol contracts used by this application boundary. + +Read-only attachment inspection exposes the same bounded current facts without consuming a durable operation identity; confirmation still requires a fresh review. diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index 8bc960ad4a..b7555f1155 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -321,6 +321,7 @@ export "." (./src/index.ts) type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile + type StorageRemovalInspection type StorageRemovalItem type StorageRemovalReceipt type StorageRemovalReview @@ -796,6 +797,7 @@ export "./client" (./src/client/index.ts) type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile + type StorageRemovalInspection type StorageRemovalItem type StorageRemovalReceipt type StorageRemovalReview @@ -1426,6 +1428,7 @@ export "./protocol" (./src/protocol.ts) type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile + type StorageRemovalInspection type StorageRemovalItem type StorageRemovalReceipt type StorageRemovalReview @@ -2127,6 +2130,7 @@ export "./server" (./src/server/index.ts) type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile + type StorageRemovalInspection type StorageRemovalItem type StorageRemovalReceipt type StorageRemovalReview @@ -9466,6 +9470,14 @@ signatures (every public export and the types it is made of) }[]; recovery: "requires-retention" | "retained" | "unavailable"; } + StorageRemovalInspection (packages/ui-sdk/src/storage-removal.ts) + export interface StorageRemovalInspection { + observedAt: number; + files: StorageRemovalFile[]; + complete: boolean; + unchecked: string[]; + selectedFileBytes: number; + } StorageRemovalItem (packages/ui-sdk/src/storage-removal.ts) export interface StorageRemovalItem { path: string; diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index 5efc69bfa6..613ebb7683 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:128-161`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:129-162`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -29,6 +29,7 @@ row's real mounting check. They do not establish mounting by themselves. | Supported operation | Mounted receipt | Detailed checks | | --- | --- | --- | | `GET /api/storage/removals` | `mounted supported handler: GET /api/storage/removals` | SM | +| `POST /api/storage/removals/inspection` | `mounted supported handler: POST /api/storage/removals/inspection` | SM | | `POST /api/storage/removals/reviews` | `mounted supported handler: POST /api/storage/removals/reviews` | SM | | `POST /api/storage/removals/undo-reviews` | `mounted supported handler: POST /api/storage/removals/undo-reviews` | SM | | `POST /api/storage/removals` | `mounted supported handler: POST /api/storage/removals` | SM | diff --git a/docs/http-api.md b/docs/http-api.md index 75a4d3482b..7bb8f5eaba 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 151 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 152 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -27,6 +27,7 @@ client code has a gap. Source owners are listed after the table. | Method | Path | Class | Purpose | Consumers, existing promise and reason | | --- | --- | --- | --- | --- | | GET | `/api/storage/removals` | S | Read removal capability and latest receipt | Current human-owner authority; exact canonical attachments only. | +| POST | `/api/storage/removals/inspection` | S | Inspect current attachment facts | Read bounded eligibility and consequences without allocating a durable operation. | | POST | `/api/storage/removals/reviews` | S | Review exact attachment paths | Bound versions, consequences and recovery coverage before confirmation. | | POST | `/api/storage/removals/undo-reviews` | S | Review retained originals for Undo | New identity and current absent-target checks. | | POST | `/api/storage/removals` | S | Confirm one reviewed selection | Durable intent and verified originals precede unlink. | @@ -184,7 +185,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 151 declared endpoints are mounted regardless of backend, renderer or +All 152 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1376,7 +1377,7 @@ hosts do not fall back to executing repository code on the host. ## Recoverable attachment removal (additive, #2012) -The six `/api/storage/removals` operations require current owner/ambient human +The seven `/api/storage/removals` operations require current owner/ambient human access and return private, no-store responses. Missing authentication is 401; delegated callers are 403. Receipts belong to their initiating principal and an unknown or another principal's identity returns 404. Origin protections apply @@ -1392,6 +1393,13 @@ admitted. Review bodies are bounded to 140,000 bytes. Paths do not choose roots, executables, commands, sidecars or a wider scope. One review per coordinator runs at a time, with a 60-second deadline and a ten-minute confirmation window. +POST `/inspection` accepts exactly `{paths}` under the same owner, path, worker, +writer-gate, deadline and response bounds. It returns `StorageRemovalInspection`: +current `observedAt`, `files`, `complete`, `unchecked` and `selectedFileBytes`. +It neither allocates a durable receipt nor exposes an identity or fingerprint +that can be confirmed. Inspecting files therefore cannot consume the operation +receipt budget. Selection still requires a fresh durable review before removal. + Review returns a durable `StorageRemovalReceipt` in `review` state. Its review contains the identity, action, optional source identity, opaque fingerprint, observation/expiry times, selected logical file bytes, unchecked scopes and diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index 8807e0b541..f65238b26b 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3278,7 +3278,7 @@ unchecked scopes prevent interpreting an empty reference list as proof of no consumers. Core semantics have no dependency on the UI database. Server contracts and bounds are defined by the [removal HTTP contract](../http-api.md#recoverable-attachment-removal-additive-2012). -`@schlessera/brain-ui-sdk/protocol` exports `StorageRemovalReviewRequest`, +`@schlessera/brain-ui-sdk/protocol` exports `StorageRemovalInspection`, `StorageRemovalReviewRequest`, `StorageRemovalStartRequest`, `StorageRemovalUndoRequest`, `StorageRemovalFile`, `StorageRemovalReview`, `StorageRemovalItem`, `StorageRemovalReceipt` and `StorageRemovalCapability`. They contain relative paths and opaque identities, diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index 943d86dd40..99ebff8581 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3647,4 +3647,4 @@ export interface ServerScopeDecisionReceipt { export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-journal.js"; export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageMaintenanceKind, StorageMaintenanceRequest, StorageMaintenanceItem, StorageMaintenanceObservation, StorageMaintenanceReceipt, StorageMaintenanceRow, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; -export type { StorageRemovalReviewRequest, StorageRemovalStartRequest, StorageRemovalUndoRequest, StorageRemovalFile, StorageRemovalReview, StorageRemovalItem, StorageRemovalReceipt, StorageRemovalCapability } from "./storage-removal.js"; +export type { StorageRemovalInspection, StorageRemovalReviewRequest, StorageRemovalStartRequest, StorageRemovalUndoRequest, StorageRemovalFile, StorageRemovalReview, StorageRemovalItem, StorageRemovalReceipt, StorageRemovalCapability } from "./storage-removal.js"; diff --git a/packages/ui-sdk/src/storage-removal.ts b/packages/ui-sdk/src/storage-removal.ts index 224e110351..627aa4cacc 100644 --- a/packages/ui-sdk/src/storage-removal.ts +++ b/packages/ui-sdk/src/storage-removal.ts @@ -9,6 +9,8 @@ export interface StorageRemovalFile { references: { source: string; certainty: "resolved" | "possible" }[]; recovery: "requires-retention" | "retained" | "unavailable"; } +/** Read-only current facts; no durable operation identity or authority to confirm. */ +export interface StorageRemovalInspection { observedAt: number; files: StorageRemovalFile[]; complete: boolean; unchecked: string[]; selectedFileBytes: number } export interface StorageRemovalReview { id: string; action: "remove" | "undo"; sourceId: string | null; fingerprint: string; observedAt: number; expiresAt: number; files: StorageRemovalFile[]; diff --git a/packages/ui-server/src/routes/storage-removal.ts b/packages/ui-server/src/routes/storage-removal.ts index abd429e9e0..b2d18b3cdd 100644 --- a/packages/ui-server/src/routes/storage-removal.ts +++ b/packages/ui-server/src/routes/storage-removal.ts @@ -18,6 +18,12 @@ export function createStorageRemovalRoutes(removal: StorageRemoval): Hono c.json(removal.capability(c.get("principal")!.id))); routes.get("/storage/removals/:id", c => c.json(removal.receipt(c.get("principal")!.id, c.req.param("id")))); + routes.post("/storage/removals/inspection", requireJson(), async c => { + let body: unknown; try { body = await readJsonBody(c, 140_000); } catch { return c.json({ error: "Invalid attachment JSON." }, 400); } + if (body instanceof Response) return body; const parsed = z.object({ paths }).strict().safeParse(body); + if (!parsed.success) return c.json({ error: "Select bounded attachment paths for inspection." }, 400); + return c.json(await removal.inspect(c.get("principal")!.id, parsed.data.paths)); + }); routes.post("/storage/removals/reviews", requireJson(), async c => { let body: unknown; try { body = await readJsonBody(c, 140_000); } catch { return c.json({ error: "Invalid attachment JSON." }, 400); } if (body instanceof Response) return body; const parsed = review.safeParse(body); diff --git a/packages/ui-server/src/storage/removal.ts b/packages/ui-server/src/storage/removal.ts index a2efa850a8..abf12ca3ac 100644 --- a/packages/ui-server/src/storage/removal.ts +++ b/packages/ui-server/src/storage/removal.ts @@ -4,7 +4,7 @@ import type { Database } from "bun:sqlite"; import { realpathSync } from "node:fs"; import { createHash } from "node:crypto"; import { inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment, storageAttachmentPath, storageAttachmentVacant, StorageAttachmentRefusal, type StorageAttachmentVersion, type StorageAttachmentFacts } from "@schlessera/brain/storage"; -import type { JournalVersion, StorageRemovalCapability, StorageRemovalReview, StorageRemovalReceipt, StorageRemovalFile } from "@schlessera/brain-ui-sdk/protocol"; +import type { JournalVersion, StorageRemovalInspection, StorageRemovalCapability, StorageRemovalReview, StorageRemovalReceipt, StorageRemovalFile } from "@schlessera/brain-ui-sdk/protocol"; import { BRAIN_LOCK_KEY, type KeyedLock } from "@schlessera/brain-ui-sdk/server"; import { GIT_LOCK_KEY } from "@schlessera/brain-ui-sdk/internal"; import { MAINTENANCE_ENTRY, type HostBoundary } from "../brain/host-boundary.js"; @@ -84,7 +84,7 @@ export function createStorageRemoval(db: Database, options: { if (!Array.isArray(paths) || paths.length < 1 || paths.length > STORAGE_REMOVAL_LIMITS.items || paths.some(path => typeof path !== "string" || Buffer.byteLength(path) > 4096 || /[\\\u0000-\u001f\u007f]/u.test(path) || path.split("/").some(part => !part || part === "." || part === "..")) || new Set(paths).size !== paths.length) throw new StorageUnavailable(400, "Select one bounded exact set of attachment paths."); return [...paths].sort(); } - async function review(principal: string, id: string, paths: string[], sourceId: string | null = null): Promise { + async function prepareReview(principal: string, id: string, paths: string[], sourceId: string | null, persist: boolean): Promise { authorize(principal); paths = pathsFor(paths); if (!validId(id) || sourceId !== null && !validId(sourceId)) throw new StorageUnavailable(400, "Invalid attachment operation identity."); const selection = JSON.stringify(paths), action = sourceId ? "undo" : "remove"; @@ -127,7 +127,7 @@ export function createStorageRemoval(db: Database, options: { const result: StorageRemovalReceipt = { review: reviewed, state: "review", requestedAt: now, startedAt: null, completedAt: null, reason: null, items: files.map(file => ({ path: file.path, state: file.eligible ? "not-started" : "protected", reason: file.reason, undoAvailable: false, undoReason: "no_confirmed_removal" })), before: null, after: null, index: "not-requested", commit: "not-requested", sync: "not-requested", retained: { journal: "subject-to-retention", git: "unchanged-by-operation", backups: "not-measured" } }; - journal.writeOperation(() => { + if (persist) journal.writeOperation(() => { if ((db.query("SELECT count(*) AS total FROM turn_journal_storage_operations WHERE brain=?").get(brain) as {total: number}).total >= STORAGE_REMOVAL_LIMITS.receipts) throw new StorageUnavailable(503, "Attachment receipt capacity reached; existing receipts remain readable."); db.query("INSERT INTO turn_journal_storage_operations (id,brain,principal_id,action,source_id,selection,fingerprint,journal_turn_id,state,pid,pid_start,requested_at,receipt,private_versions) VALUES (?,?,?,?,?,?,?,?,'review',?,?,?,?,?)") .run(id, brain, principal, action, sourceId, selection, reviewed.fingerprint, `storage_${id}`, SELF.pid, SELF.start, now, encode(result), encode(versions)); @@ -210,7 +210,15 @@ export function createStorageRemoval(db: Database, options: { } } return { - capability, review, receipt, + capability, receipt, + review: (principal: string, id: string, paths: string[], sourceId: string | null = null) => prepareReview(principal, id, paths, sourceId, true), + async inspect(principal: string, paths: string[]): Promise { + const { review } = await prepareReview(principal, crypto.randomUUID(), paths, null, false); + const { observedAt, files, complete, unchecked, selectedFileBytes } = review; + const result = { observedAt, files, complete, unchecked, selectedFileBytes }; + encode(result); // Same response ceiling; never persist or expose a confirmable identity. + return result; + }, start(principal: string, id: string, fingerprint: string) { authorize(principal); const row = select(id); if (!row || row.principal_id !== principal) throw new StorageUnavailable(404, "Attachment review unavailable."); diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index 5910fc8495..2f1ea05843 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -17,6 +17,7 @@ type Check = { path?: string; init?: RequestInit; status?: number; body?: object // this map is cross-checked with the spec and the actual Hono mounts. const checks: Record = { "GET /api/storage/removals": { verify: async r => { const value = await r.json(); expect(value).toHaveProperty("available"); expect(value.limits.items).toBe(32); } }, + "POST /api/storage/removals/inspection": { init: json("POST", {paths: []}), status: 400, body: {error:"Select bounded attachment paths for inspection."} }, "POST /api/storage/removals/reviews": { init: json("POST", { id: "odysseus-mounted-review", paths: [] }), status: 400, body: {error:"Select bounded attachment paths and a new review identity."} }, "POST /api/storage/removals/undo-reviews": { init: json("POST", {}), status: 400, body: {error:"Select retained originals from one removal and a new Undo identity."} }, "POST /api/storage/removals": { init: json("POST", {id:"odysseus-mounted-missing",fingerprint:"a".repeat(64)}), status:404, body:{error:"Attachment review unavailable."} }, diff --git a/packages/ui-server/tests/storage-removal-routes.test.ts b/packages/ui-server/tests/storage-removal-routes.test.ts index d3fe429cf3..3428043f8e 100644 --- a/packages/ui-server/tests/storage-removal-routes.test.ts +++ b/packages/ui-server/tests/storage-removal-routes.test.ts @@ -17,7 +17,7 @@ test("mounted removal enforces human authority, origin, exact review and durable const owner=createPrincipal(f.app.db,{authMethod:"password",label:"Odysseus",ttlSeconds:3600}),agent=createPrincipal(f.app.db,{authMethod:"delegated",createdBy:owner.id,label:"Athena",ttlSeconds:3600}),other=createPrincipal(f.app.db,{authMethod:"password",label:"Penelope",ttlSeconds:3600}); const cookie=async(id:string)=>(await generateSignedCookie("brain_ui_session",id,secret)).split(";")[0]!,ownerCookie=await cookie(owner.id),agentCookie=await cookie(agent.id),otherCookie=await cookie(other.id); const request={id:"odysseus-route-removal",paths:["attachments/chart.pdf"]}; - for(const [method,path] of [["GET","/api/storage/removals"],["GET","/api/storage/removals/missing"],["POST","/api/storage/removals/reviews"],["POST","/api/storage/removals/undo-reviews"],["POST","/api/storage/removals"],["DELETE","/api/storage/removals/missing"]]){ + for(const [method,path] of [["GET","/api/storage/removals"],["GET","/api/storage/removals/missing"],["POST","/api/storage/removals/inspection"],["POST","/api/storage/removals/reviews"],["POST","/api/storage/removals/undo-reviews"],["POST","/api/storage/removals"],["DELETE","/api/storage/removals/missing"]]){ const init={method,headers:{cookie:agentCookie,"content-type":"application/json"},...(method==="POST"?{body:JSON.stringify(request)}:{})}; expect((await f.fetch(path!,init)).status,"STORAGE_REMOVAL_AGENT_DENIED").toBe(403); expect((await f.fetch(path!,{...init,headers:{"content-type":"application/json"}})).status).toBe(401); @@ -30,6 +30,12 @@ test("mounted removal enforces human authority, origin, exact review and durable mkdirSync(join(f.brainPath,"attachments"));const original=Buffer.from([37,80,68,70,0,255,13]);writeFileSync(join(f.brainPath,request.paths[0]!),original);writeFileSync(join(f.brainPath,"notes/arrival.md"),"[Harbor](../attachments/chart.pdf)\n"); writeFileSync(join(f.brainPath,"attachments/keep.pdf"),"Penelope's unselected original"); writeFileSync(join(f.brainPath,"brain.config.ts"),'import {rmSync} from "node:fs";rmSync(process.env.BRAIN_WORKER_DATA_ROOT+"/attachments/keep.pdf");export default {};\n'); + const inspected=await post("/api/storage/removals/inspection",{paths:request.paths}); + expect(inspected.status,"STORAGE_READONLY_INSPECTION_ROUTE").toBe(200); + const observation=await inspected.json();expect(observation.files[0]).toMatchObject({eligible:true,bytes:original.length}); + expect(observation).not.toHaveProperty("id");expect(observation).not.toHaveProperty("fingerprint"); + expect(f.app.db.query("SELECT count(*) AS n FROM turn_journal_storage_operations").get(),"STORAGE_INSPECTION_NO_RECEIPT_SLOT").toEqual({n:0}); + expect(readFileSync(join(f.brainPath,request.paths[0]!))).toEqual(original); const reviewed=await post("/api/storage/removals/reviews",request),review=await reviewed.json(); expect(reviewed.status,JSON.stringify(review)).toBe(200); expect(review.review.files[0],"STORAGE_HTTP_CURRENT_NATIVE_REFERENCE").toMatchObject({eligible:true,references:[{source:"notes/arrival.md",certainty:"resolved"}]}); From 821249cf2ea9815a22c30dbd36a3f855160e792d Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 16:32:04 +0200 Subject: [PATCH 12/44] feat(storage): persist bounded history, pressure and Critical notice evidence --- .changeset/storage-history-guidance.md | 5 + docs/decisions/action-notifications.md | 2 +- docs/inbox-notifications.md | 2 +- .../plans/async-collaboration-requirements.md | 2 +- docs/plans/async-collaboration.md | 2 +- .../migrations/053_storage_guidance.sql | 36 +++++ .../ui-server/src/activity/push-sender.ts | 33 ++++ .../ui-server/src/storage/history-evidence.ts | 90 +++++++++++ .../ui-server/src/storage/history-model.ts | 128 ++++++++++++++++ .../ui-server/src/storage/history-store.ts | 142 ++++++++++++++++++ packages/ui-server/src/storage/notify.ts | 90 +++++++++++ packages/ui-server/src/storage/pressure.ts | 92 ++++++++++++ .../tests/storage-history-model.test.ts | 129 ++++++++++++++++ .../tests/storage-history-store.test.ts | 97 ++++++++++++ .../ui-server/tests/storage-notify.test.ts | 122 +++++++++++++++ .../ui-server/tests/storage-pressure.test.ts | 75 +++++++++ 16 files changed, 1043 insertions(+), 4 deletions(-) create mode 100644 .changeset/storage-history-guidance.md create mode 100644 packages/ui-server/migrations/053_storage_guidance.sql create mode 100644 packages/ui-server/src/storage/history-evidence.ts create mode 100644 packages/ui-server/src/storage/history-model.ts create mode 100644 packages/ui-server/src/storage/history-store.ts create mode 100644 packages/ui-server/src/storage/notify.ts create mode 100644 packages/ui-server/src/storage/pressure.ts create mode 100644 packages/ui-server/tests/storage-history-model.test.ts create mode 100644 packages/ui-server/tests/storage-history-store.test.ts create mode 100644 packages/ui-server/tests/storage-notify.test.ts create mode 100644 packages/ui-server/tests/storage-pressure.test.ts diff --git a/.changeset/storage-history-guidance.md b/.changeset/storage-history-guidance.md new file mode 100644 index 0000000000..c11531cae9 --- /dev/null +++ b/.changeset/storage-history-guidance.md @@ -0,0 +1,5 @@ +--- +"@schlessera/brain-ui-server": minor +--- + +Keep bounded Recent, Daily and Weekly Storage observations, qualified capacity scenarios and independent pressure causes. Retain warning and notification evidence across restart, use existing owner-authorized push subscriptions, and suppress ambiguous or restored delivery attempts. diff --git a/docs/decisions/action-notifications.md b/docs/decisions/action-notifications.md index 8a5d544f91..0d3766e65f 100644 --- a/docs/decisions/action-notifications.md +++ b/docs/decisions/action-notifications.md @@ -181,7 +181,7 @@ incrementing an Action count (`function createIntent(input: {`, `packages/ui-server/src/activity/notify.ts:91-127`). The sender processes run-bound activity rows and settles one intent from a device pass; that is not the required per-episode/per-destination Action history -(`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:194-273`). +(`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:197-276`). Reuse concrete subscriptions, authorization, revocation and bounded backoff, but add the Actions-aware aggregate and durable constituent/destination records. This decision authorizes no new channel, extension seam or daemon. diff --git a/docs/inbox-notifications.md b/docs/inbox-notifications.md index 73d7772803..897445e6cd 100644 --- a/docs/inbox-notifications.md +++ b/docs/inbox-notifications.md @@ -42,7 +42,7 @@ deadline opens the next window. Thread identity is kept on each constituent, not in the grouping key. Unmarked below-cutoff decisions and FYIs never join a window. Delivery runs after activity intents on the same reentrancy guard -(`deliverActions`, `packages/ui-server/src/activity/push-sender.ts:287-336`). +(`deliverActions`, `packages/ui-server/src/activity/push-sender.ts:320-369`). Each destination is evaluated on its own. `beginAttempt()` rechecks, in one transaction, that the subscription is still bound to the principal and the principal is usable, that the device reported a usable zone and is outside diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index 289093c5cc..e72ae798be 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -387,7 +387,7 @@ Stated before the requirements because five of them derive from it. to `failure|completion|stuck`, requires an activity `run_id`, coalesces only by dropping a later same-tag intent without updating a count (`function createIntent(input: {`, `packages/ui-server/src/activity/notify.ts:91-127`), and the sender emits one push per - pending row (`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:194-241`). Reuse subscriptions, retry budget, and delivery + pending row (`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:197-244`). Reuse subscriptions, retry budget, and delivery status; add an Actions-aware aggregate carrying group key, count, priority, quiet-hours eligibility, and an Actions deep link. The [2026-10-02 Action notification decision](../decisions/action-notifications.md) diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 9f14586013..0206111fb4 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -151,7 +151,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. Reservations must cover in-flight work, not only this retrospective sum. - **Notifications:** (`CREATE TABLE IF NOT EXISTS notification_intents (`, `packages/ui-server/migrations/007_activity.sql:99-114`), (`function createIntent(input: {`, `packages/ui-server/src/activity/notify.ts:91-127`) and - (`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:194-241`) are run-bound and do not + (`async deliverPending(notifier) {`, `packages/ui-server/src/activity/push-sender.ts:197-244`) are run-bound and do not maintain an Actions aggregate count. - **Share provenance/limits:** (`const result = await stageShareAt(`, `packages/ui-server/src/inbox/intake.ts:95`) assigns the source in server code. (`export const SHARE_MAX_TEXT_BYTES =`, `packages/ui-sdk/src/protocol.ts:2325`) bounds text, not binary uploads; diff --git a/packages/ui-server/migrations/053_storage_guidance.sql b/packages/ui-server/migrations/053_storage_guidance.sql new file mode 100644 index 0000000000..504245856f --- /dev/null +++ b/packages/ui-server/migrations/053_storage_guidance.sql @@ -0,0 +1,36 @@ +-- Bounded operational evidence, separate from disposable content indexing. +CREATE TABLE storage_history ( + brain TEXT PRIMARY KEY CHECK (length(brain) = 32), + payload_json TEXT NOT NULL CHECK (length(CAST(payload_json AS BLOB)) <= 262144), + revision INTEGER NOT NULL DEFAULT 0 CHECK (revision >= 0) +); +-- Coordination and delivery receipts are operational overhead, not history payload. +CREATE TABLE storage_scan_leases ( + brain TEXT PRIMARY KEY CHECK (length(brain) = 32), + job_id TEXT, + pid INTEGER, + process_start TEXT, + principal_id TEXT, + started_at INTEGER, + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK (cancel_requested IN (0, 1)), + last_attempt_bucket INTEGER, + result_id TEXT, + result_at INTEGER, + failure TEXT, + CHECK ((job_id IS NULL AND pid IS NULL AND process_start IS NULL AND principal_id IS NULL AND started_at IS NULL) + OR (job_id IS NOT NULL AND pid > 0 AND process_start IS NOT NULL AND principal_id IS NOT NULL AND started_at >= 0)) +); +CREATE TABLE storage_notice_receipts ( + brain TEXT NOT NULL REFERENCES storage_history(brain) ON DELETE CASCADE, + episode TEXT NOT NULL, + endpoint TEXT NOT NULL REFERENCES push_subscriptions(endpoint) ON DELETE CASCADE, + principal_id TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('pending', 'sending', 'accepted', 'failed', 'unknown', 'suppressed')), + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts BETWEEN 0 AND 3), + attempted_at INTEGER, + next_at INTEGER, + batch_id TEXT, + pid INTEGER, + process_start TEXT, + PRIMARY KEY (brain, episode, endpoint) +); diff --git a/packages/ui-server/src/activity/push-sender.ts b/packages/ui-server/src/activity/push-sender.ts index 54f11aaa58..86860ea8d3 100644 --- a/packages/ui-server/src/activity/push-sender.ts +++ b/packages/ui-server/src/activity/push-sender.ts @@ -20,6 +20,7 @@ import webpush from "web-push"; import type { ActivityNotifier } from "./notify.js"; import type { ActionAttemptOutcome, ActionNotifier } from "../inbox/notify.js"; +import type { StorageNotifier, StorageNoticeOutcome } from "../storage/notify.js"; export interface PushSubscriptionRow { endpoint: string; @@ -52,6 +53,8 @@ export interface PushSender { * frozen by the notifier before sending. Returns sends attempted. */ deliverActions(notices: ActionNotifier, now?: number): Promise; + /** One durable Critical Storage submission per episode and subscription. */ + deliverStorage(notices: StorageNotifier, now?: number): Promise; } export interface CreatePushSenderOptions { @@ -284,6 +287,36 @@ export function createPushSender( return attempts; }, + async deliverStorage(notices, at) { + let attempts = 0; + // Resolve credentials first, then acquire each durable intent immediately + // before its transport call. Authority/zone/time are rechecked per device. + const subs = deliverableRows(at ?? Date.now()); + if (!subs.length) return 0; + const keys = vapid(); + for (const sub of subs) { + const attempt = notices.begin(sub.endpoint, at ?? Date.now()); + if (!attempt) continue; + let outcome: StorageNoticeOutcome; + attempts++; + try { + await send(attempt.subscription, JSON.stringify(attempt.payload), { + vapidDetails: { subject, publicKey: keys.publicKey, privateKey: keys.privateKey }, timeout: SEND_TIMEOUT_MS, + }); + outcome = "accepted"; + } catch (error) { + const status = (error as { statusCode?: number } | null)?.statusCode; + outcome = status === 404 || status === 410 ? "gone" : typeof status === "number" && status >= 400 && status <= 599 ? "failed" : "unknown"; + } + // A database failure after transport leaves the durable sending intent; + // recovery makes it unknown and never retries a possible acceptance. + notices.finish(attempt.id, outcome, at ?? Date.now()); + if (outcome === "gone") db.query("DELETE FROM push_subscriptions WHERE endpoint = ?").run(sub.endpoint); + else if (outcome === "accepted") db.query("UPDATE push_subscriptions SET last_used_at = ? WHERE endpoint = ? AND principal_id = ?").run(at ?? Date.now(), sub.endpoint, attempt.principalId); + } + return attempts; + }, + async deliverActions(notices, at) { const now = at ?? Date.now(); // Each destination is evaluated on its own: its zone, quiet hours, diff --git a/packages/ui-server/src/storage/history-evidence.ts b/packages/ui-server/src/storage/history-evidence.ts new file mode 100644 index 0000000000..cb37a2d97c --- /dev/null +++ b/packages/ui-server/src/storage/history-evidence.ts @@ -0,0 +1,90 @@ +import { DAY, HOUR, RECENT, bucket, type Capacity, type HistoryPayload, type Sample } from "./history-model.js"; + +export type EvidenceReason = "no-observation" | "stale" | "future" | "incomplete" | "changed-scope" | "changed-capacity" | "gap" | "ordering" | "insufficient-count" | "insufficient-span" | "mixed-direction" | "single-burst" | "arithmetic"; +export interface WindowEvidence { + state: "unavailable" | "finite" | "non-depleting"; + reason: EvidenceReason | null; + start: number | null; end: number | null; samples: number; + depletionPerDay: number | null; days: number | null; +} +export interface ObservedRate { start: number; end: number; samples: number; bytesPerDay: number } +export interface CapacityEvidence { + identity: string; rootScope: string; + at: number; total: number | null; available: number | null; + fresh: boolean; short: WindowEvidence; long: WindowEvidence; + appLogical: ObservedRate | null; appAllocated: ObservedRate | null; availableChange: ObservedRate | null; +} +type Point = { sample: Sample; capacity: Capacity | null; scope: string; basis: number; identity: string; rootScope: string }; +const unavailable = (reason: EvidenceReason, window: Point[] = []): WindowEvidence => ({ state: "unavailable", reason, start: window[0]?.sample[3] ?? null, end: window.at(-1)?.sample[3] ?? null, samples: window.length, depletionPerDay: null, days: null }); + +function point(payload: HistoryPayload, sample: Sample, identity: string): Point { + const capacity = sample[9].find(row => payload.metadata.capacities[row[0]]?.[0] === identity) ?? null; + const descriptor = capacity ? payload.metadata.capacities[capacity[0]] : null; + return { sample, capacity, scope: payload.metadata.samples[sample[6]]?.[1] ?? "", basis: payload.metadata.samples[sample[6]]?.[2] ?? 0, identity: descriptor?.[0] ?? "", rootScope: descriptor?.[1] ?? "" }; +} +function invalid(value: Point, latest: Point): EvidenceReason | null { + if (value.sample[2]) return "gap"; + if (!value.capacity || value.capacity[5] !== 0 || value.capacity[1] === null || value.capacity[1] <= 0 || value.capacity[4] === null || (value.sample[7] !== 0 && value.sample[8] !== 3)) return "incomplete"; + if (value.identity !== latest.identity || value.rootScope !== latest.rootScope || value.scope !== latest.scope || value.basis !== latest.basis) return "changed-scope"; + if (value.capacity[1] !== latest.capacity?.[1]) return "changed-capacity"; + return null; +} +function suffix(points: Point[], span: number, interval: number, now: number): { values: Point[]; reason: EvidenceReason | null } { + const latest = points.at(-1); + if (!latest) return { values: [], reason: "no-observation" }; + if (latest.sample[3] > now) return { values: [], reason: "future" }; + if (now - latest.sample[3] > RECENT) return { values: [], reason: "stale" }; + const issue = invalid(latest, latest); + if (issue) return { values: [], reason: issue }; + const values: Point[] = [latest]; + let reason: EvidenceReason | null = null; + for (let index = points.length - 2; index >= 0; index--) { + const value = points[index]!, next = values[0]!; + if (value.sample[3] >= next.sample[3]) { reason = "ordering"; break; } + if (latest.sample[3] - value.sample[3] > span) break; + reason = invalid(value, latest) ?? (next.sample[3] - value.sample[3] > interval ? "gap" : null); + if (reason) break; + values.unshift(value); + } + return { values, reason }; +} +function scenario(points: Point[], minimum: number, minimumSpan: number, maximumSpan: number, maximumInterval: number, now: number): WindowEvidence { + const { values, reason } = suffix(points, maximumSpan, maximumInterval, now); + if (values.length < minimum) return unavailable(reason ?? "insufficient-count", values); + const first = values[0]!, last = values.at(-1)!, elapsed = last.sample[3] - first.sample[3]; + if (elapsed < minimumSpan) return unavailable("insufficient-span", values); + const changes = values.slice(1).map((value, index) => values[index]!.capacity![4]! - value.capacity![4]!); + const net = first.capacity![4]! - last.capacity![4]!; + const observation = { start: first.sample[3], end: last.sample[3], samples: values.length }; + if (net === 0) return changes.every(change => change === 0) + ? { ...observation, state: "non-depleting", reason: null, depletionPerDay: 0, days: null } + : unavailable("mixed-direction", values); + if (changes.filter(change => change === 0 || Math.sign(change) === Math.sign(net)).length * 3 < changes.length * 2) return unavailable("mixed-direction", values); + if (changes.some(change => Math.abs(change) > Math.abs(net) / 2)) return unavailable("single-burst", values); + const depletionPerDay = net / (elapsed / DAY), days = last.capacity![4]! / depletionPerDay; + if (!Number.isFinite(depletionPerDay) || !Number.isFinite(days)) return unavailable("arithmetic", values); + return { ...observation, state: net > 0 ? "finite" : "non-depleting", reason: null, depletionPerDay, days: net > 0 ? days : null }; +} +function rate(points: Point[], field: 4 | 8 | 9, now: number): ObservedRate | null { + let { values } = suffix(points, 7 * DAY, 12 * HOUR, now); + const lastUnknown = values.findLastIndex(value => value.capacity?.[field] === null); + if (lastUnknown >= 0) values = values.slice(lastUnknown + 1); + const first = values[0], last = values.at(-1); + if (values.length < 5 || !first || !last || last.sample[3] - first.sample[3] < 48 * HOUR) return null; + const bytesPerDay = (last.capacity![field]! - first.capacity![field]!) / ((last.sample[3] - first.sample[3]) / DAY); + return Number.isFinite(bytesPerDay) ? { start: first.sample[3], end: last.sample[3], samples: values.length, bytesPerDay } : null; +} + +/** Named evidence has no interpolation, confidence interval or implied infinite capacity. */ +export function capacityEvidence(payload: HistoryPayload, identity: string, now: number): CapacityEvidence | null { + const recent = payload.samples.filter(row => row[1] === 0).map(row => point(payload, row, identity)); + const latest = recent.at(-1); + if (!latest?.capacity) return null; + const daily = payload.samples.filter(row => row[1] === 1 && bucket(1, row[3]) < bucket(1, latest.sample[3])).map(row => point(payload, row, identity)); + const long = [...daily.filter(value => value.sample[3] !== latest.sample[3]), latest]; + return { identity, rootScope: latest.rootScope, at: latest.sample[3], total: latest.capacity[1], available: latest.capacity[4], + fresh: latest.capacity[5] !== 2 && latest.capacity[1] !== null && latest.capacity[1] > 0 && latest.capacity[4] !== null && latest.sample[3] <= now && now - latest.sample[3] <= RECENT, + short: scenario(recent, 5, 48 * HOUR, 7 * DAY, 12 * HOUR, now), + long: scenario(long, 15, 14 * DAY, 28 * DAY, 48 * HOUR, now), + appLogical: rate(recent, 8, now), appAllocated: rate(recent, 9, now), availableChange: rate(recent, 4, now) }; +} diff --git a/packages/ui-server/src/storage/history-model.ts b/packages/ui-server/src/storage/history-model.ts new file mode 100644 index 0000000000..7c0b6ac151 --- /dev/null +++ b/packages/ui-server/src/storage/history-model.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import type { StorageCollection } from "./collect.js"; +import { storageTotals } from "./collect.js"; + +export const HISTORY_LIMITS = Object.freeze({ slots: [64, 90, 32] as const, sampleBytes: 1152, metadataBytes: 40960, payloadBytes: 262144, capacities: 4, writeInterval: 60_000 }); +export const HOUR = 3_600_000, DAY = 24 * HOUR, RECENT = 6 * HOUR; +export type Quantity = number | null; +export type Coverage = 0 | 1 | 2; +export type Tier = 0 | 1 | 2; +export type Source = 0 | 1 | 2; +export type Capacity = [number, Quantity, Quantity, Quantity, Quantity, Coverage, Quantity, Quantity, Quantity, Quantity, Quantity, Quantity, Quantity, Quantity]; +export type Sample = [1, Tier, 0 | 1, number, number, Source, number, Coverage, 0 | 1 | 2 | 3 | 4 | 5, Capacity[]]; +export type SampleDescriptor = [string, string, 1]; +export type CapacityDescriptor = [string, string]; +export interface HistoryMetadata { + samples: Record; + capacities: Record; + state: S; +} +export interface HistoryPayload { metadata: HistoryMetadata; samples: Sample[] } +export interface Aggregate { + at: number; startedAt: number; source: Source; descriptor: SampleDescriptor; + coverage: Coverage; reason: Sample[8]; + capacities: Array<{ descriptor: CapacityDescriptor; values: OmitFirst }>; +} +type OmitFirst = T extends [unknown, ...infer Rest] ? Rest : never; +export const hashScope = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 32); +export const coverage = (value: "complete" | "partial" | "unknown"): Coverage => value === "complete" ? 0 : value === "partial" ? 1 : 2; +export const bucket = (tier: Tier, at: number) => Math.floor((at - (tier === 2 ? 4 * DAY : 0)) / (tier === 0 ? RECENT : tier === 1 ? DAY : 7 * DAY)); +export const bucketStart = (tier: Tier, index: number) => index * (tier === 0 ? RECENT : tier === 1 ? DAY : 7 * DAY) + (tier === 2 ? 4 * DAY : 0); +export const jsonBytes = (value: unknown) => Buffer.byteLength(JSON.stringify(value)); +const safe = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +const quantity = (value: unknown) => value === null || safe(value); +const id = (value: unknown, limit: number) => safe(value) && value < limit; +const descriptorHash = (value: unknown) => typeof value === "string" && /^[a-f0-9]{32}$/.test(value); + +/** Fixed tuples are private. Validate the actual bytes before every durable replacement. */ +export function validateHistory(payload: HistoryPayload): void { + const { metadata, samples } = payload; + if (jsonBytes(metadata) > HISTORY_LIMITS.metadataBytes) throw Error("history-metadata-budget"); + if (samples.length > 186 || jsonBytes(payload) > HISTORY_LIMITS.payloadBytes) throw Error("history-payload-budget"); + for (const [key, value] of Object.entries(metadata.samples)) { + if (!/^(0|[1-9]\d*)$/.test(key) || !id(Number(key), 186) || value.length !== 3 || !/^[\x20-\x7e]{1,32}$/.test(value[0]) || !descriptorHash(value[1]) || value[2] !== 1) throw Error("history-sample-descriptor"); + } + for (const [key, value] of Object.entries(metadata.capacities)) { + if (!/^(0|[1-9]\d*)$/.test(key) || !id(Number(key), 744) || value.length !== 2 || !value.every(descriptorHash)) throw Error("history-capacity-descriptor"); + } + for (const tier of [0, 1, 2] as const) { + const rows = samples.filter(row => row[1] === tier); + if (rows.length > HISTORY_LIMITS.slots[tier] || new Set(rows.map(row => bucket(tier, row[3]))).size !== rows.length) throw Error("history-tier-budget"); + } + for (const sample of samples) { + if (sample.length !== 10 || sample[0] !== 1 || !id(sample[1], 3) || !id(sample[2], 2) || !safe(sample[3]) || !safe(sample[4]) || sample[4] > sample[3] || !id(sample[5], 3) || !id(sample[6], 186) || !id(sample[7], 3) || !id(sample[8], 6) || !metadata.samples[sample[6]] || sample[9].length > 4 || jsonBytes(sample) > HISTORY_LIMITS.sampleBytes) throw Error("history-sample-budget"); + if (new Set(sample[9].map(row => row[0])).size !== sample[9].length) throw Error("history-duplicate-capacity"); + for (const row of sample[9]) { + if (row.length !== 14 || !id(row[0], 744) || !metadata.capacities[row[0]] || !id(row[5], 3) || row.some((value, index) => index !== 0 && index !== 5 && !quantity(value))) throw Error("history-capacity-shape"); + } + } +} + +/** A single scan supplies all tiers. Union attribution is deduplicated per capacity. */ +export function aggregateCollection(value: StorageCollection, source: Source, collector: string): Aggregate { + if (!/^[\x20-\x7e]{1,32}$/.test(collector)) throw Error("history-collector-version"); + const { summary, entries } = value, owner = new Map(summary.roots.map(root => [root.id, root.owner])); + const scope = summary.roots.map(root => [root.id, root.owner, root.category]).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + const capacities = [...summary.capacities].sort((a, b) => a.id.localeCompare(b.id)); + return { at: summary.observedAt, startedAt: summary.startedAt, source, + descriptor: [collector, hashScope(scope), 1], coverage: capacities.length > 4 ? 1 : coverage(summary.coverage), reason: capacities.length > 4 ? 3 : summary.coverage === "complete" ? 0 : 1, + capacities: capacities.slice(0, 4).map(capacity => { + const rows = entries.filter(entry => entry.capacityId === capacity.id), roots = summary.roots.filter(root => capacity.rootIds.includes(root.id)); + const complete = roots.length > 0 && roots.every(root => root.coverage === "complete") && !summary.errors.some(error => !error.rootId || capacity.rootIds.includes(error.rootId)); + const owners = (["brain", "app", "operator"] as const).map(kind => storageTotals(rows.filter(entry => owner.get(entry.rootId) === kind), complete)); + const union = storageTotals(rows, complete); + const known = capacity.identity === "resolved" && capacity.total !== null && capacity.total > 0 && capacity.available !== null && capacity.free !== null && capacity.used !== null && capacity.used + capacity.free === capacity.total && capacity.available <= capacity.free; + return { descriptor: [capacity.id, hashScope([...capacity.rootIds].sort())], values: [capacity.total, capacity.used, capacity.free, capacity.available, known && complete ? 0 : known ? 1 : 2, + owners[0]!.logical, owners[0]!.allocated, owners[1]!.logical, owners[1]!.allocated, owners[2]!.logical, owners[2]!.allocated, union.logical, union.allocated] }; + }) }; +} + +function collectDescriptors(payload: HistoryPayload) { + const samples = new Set(payload.samples.map(row => String(row[6]))), capacities = new Set(payload.samples.flatMap(row => row[9].map(capacity => String(capacity[0])))); + for (const key of Object.keys(payload.metadata.samples)) if (!samples.has(key)) delete payload.metadata.samples[key]; + for (const key of Object.keys(payload.metadata.capacities)) if (!capacities.has(key)) delete payload.metadata.capacities[key]; +} +function intern(dictionary: Record, value: T, limit: number) { + const encoded = JSON.stringify(value), found = Object.entries(dictionary).find(([, item]) => JSON.stringify(item) === encoded); + if (found) return Number(found[0]); + for (let index = 0; index < limit; index++) if (!(index in dictionary)) { dictionary[index] = value; return index; } + throw Error("history-descriptor-budget"); +} + +/** Pure candidate construction. Caller commits only after the whole payload validates. */ +export function retainAggregate(previous: HistoryPayload, input: Aggregate): HistoryPayload { + const next = structuredClone(previous), { samples, metadata } = next; + const latest = Math.max(0, ...samples.map(row => row[3])); + if (!safe(input.at) || !safe(input.startedAt) || input.startedAt > input.at || input.at < latest) throw Error("history-clock-reversal"); + // Age and count are independent bounds. Do not replace good endpoints with failure. + for (let index = samples.length - 1; index >= 0; index--) { + const row = samples[index]!, current = bucket(row[1], input.at), old = bucket(row[1], row[3]); + if (old < current - HISTORY_LIMITS.slots[row[1]] + 1 || old > current) samples.splice(index, 1); + else if (old < current && bucket(0, row[3]) < bucket(0, bucketStart(row[1], old + 1) - 1)) row[2] = 1; + } + const lastRecent = samples.filter(row => row[1] === 0).sort((a, b) => b[3] - a[3])[0]; + const changed = lastRecent && (JSON.stringify(metadata.samples[lastRecent[6]]) !== JSON.stringify(input.descriptor) + || JSON.stringify(lastRecent[9].map(row => metadata.capacities[row[0]])) !== JSON.stringify(input.capacities.map(row => row.descriptor))); + const missed = !!lastRecent && bucket(0, input.at) > bucket(0, lastRecent[3]) + 1; + const incomplete = (input.coverage !== 0 && input.reason !== 3) || input.capacities.some(row => row.values[4] !== 0); + const replaced = new Map(); + for (const tier of [0, 1, 2] as const) { + const prior = samples.find(row => row[1] === tier && bucket(tier, row[3]) === bucket(tier, input.at)); + if (prior) { + if (prior[7] === 0 && input.coverage !== 0) { prior[2] = 1; continue; } + replaced.set(tier, prior); samples.splice(samples.indexOf(prior), 1); + } + } + collectDescriptors(next); + // Protect a complete bucket even when the incoming input contains a new scope. + const tiers = ([0, 1, 2] as const).filter(tier => !samples.some(row => row[1] === tier && bucket(tier, row[3]) === bucket(tier, input.at))); + if (tiers.length) { + const descriptor = intern(metadata.samples, input.descriptor, 186), capacities = input.capacities.map(row => [intern(metadata.capacities, row.descriptor, 744), ...row.values] as Capacity); + for (const tier of tiers) { + const gap = !!replaced.get(tier)?.[2] || incomplete || !!changed || missed || (!replaced.has(tier) && input.at >= bucketStart(tier, bucket(tier, input.at)) + RECENT); + samples.push([1, tier, gap ? 1 : 0, input.at, input.startedAt, input.source, descriptor, input.coverage, changed ? 5 : input.reason, capacities]); + } + } + samples.sort((a, b) => a[1] - b[1] || a[3] - b[3]); + collectDescriptors(next); validateHistory(next); return next; +} diff --git a/packages/ui-server/src/storage/history-store.ts b/packages/ui-server/src/storage/history-store.ts new file mode 100644 index 0000000000..a0c02a0930 --- /dev/null +++ b/packages/ui-server/src/storage/history-store.ts @@ -0,0 +1,142 @@ +import { z } from "zod"; +import type { Database } from "bun:sqlite"; +import { randomBytes } from "node:crypto"; +import { realpathSync } from "node:fs"; +import { capacityEvidence } from "./history-evidence.js"; +import { newPressure, observePressure, type Pressure } from "./pressure.js"; +import { HISTORY_LIMITS, bucket, hashScope, jsonBytes, retainAggregate, validateHistory, type Aggregate, type HistoryPayload } from "./history-model.js"; +export interface HistoryState { + version: 1; salt: string; + lastPersistedAt: number; lastObservedAt: number; lastAttemptBucket: number | null; + paused: "collection-unavailable" | "clock-reversal" | null; + pressures: Pressure[]; + /** Restoring an old backup cannot revive authority to send its old episodes. */ + suppressedEpisodes: string[]; + scopeLimit: boolean; +} +export type StoredHistory = HistoryPayload; +type Pending = { kind: "aggregate"; value: Aggregate } | { kind: "failure"; at: number }; +export const storageBrainKey = (brainPath: string) => hashScope(realpathSync(brainPath)); +export function emptyHistory(): StoredHistory { + return { metadata: { samples: {}, capacities: {}, state: { version: 1, salt: randomBytes(32).toString("hex"), lastPersistedAt: 0, lastObservedAt: 0, lastAttemptBucket: null, paused: null, pressures: [], suppressedEpisodes: [], scopeLimit: false } }, samples: [] }; +} +const instant = z.number().int().nonnegative().safe(); +const digest = z.string().regex(/^[a-f0-9]{32}$/); +const pressureSchema = z.object({ + identity: digest, rootScope: digest, severity: z.enum(["observed", "low", "critical"]), + capacity: z.boolean(), forecast: z.boolean(), refusals: z.array(z.object({ + target: digest, resource: z.enum(["bytes", "quota", "inodes", "unknown"]), at: instant, successAt: instant.nullable(), + }).strict()), at: instant, recoveryAt: instant.nullable(), episode: z.string().uuid().nullable(), + resolvedEpisode: z.string().uuid().nullable(), supportAt: instant.nullable(), scopeChanged: z.boolean(), +}).strict().refine(value => (value.severity === "critical") === (value.episode !== null) + && (value.severity !== "observed" || !value.capacity && !value.forecast && !value.refusals.length) + && new Set(value.refusals.map(row => row.target + ":" + row.resource)).size === value.refusals.length); +const stateSchema = z.object({ + version: z.literal(1), salt: z.string().regex(/^[a-f0-9]{64}$/), lastPersistedAt: instant, + lastObservedAt: instant, lastAttemptBucket: instant.nullable(), paused: z.enum(["collection-unavailable", "clock-reversal"]).nullable(), + pressures: z.array(pressureSchema).max(4), suppressedEpisodes: z.array(z.string().uuid()).max(8), scopeLimit: z.boolean(), +}).strict().refine(value => new Set(value.pressures.map(row => row.identity)).size === value.pressures.length); +export function parseStoredHistory(value: string): StoredHistory { + const result = JSON.parse(value) as StoredHistory; + validateHistory(result); + stateSchema.parse(result.metadata.state); + return result; +} +export function readStoredHistory(db: Database, brain: string): StoredHistory | null { + const row = db.query("SELECT payload_json FROM storage_history WHERE brain = ?").get(brain) as { payload_json: string } | null; + return row ? parseStoredHistory(row.payload_json) : null; +} +export function createStorageHistory(db: Database, brainPath: string, options: { now?: () => number } = {}) { + const brain = storageBrainKey(brainPath), now = options.now ?? Date.now; + let pending: Pending | null = null, unavailable: string | null = null, closed = false; + // Constructing the store never collects filesystem inventory. Failure here is + // a capability result; it cannot prevent existing recovery operations booting. + try { db.query("INSERT OR IGNORE INTO storage_history (brain, payload_json) VALUES (?, ?)").run(brain, JSON.stringify(emptyHistory())); } + catch { unavailable = "history-write-unavailable"; } + const read = () => { + try { return readStoredHistory(db, brain); } + catch { unavailable = "history-read-unavailable"; return null; } + }; + function offer(value: Pending) { + if (closed) return; + const at = value.kind === "aggregate" ? value.value.at : value.at; + const stored = read(); + if (stored && at < stored.metadata.state.lastObservedAt) { + unavailable = "history-clock-reversal"; + // Do not overwrite a newer pending observation after a clock reversal. + return; + } + const prior = pending && (pending.kind === "aggregate" ? pending.value.at : pending.at); + if (prior === null || prior <= at) pending = value; + flush(); + } + function flush(): boolean { + if (closed || !pending) return false; + const candidate = pending, at = candidate.kind === "aggregate" ? candidate.value.at : candidate.at, time = now(); + try { + const changed = db.transaction(() => { + const previous = readStoredHistory(db, brain); + if (!previous) throw Error("history-unavailable"); + const state = previous.metadata.state; + if (time < state.lastPersistedAt || at < state.lastObservedAt || at > time) throw Error("history-clock-reversal"); + if (time - state.lastPersistedAt < HISTORY_LIMITS.writeInterval) return false; + let next: StoredHistory; + if (candidate.kind === "aggregate") { + next = retainAggregate(previous, candidate.value); + // A partial refresh may preserve the complete stored endpoint. Evaluate + // the actual new capacity observation, never give the old total a new time. + const evaluation = retainAggregate({ ...structuredClone(next), samples: next.samples.filter(row => row[1] !== 0 || bucket(0, row[3]) !== bucket(0, at)) }, candidate.value); + const incoming = candidate.value.capacities.map(row => row.descriptor); + const pressures = state.pressures.filter(value => value.severity !== "observed" || incoming.some(([identity]) => identity === value.identity)) + .map(value => value.severity === "observed" ? newPressure(value.identity, incoming.find(([identity]) => identity === value.identity)![1]) : value); + for (const [identity, rootScope] of incoming) if (!pressures.some(value => value.identity === identity) && pressures.length < 4) pressures.push(newPressure(identity, rootScope)); + next.metadata.state.pressures = pressures.map(value => observePressure(value, capacityEvidence(evaluation, value.identity, time), time)); + next.metadata.state.scopeLimit = candidate.value.reason === 3 || incoming.some(([identity]) => !pressures.some(value => value.identity === identity)); + next.metadata.state.paused = null; + } else { + next = structuredClone(previous); + for (const row of next.samples) if (bucket(row[1], row[3]) === bucket(row[1], at)) row[2] = 1; + next.metadata.state.pressures = state.pressures.map(value => observePressure(value, null, time)); + next.metadata.state.paused = "collection-unavailable"; + } + next.metadata.state.lastPersistedAt = time; + next.metadata.state.lastObservedAt = at; + next.metadata.state.lastAttemptBucket = bucket(0, at); + const episodes = new Set(next.metadata.state.pressures.flatMap(value => [value.episode, value.resolvedEpisode].filter((id): id is string => id !== null))); + next.metadata.state.suppressedEpisodes = next.metadata.state.suppressedEpisodes.filter(value => episodes.has(value)); + validateHistory(next); + db.query("UPDATE storage_history SET payload_json = ?, revision = revision + 1 WHERE brain = ?").run(JSON.stringify(next), brain); + return true; + }).immediate(); + if (changed) { if (pending === candidate) pending = null; unavailable = null; } + return changed; + } catch (error) { + unavailable = error instanceof Error && error.message.startsWith("history-") ? error.message : "history-write-unavailable"; + return false; + } + } + return { + brain, read, flush, + offer(value: Aggregate) { offer({ kind: "aggregate", value }); }, + failure(at = now()) { offer({ kind: "failure", at }); }, + status() { const history = read(); return { history, pending: pending !== null, paused: unavailable ?? history?.metadata.state.paused ?? null, payloadBytes: history ? jsonBytes(history) : null }; }, + close() { closed = true; pending = null; }, + }; +} +export type StorageHistory = ReturnType; + +/** Called inside the existing operational restore transaction. */ +export function invalidateRestoredStorage(db: Database, at: number): void { + const rows = db.query("SELECT brain, payload_json FROM storage_history").all() as Array<{ brain: string; payload_json: string }>; + for (const row of rows) { + const payload = parseStoredHistory(row.payload_json), state = payload.metadata.state; + state.suppressedEpisodes = state.pressures.flatMap(value => [value.episode, value.resolvedEpisode].filter((id): id is string => id !== null)); + state.pressures = state.pressures.map(value => ({ ...value, recoveryAt: null, supportAt: null })); + state.lastPersistedAt = Math.max(state.lastPersistedAt, at); + state.paused = "collection-unavailable"; + validateHistory(payload); + db.query("UPDATE storage_history SET payload_json = ?, revision = revision + 1 WHERE brain = ?").run(JSON.stringify(payload), row.brain); + } + db.query("UPDATE storage_notice_receipts SET state = 'suppressed', pid = NULL, process_start = NULL WHERE state NOT IN ('accepted', 'unknown')").run(); + db.query("DELETE FROM storage_scan_leases").run(); +} diff --git a/packages/ui-server/src/storage/notify.ts b/packages/ui-server/src/storage/notify.ts new file mode 100644 index 0000000000..75d47ff8b9 --- /dev/null +++ b/packages/ui-server/src/storage/notify.ts @@ -0,0 +1,90 @@ +import type { Database } from "bun:sqlite"; +import { randomUUID } from "node:crypto"; +import { isUsablePrincipal, resolvePrincipal } from "../db/principals.js"; +import { noticeTimeZone, localHour } from "../inbox/notify.js"; +import { processAlive, SELF } from "../brain/writer-gate.js"; +import type { StorageHistory } from "./history-store.js"; +import { hashScope, RECENT } from "./history-model.js"; +export type StorageNoticeOutcome = "accepted" | "failed" | "unknown" | "gone"; +export type StorageNoticeState = "pending" | "sending" | "accepted" | "failed" | "unknown" | "suppressed"; +interface Receipt { episode: string; endpoint: string; principal_id: string; state: StorageNoticeState; attempts: number; attempted_at: number | null; next_at: number | null; pid: number | null; process_start: string | null } +export interface StorageNoticeAttempt { + id: string; + principalId: string; + subscription: { endpoint: string; keys: { p256dh: string; auth: string } }; + payload: { title: "Storage needs attention."; body: "Open Storage for details."; tag: string; url: "/#/settings/storage" }; +} +/** Storage's fixed policy uses the existing subscriptions, timezone reports and transport. */ +export function createStorageNotifier(db: Database, history: StorageHistory) { + const brain = history.brain; + let lastClock = 0; + function owner(id: string, now: number) { + const principal = resolvePrincipal(db, id); + return principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, now); + } + function reap(now: number) { + const rows = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ? AND state = 'sending'").all(brain) as Receipt[]; + for (const row of rows) if (!row.pid || !row.process_start || !processAlive(row.pid, row.process_start) || row.attempted_at !== null && now - row.attempted_at >= 60_000) { + db.query("UPDATE storage_notice_receipts SET state = 'unknown', next_at = NULL, pid = NULL, process_start = NULL WHERE brain = ? AND episode = ? AND endpoint = ? AND state = 'sending'").run(brain, row.episode, row.endpoint); + } + } + return { + begin(endpoint: string, now = Date.now()): StorageNoticeAttempt | null { + if (!Number.isSafeInteger(now) || now < lastClock) return null; + lastClock = now; + return db.transaction((): StorageNoticeAttempt | null => { + if (db.query("SELECT 1 FROM inbox_recovery_state WHERE status = 'pending'").get()) return null; + reap(now); + const status = history.status(), payload = status.history; + if (!payload) return null; + const state = payload.metadata.state; + const lastAttempt = db.query("SELECT MAX(attempted_at) AS at FROM storage_notice_receipts WHERE brain = ?").get(brain) as { at: number | null }; + if (state.lastPersistedAt > now || lastAttempt.at !== null && lastAttempt.at > now) return null; + const retained = new Set(state.pressures.flatMap(row => [row.episode, row.resolvedEpisode].filter((id): id is string => id !== null))); + const active = new Set(state.pressures.filter(row => row.severity === "critical" && row.episode).map(row => row.episode!)); + const rows = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ?").all(brain) as Receipt[]; + for (const row of rows) { + if (!active.has(row.episode) && ["pending", "failed"].includes(row.state)) db.query("UPDATE storage_notice_receipts SET state = 'suppressed', next_at = NULL WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); + if (!retained.has(row.episode) && row.state !== "sending") db.query("DELETE FROM storage_notice_receipts WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); + } + const sub = db.query("SELECT principal_id, time_zone, p256dh, auth FROM push_subscriptions WHERE endpoint = ?").get(endpoint) as { principal_id: string | null; time_zone: string | null; p256dh: string; auth: string } | null; + if (!sub?.principal_id || !owner(sub.principal_id, now)) return null; + const eligible = state.pressures.filter(row => row.severity === "critical" && row.episode && !row.scopeChanged && row.supportAt !== null && row.supportAt <= now && now - row.supportAt <= RECENT && !state.suppressedEpisodes.includes(row.episode)); + for (const row of eligible) db.query("INSERT OR IGNORE INTO storage_notice_receipts (brain, episode, endpoint, principal_id, state) VALUES (?, ?, ?, ?, 'pending')").run(brain, row.episode!, endpoint, sub.principal_id); + if (status.paused || status.pending) return null; + const zone = noticeTimeZone(sub.time_zone); + if (!zone) return null; + const hour = localHour(zone, now); + if (hour >= 22 || hour < 8) return null; + const included: Receipt[] = []; + for (const pressure of eligible) { + const row = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ? AND episode = ? AND endpoint = ?").get(brain, pressure.episode!, endpoint) as Receipt; + if (row.principal_id !== sub.principal_id) { + if (["pending", "failed"].includes(row.state)) db.query("UPDATE storage_notice_receipts SET state = 'suppressed', next_at = NULL WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, endpoint); + continue; + } + if (row.attempts >= 3 || !["pending", "failed"].includes(row.state) || row.attempts > 0 && row.next_at === null || row.next_at !== null && row.next_at > now || row.attempted_at !== null && row.attempted_at > now) continue; + included.push(row); + } + if (!included.length) return null; + const id = randomUUID(); + for (const row of included) db.query("UPDATE storage_notice_receipts SET state = 'sending', attempts = attempts + 1, attempted_at = ?, next_at = NULL, batch_id = ?, pid = ?, process_start = ? WHERE brain = ? AND episode = ? AND endpoint = ?").run(now, id, SELF.pid, SELF.start, brain, row.episode, endpoint); + return { id, principalId: sub.principal_id, subscription: { endpoint, keys: { p256dh: sub.p256dh, auth: sub.auth } }, payload: { title: "Storage needs attention.", body: "Open Storage for details.", tag: `brain-storage:${hashScope(included.map(row => row.episode).sort())}`, url: "/#/settings/storage" } }; + }).immediate(); + }, + finish(id: string, outcome: StorageNoticeOutcome, now = Date.now()) { + db.transaction(() => { + const rows = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ? AND batch_id = ? AND state = 'sending'").all(brain, id) as Receipt[]; + for (const row of rows) { + const next = outcome === "failed" && row.attempts < 3 ? Math.max(now, row.attempted_at ?? now) + (row.attempts === 1 ? 60_000 : 300_000) : null; + db.query("UPDATE storage_notice_receipts SET state = ?, next_at = ?, pid = NULL, process_start = NULL WHERE brain = ? AND episode = ? AND endpoint = ? AND batch_id = ? AND state = 'sending'").run(outcome === "gone" ? "suppressed" : outcome, next, brain, row.episode, row.endpoint, id); + } + }).immediate(); + }, + receipts(principalId: string, now = Date.now()): Array<{ episode: string; state: StorageNoticeState; attempts: number; nextAt: number | null }> { + if (!owner(principalId, now)) throw Error("Storage requires current human-owner access."); + return db.query("SELECT r.episode, r.state, r.attempts, r.next_at AS nextAt FROM storage_notice_receipts r JOIN push_subscriptions s ON s.endpoint = r.endpoint WHERE r.brain = ? AND s.principal_id = ? AND r.principal_id = ? ORDER BY r.episode, r.endpoint").all(brain, principalId, principalId) as Array<{ episode: string; state: StorageNoticeState; attempts: number; nextAt: number | null }>; + }, + }; +} +export type StorageNotifier = ReturnType; diff --git a/packages/ui-server/src/storage/pressure.ts b/packages/ui-server/src/storage/pressure.ts new file mode 100644 index 0000000000..58b814880c --- /dev/null +++ b/packages/ui-server/src/storage/pressure.ts @@ -0,0 +1,92 @@ +import { randomUUID } from "node:crypto"; +import type { CapacityEvidence, WindowEvidence } from "./history-evidence.js"; +import { RECENT } from "./history-model.js"; +export type Severity = "observed" | "low" | "critical"; +export interface Refusal { + target: string; + resource: "bytes" | "quota" | "inodes" | "unknown"; + at: number; + successAt: number | null; +} +export interface Pressure { + identity: string; rootScope: string; + severity: Severity; capacity: boolean; forecast: boolean; refusals: Refusal[]; + at: number; recoveryAt: number | null; + episode: string | null; resolvedEpisode: string | null; + supportAt: number | null; scopeChanged: boolean; +} +export const newPressure = (identity: string, rootScope: string): Pressure => ({ identity, rootScope, severity: "observed", capacity: false, forecast: false, refusals: [], at: 0, recoveryAt: null, episode: null, resolvedEpisode: null, supportAt: null, scopeChanged: false }); +const rank = (severity: Severity) => severity === "critical" ? 2 : severity === "low" ? 1 : 0; +const within = (evidence: WindowEvidence, days: number) => evidence.state === "finite" && evidence.days !== null && evidence.days <= days; +const beyond = (evidence: WindowEvidence, days: number) => evidence.state === "non-depleting" || evidence.state === "finite" && evidence.days !== null && evidence.days > days; +function percentage(evidence: CapacityEvidence, threshold: number) { + return evidence.total !== null && Number.isSafeInteger(evidence.total) && evidence.total > 0 && evidence.available !== null && Number.isSafeInteger(evidence.available) && evidence.available >= 0 + ? BigInt(evidence.available) * 100n <= BigInt(evidence.total) * BigInt(threshold) : null; +} + +/** Independent causes survive missing evidence. Only a complete recovery pair can downgrade. */ +export function observePressure(previous: Pressure, evidence: CapacityEvidence | null, now: number): Pressure { + const next = structuredClone(previous); + if (!evidence || !evidence.fresh || evidence.at > now || now - evidence.at > RECENT || evidence.at < previous.at) { + next.recoveryAt = null; next.supportAt = null; next.at = Math.max(next.at, now); return next; + } + if (evidence.identity !== previous.identity || evidence.rootScope !== previous.rootScope) { + next.recoveryAt = null; next.supportAt = null; next.scopeChanged = true; return next; + } + next.scopeChanged = false; + const criticalCapacity = percentage(evidence, 5) === true, lowCapacity = percentage(evidence, 10) === true; + const criticalForecast = within(evidence.short, 7) && within(evidence.long, 7), lowForecast = within(evidence.short, 30) || within(evidence.long, 30); + next.capacity ||= lowCapacity; next.forecast ||= lowForecast; + const incoming: Severity = next.refusals.length || criticalCapacity || criticalForecast ? "critical" : lowCapacity || lowForecast ? "low" : "observed"; + const capacityRecovery = percentage(evidence, previous.severity === "critical" ? 7 : 12) === false; + const forecastRecovery = previous.severity === "critical" + ? beyond(evidence.short, 10) || beyond(evidence.long, 10) + : beyond(evidence.short, 45) && beyond(evidence.long, 45); + const refusalRecovery = next.refusals.every(refusal => refusal.resource === "bytes" && refusal.successAt !== null && refusal.successAt > refusal.at && percentage(evidence, 7) === false); + // A known matching recovery of quota/inode limits still lacks their capacity + // observation in this version. Keep those refusals explicit and unresolved. + const supportedCapacity = next.capacity && percentage(evidence, 7) === true; + const supportedForecast = next.forecast && evidence.short.state !== "unavailable" && evidence.long.state !== "unavailable" && !forecastRecovery; + const supportedRefusal = next.refusals.some(refusal => refusal.at <= now && now - refusal.at <= RECENT) && !refusalRecovery; + next.supportAt = supportedCapacity || supportedForecast || supportedRefusal ? evidence.at : null; + if (rank(incoming) > rank(previous.severity)) { + next.severity = incoming; next.recoveryAt = null; + if (incoming === "critical") { next.episode = randomUUID(); next.supportAt = evidence.at; } + } else if (previous.severity !== "observed") { + const allRecover = (!next.capacity || capacityRecovery) && (!next.forecast || forecastRecovery) && refusalRecovery; + if (!allRecover) next.recoveryAt = null; + else if (evidence.at > previous.at) { + if (next.recoveryAt === null) next.recoveryAt = evidence.at; + else if (evidence.at - next.recoveryAt >= 60_000) { + next.recoveryAt = null; next.supportAt = null; + if (previous.severity === "critical") { + next.severity = "low"; next.resolvedEpisode = next.episode; next.episode = null; + if (next.refusals.length) next.capacity = true; + next.refusals = []; + } else { + next.severity = "observed"; next.capacity = false; next.forecast = false; next.refusals = []; + } + } + } + } + next.at = Math.max(previous.at, evidence.at); + return next; +} + +/** Only an owning operation reports effects; this function never probes a write. */ +export function ownerWrite(previous: Pressure, report: { target: string; resource: Refusal["resource"]; at: number; outcome: "no-space" | "success" }, now: number): Pressure { + const next = structuredClone(previous); + if (report.at > now || report.at < next.at || !/^[a-f0-9]{32}$/.test(report.target)) { next.recoveryAt = null; return next; } + if (report.outcome === "no-space") { + if (next.severity !== "critical") next.episode = randomUUID(); + next.severity = "critical"; next.recoveryAt = null; next.supportAt = report.at; + const refusal = next.refusals.find(value => value.target === report.target && value.resource === report.resource); + if (refusal) { refusal.at = report.at; refusal.successAt = null; } + else next.refusals.push({ target: report.target, resource: report.resource, at: report.at, successAt: null }); + } else { + const refusal = next.refusals.find(value => value.target === report.target && value.resource === report.resource); + if (refusal && report.at > refusal.at) { refusal.successAt = report.at; next.recoveryAt = null; } + } + next.at = Math.max(next.at, report.at); + return next; +} diff --git a/packages/ui-server/tests/storage-history-model.test.ts b/packages/ui-server/tests/storage-history-model.test.ts new file mode 100644 index 0000000000..3c68f76dd4 --- /dev/null +++ b/packages/ui-server/tests/storage-history-model.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from "bun:test"; +import { DAY, HOUR, RECENT, HISTORY_LIMITS, bucket, bucketStart, hashScope, jsonBytes, retainAggregate, validateHistory, type Aggregate, type HistoryPayload } from "../src/storage/history-model.js"; +import { capacityEvidence } from "../src/storage/history-evidence.js"; + +const at = Date.UTC(2026, 6, 12); +const identity = hashScope("Odysseus volume"), scope = hashScope("Odysseus stores"); +const empty = (): HistoryPayload<{ marker: string }> => ({ metadata: { samples: {}, capacities: {}, state: { marker: "Odysseus" } }, samples: [] }); +function observation(time: number, available = 900_000_000, partial = false): Aggregate { + return { at: time, startedAt: time - 100, source: 0, descriptor: ["0.44.0", scope, 1], coverage: partial ? 1 : 0, reason: partial ? 1 : 0, + capacities: [{ descriptor: [identity, scope], values: [1_000_000_000, 1_000_000_000 - available, available, available, partial ? 1 : 0, 40_000, 4096, 30_000 + (time - at) / HOUR, 8192 + (time - at) / HOUR, 20_000, 4096, 90_000, 16384] }] }; +} +function steady(days = 30, available: (index: number) => number = index => 900_000_000 - index * 1_000_000) { + let payload = empty(); + for (let index = 0; index <= days * 4; index++) payload = retainAggregate(payload, observation(at + index * RECENT, available(index))); + return payload; +} + +test("one observation updates all tiers, Monday weeks and actual endpoint timestamps", () => { + const payload = retainAggregate(empty(), observation(at + 10 * HOUR)); + expect(payload.samples.map(row => row[1])).toEqual([0, 1, 2]); + expect(payload.samples.every(row => row[3] === at + 10 * HOUR)).toBe(true); + expect(new Date(bucketStart(2, bucket(2, at))).getUTCDay()).toBe(1); + expect(payload.samples.map(row => row[2])).toEqual([0, 1, 1]); +}); +test("186 full records remain bounded by tier age, slots, dictionaries and canonical bytes", () => { + let payload = empty(); + for (let index = 0; index < 240 * 4; index++) { + const input = observation(at + index * RECENT); + input.capacities = Array.from({ length: 4 }, (_, slot) => ({ descriptor: [hashScope(`Odysseus ${slot}`), scope], values: [...input.capacities[0]!.values] })); + payload = retainAggregate(payload, input); + } + expect(payload.samples.length, "STORAGE_HISTORY_186_SLOTS").toBe(186); + expect(([0, 1, 2] as const).map(tier => payload.samples.filter(row => row[1] === tier).length)).toEqual([64, 90, 32]); + const latest = payload.samples.at(-1)![3]; + expect(payload.samples.every(row => bucket(row[1], row[3]) >= bucket(row[1], latest) - HISTORY_LIMITS.slots[row[1]] + 1)).toBe(true); + expect(jsonBytes(payload)).toBeLessThanOrEqual(262144); + expect(jsonBytes(payload.metadata)).toBeLessThanOrEqual(40960); + expect(Object.keys(payload.metadata.samples)).toHaveLength(1); + expect(Object.keys(payload.metadata.capacities)).toHaveLength(4); +}); +test("worst safe numeric fields obey per-sample and combined metadata limits", () => { + const input = observation(Number.MAX_SAFE_INTEGER); + input.startedAt = input.at; + input.descriptor[0] = "v".repeat(32); + input.capacities = Array.from({ length: 4 }, (_, index) => ({ descriptor: [hashScope(index), scope], values: [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER] })); + const payload = retainAggregate(empty(), input); + expect(Math.max(...payload.samples.map(jsonBytes))).toBeLessThanOrEqual(1152); + payload.metadata.state.marker = "x".repeat(40960); + expect(() => validateHistory(payload), "STORAGE_HISTORY_SHARED_METADATA_CAP").toThrow("history-metadata-budget"); +}); +test("partial refresh preserves complete endpoints but makes bucket gaps sticky", () => { + let payload = retainAggregate(empty(), observation(at)); + payload = retainAggregate(payload, observation(at + HOUR, 100_000, true)); + expect(payload.samples.every(row => row[3] === at && row[2] === 1 && row[7] === 0)).toBe(true); + payload = retainAggregate(payload, observation(at + 2 * HOUR)); + expect(payload.samples.every(row => row[3] === at + 2 * HOUR && row[2] === 1)).toBe(true); +}); +test("missed final intervals, outages and scope changes mark actual points without filling gaps", () => { + let payload = retainAggregate(empty(), observation(at)); + payload = retainAggregate(payload, observation(at + 2 * DAY)); + expect(payload.samples.filter(row => row[1] === 0).map(row => row[3])).toEqual([at, at + 2 * DAY]); + expect(payload.samples.find(row => row[1] === 1 && row[3] === at)?.[2]).toBe(1); + const changed = observation(at + 2 * DAY + RECENT); changed.descriptor[1] = hashScope("Troy stores"); + payload = retainAggregate(payload, changed); + expect(payload.samples.filter(row => row[3] === changed.at).every(row => row[2] === 1)).toBe(true); + const later = retainAggregate(payload, observation(at + 300 * DAY)); + expect(later.samples).toHaveLength(3); +}); +test("clock reversal refuses before mutating previous evidence and descriptor references", () => { + const payload = steady(1), encoded = JSON.stringify(payload); + expect(() => retainAggregate(payload, observation(at)), "STORAGE_HISTORY_CLOCK_REVERSES_PAUSE").toThrow("history-clock-reversal"); + expect(JSON.stringify(payload)).toBe(encoded); +}); +test("dictionary churn stops at the shared byte ceiling, retaining prior valid evidence", () => { + let payload = empty(), failed = false; + for (let index = 0; index < 1500; index++) { + const input = observation(at + index * RECENT); + input.descriptor = ["v".repeat(32), hashScope(`Odysseus epoch ${index}`), 1]; + input.capacities = Array.from({ length: 4 }, (_, slot) => ({ descriptor: [hashScope(`${index}:${slot}`), hashScope(`scope ${index}:${slot}`)], values: [...input.capacities[0]!.values] })); + const before = JSON.stringify(payload); + try { payload = retainAggregate(payload, input); } + catch (error) { + expect(String(error), "STORAGE_HISTORY_DICTIONARY_CHURN_PAUSES").toContain("history-metadata-budget"); + expect(JSON.stringify(payload)).toBe(before); validateHistory(payload); failed = true; break; + } + } + expect(failed).toBe(true); +}); +test("short/long scenarios and observed App growth use distinct endpoint quantities", () => { + const payload = steady(), now = at + 30 * DAY, result = capacityEvidence(payload, identity, now)!; + expect(result.short.state).toBe("finite"); expect(result.long.state).toBe("finite"); + expect(result.short.depletionPerDay).toBe(4_000_000); + expect(result.long.depletionPerDay).toBe(4_000_000); + expect(result.short.days).toBe(195); expect(result.long.days).toBe(195); + expect(result.appAllocated?.bytesPerDay).toBe(24); + expect(result.availableChange?.bytesPerDay).toBe(-4_000_000); + expect(result.long.samples).toBeLessThanOrEqual(29); +}); +test("minimum spans, stale/future evidence and gaps refuse scenarios without infinity", () => { + const payload = steady(1), now = at + DAY; + expect(capacityEvidence(payload, identity, now)?.short.reason).toBe("insufficient-span"); + expect(capacityEvidence(payload, identity, now + RECENT + 1)?.short.reason).toBe("stale"); + expect(capacityEvidence(payload, identity, now - 1)?.short.reason).toBe("future"); + const gapped = steady(); gapped.samples.filter(row => row[1] === 0).at(-2)![2] = 1; + expect(capacityEvidence(gapped, identity, at + 30 * DAY)?.short.state).toBe("unavailable"); + const longGap = steady(); longGap.samples.filter(row => row[1] === 1).at(-2)![2] = 1; + expect(capacityEvidence(longGap, identity, at + 30 * DAY)?.long.state).toBe("unavailable"); +}); +test("flat and smooth replenishment are qualified non-depletion; burst and zero-net reversal refuse", () => { + for (const values of [() => 500_000_000, (i: number) => 500_000_000 + i * 1_000_000]) { + const result = capacityEvidence(steady(30, values), identity, at + 30 * DAY)!; + expect(result.short.state).toBe("non-depleting"); expect(result.short.days).toBeNull(); + expect(result.long.state).toBe("non-depleting"); + } + const burst = steady(30, i => i < 119 ? 800_000_000 : 400_000_000); + expect(capacityEvidence(burst, identity, at + 30 * DAY)?.short.reason).toBe("single-burst"); + const mixed = steady(30, i => 500_000_000 + (i % 2) * 1_000_000); + expect(capacityEvidence(mixed, identity, at + 30 * DAY)?.short.reason).toBe("mixed-direction"); +}); +test("changed capacity, roots, unknown allocation and unrounded zero horizon preserve evidence boundaries", () => { + const payload = steady(); + const latest = payload.samples.filter(row => row[1] === 0).at(-1)!; + latest[9][0]![1] = 2_000_000_000; + expect(capacityEvidence(payload, identity, at + 30 * DAY)?.short.state, "STORAGE_HISTORY_COMPARABLE_TOTAL").toBe("unavailable"); + const absent = steady(); absent.samples.filter(row => row[1] === 0).at(-2)![9][0]![9] = null; + expect(capacityEvidence(absent, identity, at + 30 * DAY)?.appAllocated).toBeNull(); + const zero = steady(30, i => (120 - i) * 1_000_000); + expect(capacityEvidence(zero, identity, at + 30 * DAY)?.short.days).toBe(0); +}); diff --git a/packages/ui-server/tests/storage-history-store.test.ts b/packages/ui-server/tests/storage-history-store.test.ts new file mode 100644 index 0000000000..f7e2e03c7a --- /dev/null +++ b/packages/ui-server/tests/storage-history-store.test.ts @@ -0,0 +1,97 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { Database } from "bun:sqlite"; +import { createUiDb } from "../src/db/client.js"; +import { createStorageHistory, invalidateRestoredStorage, parseStoredHistory } from "../src/storage/history-store.js"; +import { hashScope, RECENT, type Aggregate } from "../src/storage/history-model.js"; +const roots: string[] = [], databases: Database[] = []; +afterEach(() => { for (const db of databases.splice(0)) db.close(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +const start = Date.UTC(2026, 6, 12, 9, 41), identity = hashScope("Odysseus volume"), scope = hashScope("Odysseus stores"); +const aggregate = (at: number, available = 500_000_000): Aggregate => ({ at, startedAt: at - 10, source: 1, descriptor: ["0.44.0", scope, 1], coverage: 0, reason: 0, capacities: [{ descriptor: [identity, scope], values: [1_000_000_000, 1_000_000_000 - available, available, available, 0, 1, 4096, 2, 8192, 0, 0, 3, 12288] }] }); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-history-")); roots.push(root); + const path = join(root, "operations.sqlite"), db = createUiDb(path); databases.push(db); + let now = start; + const history = createStorageHistory(db, root, { now: () => now }); + return { root, path, db, history, advance(value: number) { now = value; } }; +} +test("SQLite migrations retain tiers and stable opaque identity across reopen; only newest pending aggregate survives", () => { + const f = fixture(), salt = f.history.read()!.metadata.state.salt; + f.history.offer(aggregate(start)); + const first = f.history.read()!, revision = () => (f.db.query("SELECT revision FROM storage_history").get() as { revision: number }).revision; + expect(first.samples).toHaveLength(3); expect(revision()).toBe(1); + for (let index = 1; index < 60; index++) { f.advance(start + index * 1000); f.history.offer(aggregate(start + index * 1000, 500_000_000 - index)); } + expect(revision(), "STORAGE_HISTORY_ONE_TRANSACTION_PER_MINUTE").toBe(1); + expect(f.history.status().pending).toBe(true); + f.advance(start + 60_000); expect(f.history.flush()).toBe(true); expect(revision()).toBe(2); + expect(f.history.read()!.samples.every(row => row[3] === start + 59_000)).toBe(true); + f.history.close(); + const db = createUiDb(f.path); databases.push(db); + const reopened = createStorageHistory(db, f.root, { now: () => start + 60_001 }); + expect(reopened.read()!.metadata.state.salt, "STORAGE_HISTORY_STABLE_IDENTITY").toBe(salt); + expect(reopened.status().pending).toBe(false); expect(reopened.read()!.samples).toEqual(f.history.read()!.samples); +}); +test("an actual SQLite page quota preserves the previous valid payload when a history write fails", () => { + const f = fixture(); + const before = (f.db.query("SELECT payload_json FROM storage_history").get() as { payload_json: string }).payload_json; + f.db.exec("CREATE TABLE storage_test_ballast (value BLOB)"); + f.db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + const pages = (f.db.query("PRAGMA page_count").get() as { page_count: number }).page_count; + f.db.exec(`PRAGMA max_page_count = ${pages}`); + // Fill any already-free pages, so the next new history overflow page hits + // SQLite's real SQLITE_FULL boundary instead of a mocked database method. + for (;;) { try { f.db.query("INSERT INTO storage_test_ballast VALUES (zeroblob(4096))").run(); } catch { break; } } + const input = aggregate(start); + input.capacities = Array.from({ length: 4 }, (_, index) => ({ descriptor: [hashScope(index), scope], values: [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0, 0, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER] })); + f.history.offer(input); + expect(f.history.status().paused, "STORAGE_HISTORY_SQLITE_FULL_PRESERVES_EVIDENCE").toBe("history-write-unavailable"); + expect((f.db.query("SELECT payload_json FROM storage_history").get() as { payload_json: string }).payload_json).toBe(before); + expect(f.history.status().pending).toBe(true); +}); +test("failed refresh and clock reversal never manufacture new endpoints or clear warning causes", () => { + const f = fixture(); f.history.offer(aggregate(start, 50_000_000)); + expect(f.history.read()!.metadata.state.pressures[0]!.severity).toBe("critical"); + f.advance(start + 60_000); f.history.failure(); + const after = f.history.read()!; + expect(after.samples.every(row => row[3] === start && row[2] === 1)).toBe(true); + expect(after.metadata.state.pressures[0]!.severity).toBe("critical"); + expect(after.metadata.state.pressures[0]!.supportAt).toBeNull(); + f.advance(start - 1); f.history.offer(aggregate(start - 1)); + expect(f.history.status().paused).toBe("history-clock-reversal"); + expect(f.history.read()).toEqual(after); +}); +test("operational restore suppresses every old active episode before fresh observations can dispatch", () => { + const f = fixture(); f.history.offer(aggregate(start, 50_000_000)); + const episode = f.history.read()!.metadata.state.pressures[0]!.episode!; + f.db.transaction(() => invalidateRestoredStorage(f.db, start + 60_000)).immediate(); + const restored = f.history.read()!; + expect(restored.metadata.state.suppressedEpisodes, "STORAGE_RESTORE_INVALIDATES_DELIVERY_AUTHORITY").toContain(episode); + expect(restored.metadata.state.pressures[0]!.supportAt).toBeNull(); + f.advance(start + 120_000); f.history.offer(aggregate(start + 120_000, 40_000_000)); + expect(f.history.read()!.metadata.state.suppressedEpisodes).toContain(episode); + expect(f.history.read()!.metadata.state.pressures[0]!.episode).toBe(episode); +}); +test("186 four-capacity observations charge serialized history separately from actual database and WAL allocation", () => { + const f = fixture(); + const physical = () => ([f.path, f.path + "-wal", f.path + "-shm"] as const).map(path => { try { const value = statSync(path); return { file: path.slice(f.root.length + 1), logical: value.size, allocated: value.blocks * 512 }; } catch { return { file: path.slice(f.root.length + 1), logical: 0, allocated: 0 }; } }); + const before = physical(); + for (let index = 0; index < 230 * 4; index++) { + const at = start + index * RECENT, input = aggregate(at); + input.capacities = Array.from({ length: 4 }, (_, slot) => ({ descriptor: [hashScope(`Odysseus ${slot}`), scope], values: [...input.capacities[0]!.values] })); + f.advance(at); f.history.offer(input); + } + const result = f.history.status(), after = physical(); + expect(result.history!.samples.length, "STORAGE_HISTORY_SQLITE_186_RECORDS").toBe(186); + expect(result.payloadBytes).toBeLessThanOrEqual(262144); + expect(after.reduce((sum, row) => sum + row.allocated, 0)).toBeGreaterThan(result.payloadBytes!); + // This is a measured fixture receipt, never a production byte-size promise. + console.log("STORAGE_HISTORY_PHYSICAL_RECEIPT", JSON.stringify({ samples: 186, capacities: 4, payloadBytes: result.payloadBytes, before, after })); +}); +test("malformed persisted warning authority is rejected instead of interpreted as Critical", () => { + const f = fixture(); f.history.offer(aggregate(start)); + const payload = f.history.read()!; + (payload.metadata.state.pressures[0] as unknown as { severity: string }).severity = "unrecognised"; + expect(() => parseStoredHistory(JSON.stringify(payload))).toThrow(); +}); diff --git a/packages/ui-server/tests/storage-notify.test.ts b/packages/ui-server/tests/storage-notify.test.ts new file mode 100644 index 0000000000..766ab3158a --- /dev/null +++ b/packages/ui-server/tests/storage-notify.test.ts @@ -0,0 +1,122 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createUiDb } from "../src/db/client.js"; +import { createPrincipal, revokePrincipal } from "../src/db/principals.js"; +import { createPushSender } from "../src/activity/push-sender.js"; +import { createActionNotifier } from "../src/inbox/notify.js"; +import { createStorageHistory, invalidateRestoredStorage } from "../src/storage/history-store.js"; +import { createStorageNotifier } from "../src/storage/notify.js"; +import { hashScope, HOUR, type Aggregate } from "../src/storage/history-model.js"; +const close: Array<() => void> = []; +afterEach(() => { for (const f of close.splice(0).reverse()) f(); }); +const start = Date.UTC(2026, 6, 12, 9, 41), scope = hashScope("Odysseus stores"); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-notices-")); close.push(() => rmSync(root, { recursive: true, force: true })); + const path = join(root, "operations.sqlite"), db = createUiDb(path); close.push(() => db.close()); + let now = start, mode: "accepted" | "failed" | "unknown" = "accepted"; + let before: ((endpoint: string) => void) | null = null; + const sent: Array<{ endpoint: string; payload: Record; p256dh: string }> = []; + const history = createStorageHistory(db, root, { now: () => now }), notices = createStorageNotifier(db, history); + const sender = createPushSender(db, { send: async (sub, payload) => { + sent.push({ endpoint: sub.endpoint, payload: JSON.parse(payload), p256dh: sub.keys.p256dh }); before?.(sub.endpoint); + if (mode === "failed") throw Object.assign(Error("provider refused"), { statusCode: 500 }); + if (mode === "unknown") throw Error("reply unavailable"); + } }); + const owner = createPrincipal(db, { authMethod: "password", label: "Odysseus", ttlSeconds: 365 * 24 * 3600 }); + const zones = createActionNotifier(db, { now: () => now }); + const device = (name: string, zone: string | null = "UTC", principalId = owner.id) => { + const endpoint = `https://push.example.test/${name}`; + expect(sender.subscribe({ endpoint, keys: { p256dh: name, auth: "opaque" } }, principalId)).toBe(true); + if (zone) zones.reportDeviceZone(principalId, endpoint, zone, now); + return endpoint; + }; + function observe(at: number, available = 5000, capacities = 1) { + now = at; + const value: Aggregate = { at, startedAt: at - 10, source: 1, descriptor: ["0.44.0", scope, 1], coverage: 0, reason: 0, + capacities: Array.from({ length: capacities }, (_, index) => ({ descriptor: [hashScope(`Odysseus volume ${index}`), scope], values: [100_000, 100_000 - available, available, available, 0, 1, 4096, 1, 4096, 0, 0, 2, 8192] })) }; + history.offer(value); + } + return { root, path, db, owner, history, notices, sender, sent, zones, device, observe, + at(value: number) { now = value; }, mode(value: typeof mode) { mode = value; }, before(value: typeof before) { before = value; }, + deliver: () => sender.deliverStorage(notices, now) }; +} +test("multiple Critical capacities coalesce into one generic push and durably record every episode", async () => { + const f = fixture(); f.device("Ithaca"); f.observe(start, 5000, 4); + expect(await f.deliver()).toBe(1); expect(f.sent).toHaveLength(1); + expect(f.sent[0]!.payload).toEqual({ title: "Storage needs attention.", body: "Open Storage for details.", tag: expect.stringMatching(/^brain-storage:[a-f0-9]{32}$/), url: "/#/settings/storage" }); + const rows = f.notices.receipts(f.owner.id, start); expect(rows).toHaveLength(4); expect(rows.every(row => row.state === "accepted" && row.attempts === 1)).toBe(true); + expect(await f.deliver(), "STORAGE_PUSH_DURABLE_DEDUP").toBe(0); + const db = createUiDb(f.path); close.push(() => db.close()); + const other = createStorageNotifier(db, createStorageHistory(db, f.root)); + expect(await f.sender.deliverStorage(other, start + 60_000)).toBe(0); +}); +test("known refusal retries only after one/five minutes and never after the third attempt", async () => { + const f = fixture(); f.device("Ithaca"); f.observe(start); f.mode("failed"); + expect(await f.deliver()).toBe(1); + f.at(start + 59_999); expect(await f.deliver()).toBe(0); + f.at(start + 60_000); expect(await f.deliver()).toBe(1); + f.at(start + 359_999); expect(await f.deliver()).toBe(0); + f.at(start + 360_000); expect(await f.deliver()).toBe(1); + f.at(start + HOUR); expect(await f.deliver()).toBe(0); + expect(f.notices.receipts(f.owner.id, start + HOUR)[0]).toMatchObject({ state: "failed", attempts: 3, nextAt: null }); +}); +test("ambiguous acceptance and an abandoned durable intent never trigger an automatic retry", async () => { + const f = fixture(), endpoint = f.device("Ithaca"); f.observe(start); f.mode("unknown"); + await f.deliver(); f.at(start + HOUR); expect(await f.deliver()).toBe(0); + expect(f.notices.receipts(f.owner.id, start + HOUR)[0]?.state).toBe("unknown"); + const other = f.device("Troy"); + const attempt = f.notices.begin(other, start + HOUR); expect(attempt).not.toBeNull(); + f.db.query("UPDATE storage_notice_receipts SET pid = 2147483647, process_start = 'missing' WHERE endpoint = ?").run(other); + expect(f.notices.begin(other, start + HOUR + 1), "STORAGE_PUSH_ABANDONED_INTENT_UNKNOWN").toBeNull(); + expect(f.notices.receipts(f.owner.id, start + HOUR).filter(row => row.state === "unknown")).toHaveLength(2); + expect(f.sent.map(row => row.endpoint)).toEqual([endpoint]); +}); +test("unknown zone, strict quiet hours, stale evidence and clock reversal defer without delivery scans", async () => { + const f = fixture(), endpoint = f.device("Ithaca", null); f.observe(start); + expect(await f.deliver()).toBe(0); + f.zones.reportDeviceZone(f.owner.id, endpoint, "Europe/Athens", start); + const evening = Date.UTC(2026, 6, 12, 19); f.observe(evening); expect(await f.deliver()).toBe(0); + f.at(Date.UTC(2026, 6, 13, 4, 59, 59)); expect(await f.deliver()).toBe(0); + const morning = Date.UTC(2026, 6, 13, 5); f.at(morning); expect(await f.deliver()).toBe(0); // stale, no delivery-only refresh + f.observe(morning); f.at(morning - 1); expect(await f.deliver()).toBe(0); + f.at(morning); expect(await f.deliver()).toBe(1); +}); +test("DST boundaries use the destination's authenticated current timezone", async () => { + const f = fixture(); f.device("Ithaca", "Europe/Athens"); + f.observe(Date.UTC(2026, 9, 24, 19)); expect(await f.deliver()).toBe(0); // 22:00 before fallback + f.observe(Date.UTC(2026, 9, 25, 1)); expect(await f.deliver()).toBe(0); // repeated 03:00 + f.observe(Date.UTC(2026, 9, 25, 5, 59)); expect(await f.deliver()).toBe(0); + f.observe(Date.UTC(2026, 9, 25, 6)); expect(await f.deliver()).toBe(1); // 08:00 after fallback +}); +test("revocation and rebinding are rechecked for each actual transport attempt", async () => { + const f = fixture(), first = f.device("Ithaca"), second = f.device("Troy"); f.observe(start); + f.before(endpoint => { if (endpoint === first) revokePrincipal(f.db, f.owner.id, start); }); + expect(await f.deliver(), "STORAGE_PUSH_CURRENT_OWNER_AUTHORITY").toBe(1); + expect(f.sent.map(row => row.endpoint)).toEqual([first]); + expect(() => f.notices.receipts(f.owner.id, start)).toThrow(); + const other = fixture(), one = other.device("Ithaca"), two = other.device("Troy"); other.observe(start); + other.before(endpoint => { if (endpoint === one) other.sender.subscribe({ endpoint: two, keys: { p256dh: "new-key", auth: "new-auth" } }, other.owner.id); }); + expect(await other.deliver()).toBe(2); expect(other.sent.find(row => row.endpoint === two)?.p256dh).toBe("new-key"); + expect(second).toContain("Troy"); +}); +test("confirmed recovery suppresses pending notices and restoration never replays an old pending episode", async () => { + const f = fixture(); f.device("Ithaca", null); f.observe(start); await f.deliver(); + f.observe(start + 60_000, 8000); f.observe(start + 120_000, 8000); await f.deliver(); + expect(f.notices.receipts(f.owner.id, start + 120_000)[0]?.state).toBe("suppressed"); + const other = fixture(), endpoint = other.device("Troy", null); other.observe(start); await other.deliver(); + other.db.transaction(() => invalidateRestoredStorage(other.db, start + 60_000)).immediate(); + other.zones.reportDeviceZone(other.owner.id, endpoint, "UTC", start + 60_000); + other.observe(start + 120_000); expect(await other.deliver(), "STORAGE_PUSH_RESTORE_NEVER_REPLAYS").toBe(0); + expect(other.sent).toHaveLength(0); +}); + +test("delegated principals cannot dispatch or read Storage notice receipts", async () => { + const f = fixture(); + const agent = createPrincipal(f.db, { authMethod: "delegated", createdBy: f.owner.id, label: "Odysseus agent", ttlSeconds: 3600 }); + f.device("Troy", "UTC", agent.id); f.observe(start); + expect(await f.deliver(), "STORAGE_PUSH_DELEGATED_OWNER_DENIED").toBe(0); + expect(() => f.notices.receipts(agent.id, start)).toThrow(); + expect(f.db.query("SELECT COUNT(*) AS count FROM storage_notice_receipts").get()).toEqual({ count: 0 }); +}); diff --git a/packages/ui-server/tests/storage-pressure.test.ts b/packages/ui-server/tests/storage-pressure.test.ts new file mode 100644 index 0000000000..6d32e88c28 --- /dev/null +++ b/packages/ui-server/tests/storage-pressure.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "bun:test"; +import { newPressure, observePressure, ownerWrite } from "../src/storage/pressure.js"; +import type { CapacityEvidence, WindowEvidence } from "../src/storage/history-evidence.js"; +import { hashScope, HOUR, RECENT } from "../src/storage/history-model.js"; +const at = Date.UTC(2026, 6, 12, 9, 41), identity = hashScope("Odysseus volume"), rootScope = hashScope("Odysseus stores"), target = hashScope("Odysseus attachment"); +const unavailable: WindowEvidence = { state: "unavailable", reason: "insufficient-count", start: null, end: null, samples: 0, depletionPerDay: null, days: null }; +const finite = (days: number): WindowEvidence => ({ state: "finite", reason: null, start: at - 2 * 24 * HOUR, end: at, samples: 9, depletionPerDay: 100, days }); +const nondepleting: WindowEvidence = { ...finite(0), state: "non-depleting", days: null }; +const evidence = (time: number, available: number, short = unavailable, long = unavailable): CapacityEvidence => ({ identity, rootScope, at: time, total: 100_000, available, fresh: true, short, long, appLogical: null, appAllocated: null, availableChange: null }); + +test("capacity exact entry/recovery boundaries require two additional observations after downgrade", () => { + let value = newPressure(identity, rootScope); + value = observePressure(value, evidence(at, 10_001), at); expect(value.severity).toBe("observed"); + value = observePressure(value, evidence(at + 1, 10_000), at + 1); expect(value.severity).toBe("low"); + value = observePressure(value, evidence(at + 2, 5000), at + 2); expect(value.severity).toBe("critical"); + const episode = value.episode; expect(episode).toBeTruthy(); + value = observePressure(value, evidence(at + 60_002, 7000), at + 60_002); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, evidence(at + 120_002, 7001), at + 120_002); expect(value.severity).toBe("critical"); + value = observePressure(value, evidence(at + 180_001, 7001), at + 180_001); expect(value.severity).toBe("critical"); + value = observePressure(value, evidence(at + 180_002, 7001), at + 180_002); expect(value.severity, "STORAGE_PRESSURE_TWO_FRESH_OBSERVATIONS").toBe("low"); + expect(value.resolvedEpisode).toBe(episode); expect(value.episode).toBeNull(); + value = observePressure(value, evidence(at + 240_002, 12000), at + 240_002); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, evidence(at + 300_002, 12001), at + 300_002); expect(value.severity).toBe("low"); + value = observePressure(value, evidence(at + 360_002, 12001), at + 360_002); expect(value.severity).toBe("observed"); +}); +test("forecast entry keeps either/both distinction and missing evidence cannot recover active causes", () => { + let value = observePressure(newPressure(identity, rootScope), evidence(at, 50000, finite(7), unavailable), at); + expect(value.severity).toBe("low"); + value = observePressure(value, evidence(at + 60_000, 50000, finite(7), finite(7)), at + 60_000); expect(value.severity).toBe("critical"); + for (let index = 2; index < 5; index++) value = observePressure(value, evidence(at + index * 60_000, 50000), at + index * 60_000); + expect(value.severity, "STORAGE_PRESSURE_UNKNOWN_NEVER_RECOVERS").toBe("critical"); + value = observePressure(value, evidence(at + 5 * 60_000, 50000, finite(10), unavailable), at + 5 * 60_000); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, evidence(at + 6 * 60_000, 50000, finite(10.00001), unavailable), at + 6 * 60_000); + value = observePressure(value, evidence(at + 7 * 60_000, 50000, nondepleting, unavailable), at + 7 * 60_000); expect(value.severity).toBe("low"); + value = observePressure(value, evidence(at + 8 * 60_000, 50000, finite(50), unavailable), at + 8 * 60_000); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, evidence(at + 9 * 60_000, 50000, finite(45.001), nondepleting), at + 9 * 60_000); + value = observePressure(value, evidence(at + 10 * 60_000, 50000, finite(45.001), nondepleting), at + 10 * 60_000); expect(value.severity).toBe("observed"); +}); +test("every active cause must recover; stale, reversal, failed and incompatible observations reset pairs", () => { + let value = observePressure(newPressure(identity, rootScope), evidence(at, 5000, finite(7), finite(7)), at); + value = observePressure(value, evidence(at + 60_000, 13000), at + 60_000); expect(value.recoveryAt).toBeNull(); + const good = (time: number) => evidence(time, 13000, nondepleting, nondepleting); + value = observePressure(value, good(at + 120_000), at + 120_000); expect(value.recoveryAt).not.toBeNull(); + value = observePressure(value, null, at + 130_000); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, good(at + 180_000), at + 180_000); expect(value.severity).toBe("critical"); + value = observePressure(value, good(at + 170_000), at + 180_000); expect(value.recoveryAt).toBeNull(); + value = observePressure(value, { ...good(at + 240_000), rootScope: hashScope("Troy stores") }, at + 240_000); expect(value.scopeChanged).toBe(true); expect(value.supportAt).toBeNull(); + value = observePressure(value, good(at + 300_000), at + 300_000); + value = observePressure(value, good(at + 300_000), at + 300_000 + RECENT + 1); expect(value.recoveryAt).toBeNull(); expect(value.severity).toBe("critical"); +}); +test("owner refusal requires matching target/resource success and the fresh capacity recovery pair", () => { + let value = ownerWrite(newPressure(identity, rootScope), { target, resource: "bytes", at, outcome: "no-space" }, at); + const good = (time: number) => evidence(time, 50000); + for (let index = 1; index <= 3; index++) value = observePressure(value, good(at + index * 60_000), at + index * 60_000); + expect(value.severity).toBe("critical"); + value = ownerWrite(value, { target: hashScope("Troy attachment"), resource: "bytes", at: at + 240_000, outcome: "success" }, at + 240_000); + expect(value.refusals[0]?.successAt).toBeNull(); + value = ownerWrite(value, { target, resource: "quota", at: at + 300_000, outcome: "success" }, at + 300_000); expect(value.refusals[0]?.successAt).toBeNull(); + value = ownerWrite(value, { target, resource: "bytes", at: at + 360_000, outcome: "success" }, at + 360_000); + value = observePressure(value, good(at + 420_000), at + 420_000); + value = observePressure(value, good(at + 480_000), at + 480_000); expect(value.severity, "STORAGE_REFUSAL_MATCHING_SUCCESS").toBe("low"); + expect(value.refusals).toHaveLength(0); expect(value.capacity).toBe(true); +}); + +test("a later byte refusal cannot replace unresolved quota or another target's refusal", () => { + let value = ownerWrite(newPressure(identity, rootScope), { target, resource: "quota", at, outcome: "no-space" }, at); + const other = hashScope("Troy attachment"); + value = ownerWrite(value, { target: other, resource: "bytes", at: at + 1, outcome: "no-space" }, at + 1); + value = ownerWrite(value, { target: other, resource: "bytes", at: at + 2, outcome: "success" }, at + 2); + value = observePressure(value, evidence(at + 60_002, 50000), at + 60_002); + value = observePressure(value, evidence(at + 120_002, 50000), at + 120_002); + expect(value.severity, "STORAGE_REFUSALS_REMAIN_INDEPENDENT").toBe("critical"); + expect(value.refusals).toHaveLength(2); + expect(value.refusals[0]?.resource).toBe("quota"); +}); From 6c5c3a0b6efc4aa75ad64cd3f0a764ada2a5d61a Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 17:25:34 +0200 Subject: [PATCH 13/44] CONTRACT: feat(storage): share bounded sampling and expose owner guidance --- .changeset/storage-history-guidance.md | 1 + api-report/ui-sdk.txt | 188 ++++++++++++++++++ docs/README.md | 3 + docs/decisions/claude-code-runtime.md | 4 +- docs/decisions/http-api-boundary.md | 8 +- docs/decisions/scheduled-tasks.md | 2 +- docs/decisions/session-principals.md | 6 +- docs/http-api-coverage.md | 13 +- docs/http-api.md | 49 ++++- docs/inbox-notifications.md | 2 +- docs/integration-contract/http.md | 12 ++ .../plans/async-collaboration-requirements.md | 2 +- docs/plans/async-collaboration.md | 2 +- docs/plans/navigation-recovery.md | 2 +- docs/plans/storage-accounting.md | 4 +- docs/storage-history.md | 92 +++++++++ packages/ui-sdk/src/protocol.ts | 1 + packages/ui-sdk/src/storage-guidance.ts | 53 +++++ .../migrations/053_storage_guidance.sql | 6 + .../ui-server/src/activity/push-sender.ts | 5 +- packages/ui-server/src/app.ts | 37 +++- packages/ui-server/src/inbox/snapshot.ts | 3 + packages/ui-server/src/routes/storage.ts | 7 +- packages/ui-server/src/storage/collect.ts | 7 +- packages/ui-server/src/storage/coordinator.ts | 88 ++++++-- packages/ui-server/src/storage/guidance.ts | 52 +++++ .../ui-server/src/storage/history-model.ts | 2 +- .../ui-server/src/storage/history-store.ts | 70 +++++-- packages/ui-server/src/storage/maintenance.ts | 3 + packages/ui-server/src/storage/notify.ts | 31 ++- packages/ui-server/src/storage/process.ts | 29 ++- packages/ui-server/src/storage/removal.ts | 17 +- packages/ui-server/src/storage/sampling.ts | 40 ++++ packages/ui-server/src/storage/shared-scan.ts | 177 +++++++++++++++++ .../tests/fixtures/storage-notify-process.ts | 15 ++ .../tests/fixtures/storage-shared-process.ts | 41 ++++ .../tests/http-supported-mounts.test.ts | 2 + .../ui-server/tests/storage-guidance.test.ts | 100 ++++++++++ .../tests/storage-history-model.test.ts | 8 + .../tests/storage-history-store.test.ts | 44 ++++ .../ui-server/tests/storage-notify.test.ts | 63 ++++++ .../ui-server/tests/storage-pressure.test.ts | 2 +- .../ui-server/tests/storage-routes.test.ts | 2 +- .../tests/storage-shared-scan.test.ts | 132 ++++++++++++ 44 files changed, 1346 insertions(+), 81 deletions(-) create mode 100644 docs/storage-history.md create mode 100644 packages/ui-sdk/src/storage-guidance.ts create mode 100644 packages/ui-server/src/storage/guidance.ts create mode 100644 packages/ui-server/src/storage/sampling.ts create mode 100644 packages/ui-server/src/storage/shared-scan.ts create mode 100644 packages/ui-server/tests/fixtures/storage-notify-process.ts create mode 100644 packages/ui-server/tests/fixtures/storage-shared-process.ts create mode 100644 packages/ui-server/tests/storage-guidance.test.ts create mode 100644 packages/ui-server/tests/storage-shared-scan.test.ts diff --git a/.changeset/storage-history-guidance.md b/.changeset/storage-history-guidance.md index c11531cae9..acd3609dbf 100644 --- a/.changeset/storage-history-guidance.md +++ b/.changeset/storage-history-guidance.md @@ -1,5 +1,6 @@ --- "@schlessera/brain-ui-server": minor +"@schlessera/brain-ui-sdk": minor --- Keep bounded Recent, Daily and Weekly Storage observations, qualified capacity scenarios and independent pressure causes. Retain warning and notification evidence across restart, use existing owner-authorized push subscriptions, and suppress ambiguous or restored delivery attempts. diff --git a/api-report/ui-sdk.txt b/api-report/ui-sdk.txt index b7555f1155..cf33231211 100644 --- a/api-report/ui-sdk.txt +++ b/api-report/ui-sdk.txt @@ -306,18 +306,27 @@ export "." (./src/index.ts) type SpeechCapabilities type StateRoomObservation type StorageBytes + type StorageCapacityEvidence type StorageCapacityView type StorageCoverage + type StorageEvidenceReason type StorageFileEntry type StorageFilter + type StorageGuidanceResponse + type StorageHistoryCapacity + type StorageHistoryObservation + type StorageHistoryResponse + type StorageHistoryTier type StorageMaintenanceItem type StorageMaintenanceKind type StorageMaintenanceObservation type StorageMaintenanceReceipt type StorageMaintenanceRequest type StorageMaintenanceRow + type StorageObservedRate type StorageOwner type StoragePage + type StoragePressureView type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile @@ -329,6 +338,8 @@ export "." (./src/index.ts) type StorageRemovalStartRequest type StorageRemovalUndoRequest type StorageRoot + type StorageScenario + type StorageSeverity type StorageSnapshot type StorageStatus type StorageTreeFilter @@ -782,18 +793,27 @@ export "./client" (./src/client/index.ts) type SpeechCapabilities type StateRoomObservation type StorageBytes + type StorageCapacityEvidence type StorageCapacityView type StorageCoverage + type StorageEvidenceReason type StorageFileEntry type StorageFilter + type StorageGuidanceResponse + type StorageHistoryCapacity + type StorageHistoryObservation + type StorageHistoryResponse + type StorageHistoryTier type StorageMaintenanceItem type StorageMaintenanceKind type StorageMaintenanceObservation type StorageMaintenanceReceipt type StorageMaintenanceRequest type StorageMaintenanceRow + type StorageObservedRate type StorageOwner type StoragePage + type StoragePressureView type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile @@ -805,6 +825,8 @@ export "./client" (./src/client/index.ts) type StorageRemovalStartRequest type StorageRemovalUndoRequest type StorageRoot + type StorageScenario + type StorageSeverity type StorageSnapshot type StorageStatus type StorageTreeFilter @@ -1413,18 +1435,27 @@ export "./protocol" (./src/protocol.ts) type SpeechCapabilities type StateRoomObservation type StorageBytes + type StorageCapacityEvidence type StorageCapacityView type StorageCoverage + type StorageEvidenceReason type StorageFileEntry type StorageFilter + type StorageGuidanceResponse + type StorageHistoryCapacity + type StorageHistoryObservation + type StorageHistoryResponse + type StorageHistoryTier type StorageMaintenanceItem type StorageMaintenanceKind type StorageMaintenanceObservation type StorageMaintenanceReceipt type StorageMaintenanceRequest type StorageMaintenanceRow + type StorageObservedRate type StorageOwner type StoragePage + type StoragePressureView type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile @@ -1436,6 +1467,8 @@ export "./protocol" (./src/protocol.ts) type StorageRemovalStartRequest type StorageRemovalUndoRequest type StorageRoot + type StorageScenario + type StorageSeverity type StorageSnapshot type StorageStatus type StorageTreeFilter @@ -2115,18 +2148,27 @@ export "./server" (./src/server/index.ts) type StartTurnRequest type StateRoomObservation type StorageBytes + type StorageCapacityEvidence type StorageCapacityView type StorageCoverage + type StorageEvidenceReason type StorageFileEntry type StorageFilter + type StorageGuidanceResponse + type StorageHistoryCapacity + type StorageHistoryObservation + type StorageHistoryResponse + type StorageHistoryTier type StorageMaintenanceItem type StorageMaintenanceKind type StorageMaintenanceObservation type StorageMaintenanceReceipt type StorageMaintenanceRequest type StorageMaintenanceRow + type StorageObservedRate type StorageOwner type StoragePage + type StoragePressureView type StorageReferenceEvidence type StorageRemovalCapability type StorageRemovalFile @@ -2138,6 +2180,8 @@ export "./server" (./src/server/index.ts) type StorageRemovalStartRequest type StorageRemovalUndoRequest type StorageRoot + type StorageScenario + type StorageSeverity type StorageSnapshot type StorageStatus type StorageTreeFilter @@ -9297,6 +9341,20 @@ signatures (every public export and the types it is made of) allocated: number | null; coverage: StorageCoverage; } + StorageCapacityEvidence (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageCapacityEvidence { + identity: string; + rootScope: string; + at: number; + total: number | null; + available: number | null; + fresh: boolean; + short: StorageScenario; + long: StorageScenario; + appLogical: StorageObservedRate | null; + appAllocated: StorageObservedRate | null; + availableChange: StorageObservedRate | null; + } StorageCapacityView (packages/ui-sdk/src/storage.ts) export interface StorageCapacityView { id: string; @@ -9315,6 +9373,8 @@ signatures (every public export and the types it is made of) } StorageCoverage (packages/ui-sdk/src/storage.ts) export type StorageCoverage = "complete" | "partial" | "unknown"; + StorageEvidenceReason (packages/ui-sdk/src/storage-guidance.ts) + export type StorageEvidenceReason = "no-observation" | "stale" | "future" | "incomplete" | "changed-scope" | "changed-capacity" | "gap" | "ordering" | "insufficient-count" | "insufficient-span" | "mixed-direction" | "single-burst" | "arithmetic"; StorageFileEntry (packages/ui-sdk/src/storage.ts) export interface StorageFileEntry { id: string; @@ -9350,6 +9410,89 @@ signatures (every public export and the types it is made of) largerThan?: number; cursor?: string; } + StorageGuidanceResponse (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageGuidanceResponse { + version: 1; + state: "current" | "unknown" | "unavailable"; + severity: StorageSeverity | null; + paused: string | null; + pending: boolean; + observedAt: number | null; + scopeLimit: boolean; + capacities: StoragePressureView[]; + untracked: string[]; + notifications: { + permission: "browser-managed"; + quietHours: { + start: 22; + end: 8; + unknownZone: "defer"; + }; + subscriptionsTruncated: boolean; + receiptsTruncated: boolean; + subscriptions: Array<{ + zoneKnown: boolean; + quiet: boolean | null; + }>; + receipts: Array<{ + episode: string; + state: "pending" | "sending" | "accepted" | "failed" | "unknown" | "suppressed"; + attempts: number; + nextAt: number | null; + }>; + }; + } + StorageHistoryCapacity (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageHistoryCapacity { + identity: string; + rootScope: string; + total: number | null; + used: number | null; + free: number | null; + available: number | null; + coverage: StorageCoverage; + owners: Record; + union: { + logical: number | null; + allocated: number | null; + }; + } + StorageHistoryObservation (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageHistoryObservation { + tier: StorageHistoryTier; + gap: boolean; + at: number; + startedAt: number; + source: "scheduled" | "refresh" | "operation"; + collector: string; + scope: string; + basis: "storage-v1"; + coverage: StorageCoverage; + reason: "none" | "partial" | "capacity" | "scope-limit" | "budget" | "changed"; + capacities: StorageHistoryCapacity[]; + } + StorageHistoryResponse (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageHistoryResponse { + version: 1; + state: "available" | "unavailable"; + paused: string | null; + pending: boolean; + lastObservedAt: number | null; + payloadBytes: number | null; + limits: { + sixHour: 64; + daily: 90; + weekly: 32; + capacities: 4; + payloadBytes: 262144; + }; + observations: StorageHistoryObservation[]; + } + StorageHistoryTier (packages/ui-sdk/src/storage-guidance.ts) + export type StorageHistoryTier = "six-hour" | "daily" | "weekly"; StorageMaintenanceItem (packages/ui-sdk/src/storage.ts) export interface StorageMaintenanceItem { id: string; @@ -9415,6 +9558,13 @@ signatures (every public export and the types it is made of) headroom: StorageMaintenanceReceipt["headroom"]; latest: StorageMaintenanceReceipt | null; } + StorageObservedRate (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageObservedRate { + start: number; + end: number; + samples: number; + bytesPerDay: number; + } StorageOwner (packages/ui-sdk/src/storage.ts) export type StorageOwner = "brain" | "app" | "operator"; StoragePage (packages/ui-sdk/src/storage.ts) @@ -9427,6 +9577,32 @@ signatures (every public export and the types it is made of) matched: number; totals: StorageBytes; } + StoragePressureView (packages/ui-sdk/src/storage-guidance.ts) + export interface StoragePressureView { + identity: string; + rootScope: string; + severity: StorageSeverity; + at: number; + state: "current" | "unknown"; + scopeChanged: boolean; + recoveryAt: number | null; + episode: string | null; + causes: { + capacity: boolean; + forecast: boolean; + refusals: Array<{ + target: string; + resource: "bytes" | "quota" | "inodes" | "unknown"; + at: number; + successAt: number | null; + }>; + }; + stores: Array<{ + label: string; + owner: StorageOwner; + }>; + evidence: StorageCapacityEvidence | null; + } StorageReferenceEvidence (packages/ui-sdk/src/storage.ts) export interface StorageReferenceEvidence { entryId: string; @@ -9543,6 +9719,18 @@ signatures (every public export and the types it is made of) owner: StorageOwner; reason: string | null; } + StorageScenario (packages/ui-sdk/src/storage-guidance.ts) + export interface StorageScenario { + state: "unavailable" | "finite" | "non-depleting"; + reason: StorageEvidenceReason | null; + start: number | null; + end: number | null; + samples: number; + depletionPerDay: number | null; + days: number | null; + } + StorageSeverity (packages/ui-sdk/src/storage-guidance.ts) + export type StorageSeverity = "observed" | "low" | "critical"; StorageSnapshot (packages/ui-sdk/src/storage.ts) export interface StorageSnapshot { id: string; diff --git a/docs/README.md b/docs/README.md index 8cb9af9dc7..4ca2489d65 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,6 +3,9 @@ A file-first personal knowledge base your coding agent operates. Markdown is the source of truth; the search index is disposable and rebuilt on demand. +[Storage history and pressure evidence](storage-history.md) describes bounded +operational sampling, recovery and Critical notification receipts. + ## Get started - [Quickstart](quickstart.md) — create a private brain, capture a note and find diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index d1ac334daa..2632f7f6a6 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -64,7 +64,7 @@ next to it. of this binary. The only version probe in the server is for the `brain` CLI (`Probe the selected content CLI`, `packages/ui-server/src/brain/client.ts:121-187`, called at - `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:388`). + `probeBrainCliVersion(config.brainPath`, `packages/ui-server/src/app.ts:396`). `brain doctor` runs `claude mcp list` from `PATH` (`which("claude")`, `packages/core/src/cli/commands/doctor.ts:495-499`) — the user's own Claude Code on their own machine, to check the MCP registration, not the server's @@ -212,7 +212,7 @@ unnecessary. `node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs`. The boot probe below turns either into a refusal to start, for the same reason a missing backend refuses to boot (`A missing (or unrecognized) agent backend`, - `packages/ui-server/src/app.ts:333-338`). + `packages/ui-server/src/app.ts:341-346`). - **The published range is still a caret, and that bounds what this repo can guarantee.** `@schlessera/brain-backend-claude` depends on `^0.3.241`, and a host resolves it in its own lockfile. A host can bump the SDK — and so the diff --git a/docs/decisions/http-api-boundary.md b/docs/decisions/http-api-boundary.md index ce161725fa..587739af07 100644 --- a/docs/decisions/http-api-boundary.md +++ b/docs/decisions/http-api-boundary.md @@ -46,11 +46,11 @@ explicit supported rows, not silently excluded by this distinction. The source audit on 2026-09-30 used `e0875269b21b707482545be9e94901acb897893b`. The public health mount precedes auth -(`createHealthRoutes`, `packages/ui-server/src/app.ts:733`), the API guard follows -public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:753`), and the -WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:919`). +(`createHealthRoutes`, `packages/ui-server/src/app.ts:741`), the API guard follows +public ceremonies (`authGuard`, `packages/ui-server/src/app.ts:761`), and the +WebSocket admission is separate (`"/ws"`, `packages/ui-server/src/app.ts:942`). Static serving is conditional (`if (options.staticRoot)`, -`packages/ui-server/src/app.ts:939-960`). That order establishes access; +`packages/ui-server/src/app.ts:962-983`). That order establishes access; it does not select compatibility status. A hermetic real-app launch, with temporary brain root, in-memory database, diff --git a/docs/decisions/scheduled-tasks.md b/docs/decisions/scheduled-tasks.md index 8f17fdf144..eb4ba8eab9 100644 --- a/docs/decisions/scheduled-tasks.md +++ b/docs/decisions/scheduled-tasks.md @@ -23,7 +23,7 @@ Source inspected at `b5c30c180df5584584f261b446c4d440380d2441`: (`else if (item.status === "scheduled"`, `packages/ui-server/src/inbox/actions.ts:188-189`). The runtime ticks every minute (`INBOX_TICK_MS`, `packages/ui-server/src/inbox/runtime.ts:11-13`). - Production app wiring supplies recovery and budgets, without a dispatcher - (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:405-421`). + (`// Recovery/heartbeat only.`, `packages/ui-server/src/app.ts:413-429`). A scripted nonempty ready-item test still reports no claim, attempt or Activity root. That proves disabled wiring, not safe execution of a schedule. - Headless plumbing checks backend support, a usable principal, lifetime and diff --git a/docs/decisions/session-principals.md b/docs/decisions/session-principals.md index bcae7228ba..a8b14ff393 100644 --- a/docs/decisions/session-principals.md +++ b/docs/decisions/session-principals.md @@ -96,9 +96,9 @@ Both reviewers verified every row below against the source. | Cookie verify | `auth.ts:265-296` | strict `^(\d+)\.(\d+)$`, safe integers, future/expired rejection, `cookieEpoch === sessionsEpoch(db)` | | Global epoch | `auth.ts:302-318` | `settings` row; missing = 0; non-integer **throws** | | Global revoke | `auth.ts:321-330` | `setSetting` (overflow-guarded) + `clients.closeAll(1008, …)` | -| Auth routes | `authRoutes`, `app.ts:739` | mounted **before** the guard at `authGuard(authMode`, `app.ts:753`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | -| Passkey management | `passkeyManagementRoutes`, `app.ts:763` | mounted **after** the guard ("Mount AFTER the auth guard") | -| WS guard | `"/ws"`, `app.ts:919-936` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | +| Auth routes | `authRoutes`, `app.ts:747` | mounted **before** the guard at `authGuard(authMode`, `app.ts:761`; logout guards itself (`resolveCookiePrincipal(c, auth, deps.db)`, `auth.ts:799`) | +| Passkey management | `passkeyManagementRoutes`, `app.ts:771` | mounted **after** the guard ("Mount AFTER the auth guard") | +| WS guard | `"/ws"`, `app.ts:942-959` | origin → `isWsAuthorized` (returns a boolean, `auth.ts:221-237`) → capacity → upgrade | | WS upgrade | `ws/connection.ts:262-264` | `createWsUpgrade` ignores the request context | | WS admission | `ws/connection.ts:84` | `clients.add(ws)` — the socket's identity is unknown | | Turn record | `ws/run-session.ts:134-142` | recorder built from `{turnId, sessionId, billing}`; `RunningTurn` (`ws/turns.ts:55-77`) has no actor | diff --git a/docs/http-api-coverage.md b/docs/http-api-coverage.md index 613ebb7683..24c8777243 100644 --- a/docs/http-api-coverage.md +++ b/docs/http-api-coverage.md @@ -7,7 +7,7 @@ fake backends and injected transports keep the checks keyless. Each ordinary row runs the named `mounted supported handler: METHOD PATH` test in [http-supported-mounts.test.ts](../packages/ui-server/tests/http-supported-mounts.test.ts) -(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:129-162`). +(`mounted supported handler:`, `packages/ui-server/tests/http-supported-mounts.test.ts:131-164`). That request carries a real owner cookie and checks the handler's response or side effect. Public login verification also has a successful mounted login check in A; successful passkey cryptographic verification uses the injected @@ -39,6 +39,8 @@ row's real mounting check. They do not establish mounting by themselves. | `POST /api/storage/maintenance` | `mounted supported handler: POST /api/storage/maintenance` | SM | | `GET /api/storage/maintenance/:id` | `mounted supported handler: GET /api/storage/maintenance/:id` | SM | | `DELETE /api/storage/maintenance/:id` | `mounted supported handler: DELETE /api/storage/maintenance/:id` | SM | +| `GET /api/storage/history` | `mounted supported handler: GET /api/storage/history` | SG | +| `GET /api/storage/guidance` | `mounted supported handler: GET /api/storage/guidance` | SG | | `GET /api/storage` | `mounted supported handler: GET /api/storage` | ST | | `POST /api/storage/scans` | `mounted supported handler: POST /api/storage/scans` | ST | | `DELETE /api/storage/scans/:id` | `mounted supported handler: DELETE /api/storage/scans/:id` | ST | @@ -486,3 +488,12 @@ and [real Git history preservation and a full private tmpfs](../packages/core/te Existing [inbox unwind](../packages/ui-server/tests/inbox-unwind.test.ts) and [resolution](../packages/ui-server/tests/inbox-resolve.test.ts) tests retain live settlement and unacknowledged staging protections. + +SG: [Storage guidance](../packages/ui-server/tests/storage-guidance.test.ts) +checks real mounted owner/agent/revocation boundaries, named output, no scan on +read, actual background collection and operational export/restore suppression. +[Native scan tests](../packages/ui-server/tests/storage-shared-scan.test.ts) +exercise two processes, cancellation, cache failure and parent SIGKILL. +[Delivery tests](../packages/ui-server/tests/storage-notify.test.ts) use the +existing injected transport, including concurrent native senders and SIGKILL +after dispatch entry; they never contact a push provider. diff --git a/docs/http-api.md b/docs/http-api.md index 7bb8f5eaba..eaccd85ad4 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -10,7 +10,7 @@ compatibility guarantee. Authentication requirements are a separate property. A public liveness probe can be supported; an authenticated settings route can be internal. -The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 152 unique declared +The inventory includes the additive Queue intake, poke, scheduled-task, session-draft, session-recovery interactive HTML preview, saved-audio transcription, human-started hygiene-review, governance-event, roadblock-report and retained-journal/user-revert routes mounted by `createApp`: 154 unique declared method/path pairs, plus the conditional SPA fallback. It describes the current implementation, including limitations, rather than a proposed redesign. Unknown response fields must be tolerated. There is no HTTP API revision @@ -37,6 +37,8 @@ client code has a gap. Source owners are listed after the table. | POST | `/api/storage/maintenance` | S | Admit one identified maintenance operation | Durable owner-bound identity and credential-free worker execution. | | GET | `/api/storage/maintenance/:id` | S | Read the original operation receipt | Readback never replays an effect. | | DELETE | `/api/storage/maintenance/:id` | S | Request cancellation of remaining work | Existing committed effects remain in the receipt. | +| GET | `/api/storage/history` | S | Read bounded aggregate history | Current human-owner access; no scan on read. | +| GET | `/api/storage/guidance` | S | Read pressure, scenarios and delivery evidence | Current human-owner access; no scan on read. | | GET | `/api/storage` | S | Read current bounded inventory state | Human-owner Storage inspection; no delegated or worker access. | | POST | `/api/storage/scans` | S | Refresh configured stores | Human-owner Storage inspection; no delegated or worker access. | | DELETE | `/api/storage/scans/:id` | S | Cancel an active scan | Human-owner Storage inspection; no delegated or worker access. | @@ -185,7 +187,7 @@ Authentication factories live in [auth.ts](../packages/ui-server/src/middleware/ [passkeys.ts](../packages/ui-server/src/middleware/passkeys.ts) and [principals.ts](../packages/ui-server/src/middleware/principals.ts). [app.ts](../packages/ui-server/src/app.ts) owns prefixing and middleware order. -All 152 declared endpoints are mounted regardless of backend, renderer or +All 154 declared endpoints are mounted regardless of backend, renderer or speech-provider availability: unavailable capabilities return the responses below rather than removing their handlers. Only static serving is conditional. @@ -1473,3 +1475,46 @@ retains existing receipts. This is an admission bound, not a new retention expir Invalid requests are 400, oversized bodies 413, wrong content type 415, contention 409 and unavailable evaluation/recovery/admission 503 or an explicit refused/failed receipt. The Storage pane is delivered by its complete UI issue. + +## Storage history and pressure guidance + +`GET /api/storage/history` and `GET /api/storage/guidance` require the same +current human-owner authority as inventory. They accept no body or options, +use `Cache-Control: private, no-store`, and never collect filesystem inventory. +Unauthenticated callers receive 401; delegated principals receive 403; an +unavailable operational read returns 503. Unknown evidence is a nullable or +explicit unavailable field in a successful response, never a zero. + +History returns SDK `StorageHistoryResponse`, version 1: state, pause reason, +pending status, last actual observation time, serialized payload bytes, limits +and at most 186 named observations. Each observation identifies its UTC tier, +actual start/end, source, collector version, opaque scope, basis, coverage and +gap/reason. Its at most four capacities carry nullable total/used/free/available, +three owner attribution views and a separately deduplicated union. History has +no paths, filenames, content, credentials or persistent display names. The +private compact encoding is not a wire contract. The 256 KiB ceiling applies +to that canonical persisted payload; named HTTP expansion and SQLite pages/WAL +are separate charges. Limits are 64 six-hour, 90 daily and 32 Monday-weekly slots. + +Guidance returns SDK `StorageGuidanceResponse`, version 1: current/unknown/ +unavailable state, highest retained severity, pause/pending status, last actual +observation time, scope-limit flag, up to four capacity pressure records and +untracked current inventory IDs. Each pressure record retains independent +capacity, forecast and owner-refusal causes, recovery time and opaque episode. +Current authorized inventory supplies optional store display labels. Its +nullable evidence identifies observed rate windows and short/long conditional +capacity scenarios; unavailable scenarios carry a reason, never a guessed +number. `non-depleting` does not mean unlimited capacity. Unknown state can +retain Low/Critical severity until the cause-specific recovery rules are met. +Older retained complete observations keep their own timestamps even when newer +partial evidence changes pressure. + +Notification status uses existing browser-managed permission. It includes +strict local 22–08 quiet hours, up to 100 subscription zone/quiet summaries and +200 episode receipts, with separate truncation flags. It exposes neither +endpoints nor credentials. Receipts distinguish pending, sending, accepted, +known failure, unknown and suppressed. Accepted means transport submission; +unknown is terminal for automatic retry. Three attempts per episode/subscription +are the maximum, with one/five-minute delays only after known non-delivery. +Restored episodes never regain notification authority. See +[Storage history](storage-history.md) for sampling, recovery and physical costs. diff --git a/docs/inbox-notifications.md b/docs/inbox-notifications.md index 897445e6cd..e1dcf189be 100644 --- a/docs/inbox-notifications.md +++ b/docs/inbox-notifications.md @@ -42,7 +42,7 @@ deadline opens the next window. Thread identity is kept on each constituent, not in the grouping key. Unmarked below-cutoff decisions and FYIs never join a window. Delivery runs after activity intents on the same reentrancy guard -(`deliverActions`, `packages/ui-server/src/activity/push-sender.ts:320-369`). +(`deliverActions`, `packages/ui-server/src/activity/push-sender.ts:321-370`). Each destination is evaluated on its own. `beginAttempt()` rechecks, in one transaction, that the subscription is still bound to the principal and the principal is usable, that the device reported a usable zone and is outside diff --git a/docs/integration-contract/http.md b/docs/integration-contract/http.md index 15fe646a92..682b768c09 100644 --- a/docs/integration-contract/http.md +++ b/docs/integration-contract/http.md @@ -510,3 +510,15 @@ They bind current human-owner authority, exact versioned selections and durable identities to charged retained originals. Unknown outcomes never authorize replay. Undo is separately reviewed, cannot overwrite a replacement, and expires with actual journal detail. Existing Markdown application semantics are unchanged. + +### Storage history and pressure guidance (additive, #2058) + +The two [history/guidance operations](../http-api.md#storage-history-and-pressure-guidance) +and SDK `StorageHistoryResponse`, `StorageGuidanceResponse`, named observation, +pressure, rate and scenario types are version 1 additive contracts. They retain +current human-owner authorization, nullable unknown quantities, actual endpoint +and window times, independent causes, explicit stale/scope gaps and transport +uncertainty. Public types do not expose compact persistence tuples or host paths. +The next compatible server/SDK minor includes these endpoints; clients must +retain an unavailable state when an older host does not provide them. No core +CLI, MCP, frontmatter, config or content-index contract changes. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index e72ae798be..d3ae2794bd 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -226,7 +226,7 @@ Stated before the requirements because five of them derive from it. that is the shape to copy, including its `close()` lifecycle. - R19. **The cron backstop has independent authorization before the general guard.** Mount it on the existing listener before - (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:753`). + (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:761`). Authorize a boot-minted ephemeral token, rotated each boot and stored in a 0600 runtime file, with the actual socket address as an additional check. Proxy headers cannot authorize it. The poke succeeds in every auth mode diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index 0206111fb4..4925af10e1 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -131,7 +131,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. owns manual triggers/history; it does not tick the inbox. - **Two connections:** (`export function createUiDb(`, `packages/ui-server/src/db/client.ts:25-37`) sets WAL, foreign keys and a 5-second busy timeout. Claims are immediate transactions. -- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:753`) follows public routes; +- **Auth mounting:** (`app.use("/api/*", authGuard(`, `packages/ui-server/src/app.ts:761`) follows public routes; (`export function authGuard(`, `packages/ui-server/src/middleware/auth.ts:189-249`) binds principals in each auth mode. An internal poke needs independent token authorization before this guard. - **Permission parking:** (`requestPermission: (req) => {`, `packages/ui-server/src/ws/bridge.ts:271-365`). Timeout unwind is diff --git a/docs/plans/navigation-recovery.md b/docs/plans/navigation-recovery.md index 4ff0492e26..b5bd6996f1 100644 --- a/docs/plans/navigation-recovery.md +++ b/docs/plans/navigation-recovery.md @@ -397,7 +397,7 @@ implementation. Any PR moving these lines owns their citation repairs. - Activity resolution: (`.get("/activity/runs", (c) => {`, `packages/ui-server/src/routes/activity.ts:142-275`). -- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:753`). +- API authorization: (`app.use("/api/*", authGuard(authMode, auth, db));`, `packages/ui-server/src/app.ts:761`). - Queued acceptance: (`// Queue it as the session's next turn; report queued immediately.`, `packages/ui-server/src/ws/run-session.ts:878-922`). diff --git a/docs/plans/storage-accounting.md b/docs/plans/storage-accounting.md index 6bf477cd6d..1be2a8f2b9 100644 --- a/docs/plans/storage-accounting.md +++ b/docs/plans/storage-accounting.md @@ -67,7 +67,7 @@ Source anchors, checked against the reviewed commit: The completed #94 and #398 provide existing stats/maintenance foundations. - C: snapshot fields (`snapshotOf`, `packages/core/src/lib/stats-history.ts:132-172`), thinning (`thin`, `packages/core/src/lib/stats-history.ts:232-244`). -- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:259-294`), +- D: effective path/journal resolution (`createApp`, `packages/ui-server/src/app.ts:266-302`), WAL store (`createUiDb`, `packages/ui-server/src/db/client.ts:25-37`), and private locations (`statePaths`, `packages/ui-server/src/brain/operational-state.ts:107-113`). - E: live/archive paths (`transcriptPaths`, `packages/ui-backend-claude/src/transcript-archive.ts:29-38`). @@ -87,7 +87,7 @@ Source anchors, checked against the reviewed commit: measurements do not include the separate runtime-state tmpfs. - J: private object store (`createJournalObjects`, `packages/ui-server/src/brain/journal-objects.ts:34-48`) and D's path resolution. Existing journal/revert accounting owns reservations. -- K: consistent export (`exportInboxSnapshot`, `packages/ui-server/src/inbox/snapshot.ts:252-267`) +- K: consistent export (`exportInboxSnapshot`, `packages/ui-server/src/inbox/snapshot.ts:254-269`) and [operational recovery](../inbox-recovery.md); delivered #686 and its [generated-host counterpart](https://github.com/schlessera/brain-hosting-template/issues/9). diff --git a/docs/storage-history.md b/docs/storage-history.md new file mode 100644 index 0000000000..d389f8e9df --- /dev/null +++ b/docs/storage-history.md @@ -0,0 +1,92 @@ +# Storage history and pressure evidence + +The UI server retains aggregate observations in its operational database, +separate from disposable `brain.db`. The [approved policy](decisions/storage-guidance.md) +and [named HTTP contract](http-api.md#storage-history-and-pressure-guidance) +bind collection, qualification, recovery and delivery. + +## Collection and cost + +There is no inventory scan at boot, on a history/status read, or on a worker's +hot path. A running server attempts the next six-hour UTC boundary using a +currently usable human owner. A sleeping/restarted server does not replay missed +buckets. Explicit Refresh and existing operation measurements share the same +collector. A SQLite lease coalesces concurrent app processes; it does not lock +out file writers. The observation remains a live, non-atomic view. + +All three history tiers use that one collection: 64 recent six-hour endpoints, +90 daily endpoints and 32 Monday-based weekly endpoints. Current daily/weekly +slots are provisional endpoints, not averages. Actual times, gaps, partial +coverage and changed scope remain visible. Weekly points never supply forecast +evidence. Unknown private owners, incomplete coverage or changed capacity can +prevent qualification even when direct available-byte warnings remain usable. + +One newest aggregate can wait in memory between history transactions, at most +one per minute. Intervening failures still break comparison/recovery. Pending +owner refusals are bounded separately in memory and are applied without being +erased by a newer aggregate. A crash can lose pending observations. A failed +SQLite write preserves the last valid durable payload and reports paused +history; it never deletes recovery data to gain headroom. + +The canonical history payload is at most 256 KiB, including at most 40,960 bytes +of shared descriptors and warning state. This is not a physical SQLite bound. +Database pages, indexes, WAL and SHM are measured once with the existing +operational database inventory; do not add payload bytes to that total. A native +SQLite test records actual logical/allocated DB/WAL bytes separately from the +payload. Delivery receipts and the shared scan lease are additional operational +rows, also included in database inventory. + +Ordinary collection retains its inventory in the existing bounded memory cache. +Only an overlapping caller in another coordinator requires a disposable private +file transfer. Its cache retains at most two serialized results, 48 MiB each, +with a two-minute transfer lifetime and bounded cleanup. This separate directory +appears once in App storage. Results are unavailable after expiration; a joined +request cannot trigger a duplicate collection just because transfer failed. +Reader subprocesses have deadlines and monitor the exact parent identity, so a +parent SIGKILL cannot leave an unowned reader. A replacement lease reaps exact +recorded process identities before collecting. An unkillable process leaves +collection unavailable rather than authorizing overlapping ownership. These +bounds do not promise recovery from arbitrary kernel/filesystem I/O stalls. + +## Pressure and recovery + +Available bytes trigger Low at 10% and Critical at 5%. Qualified conditional +scenarios use separate recent and daily windows; either at 30 days triggers Low, +both at seven days trigger Critical. Each carries its actual evidence window. +Neither a projected range nor an observed rate is an accuracy guarantee. + +Each cause must recover independently. Capacity recovery is strictly above +12%/7%; forecast recovery is beyond 45/ten days using the approved both/either +rule. Recovery needs two fresh observations at least 60 seconds apart. Critical +first becomes Low; clearing Low takes another pair. Failure, reversal, changed +scope or missing evidence cannot clear an unresolved cause. Measurements cannot +promise reclaimed bytes or authorize retention changes. + +Existing owning operations can report actual no-space refusal and matching +successful writes, without a probe write. Retained attachment capture and Undo +report structured filesystem/owner errors; Git packing uses its structured core +result. An undifferentiated ENOSPC remains an unknown resource, since byte and +inode exhaustion cannot be inferred from that code alone. Worker packing cannot +identify a host mount, so its unresolved target stays explicit. Quota, inode and +unknown-resource refusals remain unresolved when no matching capacity evidence +exists. This does not claim every app write already emits refusal telemetry. + +## Critical notices + +Storage uses existing opt-in, owner-bound Web Push subscriptions and authenticated +timezone reports. Every dispatch rechecks authority, current subscription keys, +fresh supporting evidence, episode and local quiet hours. Unknown timezone defers; +strict 22:00–08:00 applies even to an owner refusal. There are no daily reminders +and no collection solely to send a notice. One generic notice can cover several +eligible capacities without exposing IDs, paths, amounts or filenames. + +A durable unique episode/subscription intent precedes transport. Known rejection +permits at most three attempts, after one and five minutes. Timeout, interruption +or a process crash after dispatch leaves unknown delivery and is not retried. +Transport acceptance does not establish device display. Existing subscriptions +own destination retention; receipts retain active/latest resolved episodes and +are purged with obsolete episodes or subscription deletion. + +Operational export includes history, episodes and receipts. Restore suppresses +all old episode delivery authority, clears collection leases and requires fresh +evidence. It does not restore content files, replay a notice or perform cleanup. diff --git a/packages/ui-sdk/src/protocol.ts b/packages/ui-sdk/src/protocol.ts index 99ebff8581..194dd47ab8 100644 --- a/packages/ui-sdk/src/protocol.ts +++ b/packages/ui-sdk/src/protocol.ts @@ -3648,3 +3648,4 @@ export type { ScopeCapReceipt, ScopeDimension, TurnScopeSummary } from "./turn-j export type { StorageCoverage, StorageOwner, StorageBytes, StorageRoot, StorageCapacityView, StorageFileEntry, StorageSnapshot, StorageStatus, StoragePage, StorageFilter, StorageReferenceEvidence, StorageMaintenanceKind, StorageMaintenanceRequest, StorageMaintenanceItem, StorageMaintenanceObservation, StorageMaintenanceReceipt, StorageMaintenanceRow, StorageTreeNode, StorageTreeFilter, StorageTreePage } from "./storage.js"; export type { StorageRemovalInspection, StorageRemovalReviewRequest, StorageRemovalStartRequest, StorageRemovalUndoRequest, StorageRemovalFile, StorageRemovalReview, StorageRemovalItem, StorageRemovalReceipt, StorageRemovalCapability } from "./storage-removal.js"; +export type { StorageHistoryTier, StorageSeverity, StorageEvidenceReason, StorageObservedRate, StorageScenario, StorageCapacityEvidence, StorageHistoryCapacity, StorageHistoryObservation, StorageHistoryResponse, StoragePressureView, StorageGuidanceResponse } from "./storage-guidance.js"; diff --git a/packages/ui-sdk/src/storage-guidance.ts b/packages/ui-sdk/src/storage-guidance.ts new file mode 100644 index 0000000000..09e1ce9d03 --- /dev/null +++ b/packages/ui-sdk/src/storage-guidance.ts @@ -0,0 +1,53 @@ +import type { StorageCoverage, StorageOwner } from "./storage.js"; +/** Versioned named projection of bounded operational evidence; never raw persistence tuples. */ +export type StorageHistoryTier = "six-hour" | "daily" | "weekly"; +export type StorageSeverity = "observed" | "low" | "critical"; +export type StorageEvidenceReason = "no-observation" | "stale" | "future" | "incomplete" | "changed-scope" | "changed-capacity" | "gap" | "ordering" | "insufficient-count" | "insufficient-span" | "mixed-direction" | "single-burst" | "arithmetic"; +export interface StorageObservedRate { start: number; end: number; samples: number; bytesPerDay: number } +export interface StorageScenario { + state: "unavailable" | "finite" | "non-depleting"; reason: StorageEvidenceReason | null; + start: number | null; end: number | null; samples: number; depletionPerDay: number | null; days: number | null; +} +export interface StorageCapacityEvidence { + identity: string; rootScope: string; at: number; total: number | null; available: number | null; fresh: boolean; + short: StorageScenario; long: StorageScenario; + appLogical: StorageObservedRate | null; appAllocated: StorageObservedRate | null; availableChange: StorageObservedRate | null; +} +export interface StorageHistoryCapacity { + identity: string; rootScope: string; total: number | null; used: number | null; free: number | null; available: number | null; + coverage: StorageCoverage; + owners: Record; + union: { logical: number | null; allocated: number | null }; +} +export interface StorageHistoryObservation { + tier: StorageHistoryTier; gap: boolean; at: number; startedAt: number; source: "scheduled" | "refresh" | "operation"; + collector: string; scope: string; basis: "storage-v1"; coverage: StorageCoverage; + reason: "none" | "partial" | "capacity" | "scope-limit" | "budget" | "changed"; + capacities: StorageHistoryCapacity[]; +} +export interface StorageHistoryResponse { + version: 1; state: "available" | "unavailable"; paused: string | null; pending: boolean; + lastObservedAt: number | null; payloadBytes: number | null; + limits: { sixHour: 64; daily: 90; weekly: 32; capacities: 4; payloadBytes: 262144 }; + observations: StorageHistoryObservation[]; +} +export interface StoragePressureView { + identity: string; rootScope: string; severity: StorageSeverity; at: number; + state: "current" | "unknown"; scopeChanged: boolean; recoveryAt: number | null; + episode: string | null; + causes: { capacity: boolean; forecast: boolean; refusals: Array<{ target: string; resource: "bytes" | "quota" | "inodes" | "unknown"; at: number; successAt: number | null }> }; + /** Only current authorized inventory supplies display labels. Empty means refresh is needed. */ + stores: Array<{ label: string; owner: StorageOwner }>; + evidence: StorageCapacityEvidence | null; +} +export interface StorageGuidanceResponse { + version: 1; state: "current" | "unknown" | "unavailable"; severity: StorageSeverity | null; + paused: string | null; pending: boolean; observedAt: number | null; scopeLimit: boolean; + capacities: StoragePressureView[]; untracked: string[]; + notifications: { + permission: "browser-managed"; quietHours: { start: 22; end: 8; unknownZone: "defer" }; + subscriptionsTruncated: boolean; receiptsTruncated: boolean; + subscriptions: Array<{ zoneKnown: boolean; quiet: boolean | null }>; + receipts: Array<{ episode: string; state: "pending" | "sending" | "accepted" | "failed" | "unknown" | "suppressed"; attempts: number; nextAt: number | null }>; + }; +} diff --git a/packages/ui-server/migrations/053_storage_guidance.sql b/packages/ui-server/migrations/053_storage_guidance.sql index 504245856f..8d9831ba1b 100644 --- a/packages/ui-server/migrations/053_storage_guidance.sql +++ b/packages/ui-server/migrations/053_storage_guidance.sql @@ -12,6 +12,12 @@ CREATE TABLE storage_scan_leases ( process_start TEXT, principal_id TEXT, started_at INTEGER, + scope TEXT, + source INTEGER CHECK (source BETWEEN 0 AND 2), + workers_json TEXT NOT NULL DEFAULT '[]' CHECK (length(workers_json) <= 1024), + shared INTEGER NOT NULL DEFAULT 0 CHECK (shared IN (0, 1)), + results_json TEXT NOT NULL DEFAULT '[]' CHECK (length(results_json) <= 2048), + last_request_at INTEGER, cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK (cancel_requested IN (0, 1)), last_attempt_bucket INTEGER, result_id TEXT, diff --git a/packages/ui-server/src/activity/push-sender.ts b/packages/ui-server/src/activity/push-sender.ts index 86860ea8d3..78b9d197b2 100644 --- a/packages/ui-server/src/activity/push-sender.ts +++ b/packages/ui-server/src/activity/push-sender.ts @@ -54,7 +54,7 @@ export interface PushSender { */ deliverActions(notices: ActionNotifier, now?: number): Promise; /** One durable Critical Storage submission per episode and subscription. */ - deliverStorage(notices: StorageNotifier, now?: number): Promise; + deliverStorage(notices: StorageNotifier, now?: number, signal?: AbortSignal): Promise; } export interface CreatePushSenderOptions { @@ -287,7 +287,7 @@ export function createPushSender( return attempts; }, - async deliverStorage(notices, at) { + async deliverStorage(notices, at, signal) { let attempts = 0; // Resolve credentials first, then acquire each durable intent immediately // before its transport call. Authority/zone/time are rechecked per device. @@ -295,6 +295,7 @@ export function createPushSender( if (!subs.length) return 0; const keys = vapid(); for (const sub of subs) { + if (signal?.aborted) break; const attempt = notices.begin(sub.endpoint, at ?? Date.now()); if (!attempt) continue; let outcome: StorageNoticeOutcome; diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index b1f26a02e2..e20adede43 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -1,3 +1,10 @@ +import { createStorageHistory } from "./storage/history-store.js"; +import { aggregateCollection } from "./storage/history-model.js"; +import { createSharedStorageScan } from "./storage/shared-scan.js"; +import { createStorageNotifier } from "./storage/notify.js"; +import { createStorageGuidance } from "./storage/guidance.js"; +import { createStorageSampling } from "./storage/sampling.js"; +import storagePackage from "../package.json"; import { createStorageRemoval } from "./storage/removal.js"; import { createStorageRemovalRoutes } from "./routes/storage-removal.js"; import { createStorageMaintenance } from "./storage/maintenance.js"; @@ -285,9 +292,10 @@ export async function createApp(options: CreateAppOptions = {}): Promise host.prepareHistory(id, messages) })); app.route("/api", createJournalRoutes({ journal })); - const storageStores: StorageStore[] = [{ path: journalRoot, label: "Retained file history", category: "journal", owner: "app", recursive: true }]; + const storageHistory = createStorageHistory(db, config.brainPath); + bootCleanup.push(() => storageHistory.close()); + const storageAuthorized = (id: string) => { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }; + const storageShared = createSharedStorageScan(db, config.brainPath, storageCacheRoot, storageAuthorized); + const storageNotices = createStorageNotifier(db, storageHistory); + const storageStores: StorageStore[] = [ + { path: journalRoot, label: "Retained file history", category: "journal", owner: "app", recursive: true }, + { path: storageCacheRoot, label: "Storage shared scan cache", category: "temporary", owner: "app", recursive: true, optional: true }, + ]; for (const [path, label] of [[config.dbPath, "Operational database"], [nativeLeasePath, "Native lease database"]] as const) { if (!path || path === ":memory:") continue; for (const suffix of ["", "-wal", "-shm", "-journal"]) storageStores.push({ path: resolve(path + suffix), label: label + suffix, category: "database", owner: "app", recursive: false, optional: suffix !== "" }); } for (const [index, path] of (config.storage?.backupRoots ?? []).entries()) storageStores.push({ path, label: `Backup store ${index + 1}`, category: "backup", owner: "operator", recursive: true }); - const storage = createStorageCoordinator({ brainPath: config.brainPath, stores: storageStores, gitRoots: [...config.storage?.gitRoots ?? []], + const storage = createStorageCoordinator({ salt: storageHistory.salt, shared: storageShared, + onCollection(value, source) { storageHistory.offer(aggregateCollection(value, source, storagePackage.version)); }, + onFailure(at) { storageHistory.failure(at); }, + onWrite(report) { storageHistory.ownerWrite(report); }, + brainPath: config.brainPath, stores: storageStores, gitRoots: [...config.storage?.gitRoots ?? []], authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, unavailable: [ { label: "Native checkpoints and output", category: "native", owner: "app", reason: "shared-native-allocation-unavailable" }, @@ -834,14 +854,17 @@ export async function createApp(options: CreateAppOptions = {}): Promise storage.close()); - app.route("/api", createStorageRoutes(storage)); + const storageGuidance = createStorageGuidance(db, storageHistory, storage, storageNotices); + const storageSampling = createStorageSampling(db, storage, storageHistory, storageShared, storageNotices, activity.pushSender); + bootCleanup.push(() => storageSampling.close()); + app.route("/api", createStorageRoutes(storage, storageGuidance)); const storageMaintenance = createStorageMaintenance(db, { brainPath: config.brainPath, storage, brain, scratch: scratchPrune, inbox, boundary, authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, }); bootCleanup.push(() => storageMaintenance.close()); app.route("/api", createStorageMaintenanceRoutes(storageMaintenance)); - const storageRemoval = createStorageRemoval(db, { brainPath: config.brainPath, boundary, journal, gate: hostExecution.gate()!, lock: host.workerLock, storage, + const storageRemoval = createStorageRemoval(db, { brainPath: config.brainPath, retainedRoot: journalRoot, boundary, journal, gate: hostExecution.gate()!, lock: host.workerLock, storage, protectedPaths: [...storageStores.map(store => store.path), ...config.storage?.gitRoots ?? []].filter(path => path === config.brainPath || path.startsWith(config.brainPath + "/")).map(path => path === config.brainPath ? "." : path.slice(config.brainPath.length + 1)), authorized(id) { const principal = resolvePrincipal(db, id); return !!principal && ["owner", "ambient"].includes(principal.kind) && isUsablePrincipal(principal, Date.now()); }, }); @@ -976,6 +999,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise void | Promise) => { try { await close(); } catch (error) { failures.push(error); } }; + await attempt(() => storageSampling.close()); await attempt(() => storageRemoval.close()); await attempt(() => storageMaintenance.close()); await attempt(() => inbox.close()); @@ -984,6 +1008,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise subscription.close()); await attempt(() => intake.close()); await attempt(() => storage.close()); + await attempt(() => { storageHistory.flush(); storageHistory.close(); }); await attempt(() => scratchPrune.close()); await attempt(() => scratchPins.close()); await attempt(() => reverts.close()); @@ -991,7 +1016,7 @@ export async function createApp(options: CreateAppOptions = {}): Promise { await host.governedTurns?.close(); }); await attempt(() => journal.close()); await attempt(() => nativeLeases.close()); - if (ephemeralJournal) await attempt(() => rmSync(journalRoot, { recursive: true, force: true })); + if (ephemeralJournal) await attempt(() => { rmSync(journalRoot, { recursive: true, force: true }); rmSync(storageCacheRoot, { recursive: true, force: true }); }); await attempt(() => activity.close()); await attempt(() => hostExecution.release()); await attempt(() => db.close()); diff --git a/packages/ui-server/src/inbox/snapshot.ts b/packages/ui-server/src/inbox/snapshot.ts index fc2e98ec95..601d78c5eb 100644 --- a/packages/ui-server/src/inbox/snapshot.ts +++ b/packages/ui-server/src/inbox/snapshot.ts @@ -10,6 +10,7 @@ import { COMPLETE_SHARE_INPUT_SCHEMA } from "@schlessera/brain-ui-sdk/tool-contr import { SHARE_STAGING_DIR } from "@schlessera/brain-ui-sdk/protocol"; import type { InboxQueueItem } from "@schlessera/brain-ui-sdk/protocol"; import { pauseRestoredSchedules, validateScheduleRelations } from "../schedules/recovery.js"; +import { invalidateRestoredStorage, parseStoredHistory } from "../storage/history-store.js"; import { clientInboxResolveSchema, hygieneEffectSchema, hygieneReviewStateSchema, inboxItemSchema, inboxOptionSchema, v1ResolutionEffectSchema } from "@schlessera/brain-ui-sdk/schemas"; import { createUiDb } from "../db/client.js"; import { createInboxStore } from "./store.js"; @@ -66,6 +67,7 @@ function validateDatabase(db: Database): void { if (JSON.stringify(db.query("PRAGMA integrity_check").all()) !== '[{"integrity_check":"ok"}]' || db.query("PRAGMA foreign_key_check").all().length) throw new Error("inbox_snapshot_relations"); + for (const row of db.query("SELECT payload_json FROM storage_history").all() as Array<{ payload_json: string }>) parseStoredHistory(row.payload_json); const store = createInboxStore(db), state = store.exportState(); store.snapshot(); const items = new Map(state.inbox_items.map(row => [row.id, inboxItemSchema.parse(JSON.parse(row.data_json as string))])); @@ -374,6 +376,7 @@ export async function restoreInboxSnapshot(input: unknown, dbPath: string, brain const claims = db.query("SELECT id FROM inbox_items WHERE queue = 'queue' AND status = 'claimed' AND deleted_at IS NULL ORDER BY id").all() as { id: string }[]; for (const row of claims) failInboxWork(db, row.id, (store.getItem(row.id) as InboxQueueItem).version, at); pauseRestoredSchedules(db, at); + invalidateRestoredStorage(db, at); db.query("UPDATE inbox_recovery_state SET status = 'ready', restored_at = ? WHERE id = 1").run(at); return claims.length; }).immediate(); diff --git a/packages/ui-server/src/routes/storage.ts b/packages/ui-server/src/routes/storage.ts index 85d22fc980..1641e97143 100644 --- a/packages/ui-server/src/routes/storage.ts +++ b/packages/ui-server/src/routes/storage.ts @@ -1,10 +1,11 @@ +import type { StorageGuidance } from "../storage/guidance.js"; import { Hono } from "hono"; import { requireJson } from "../middleware/origin.js"; import type { AppEnv } from "../app-env.js"; import type { StorageFilter, StorageTreeFilter } from "@schlessera/brain-ui-sdk/protocol"; import { StorageUnavailable, type StorageCoordinator } from "../storage/coordinator.js"; /** Ordinary content authorization does not grant access to deployment inventory. */ -export function createStorageRoutes(storage: StorageCoordinator): Hono { +export function createStorageRoutes(storage: StorageCoordinator, guidance?: StorageGuidance): Hono { const routes = new Hono(); routes.onError((error, c) => c.json({ error: error instanceof StorageUnavailable ? error.message : "Storage temporarily unavailable." }, error instanceof StorageUnavailable ? error.status : 503)); routes.use("/storage*", async (c, next) => { @@ -15,6 +16,10 @@ export function createStorageRoutes(storage: StorageCoordinator): Hono { try { storage.status(principal.id); await next(); } catch (error) { return c.json({ error: error instanceof StorageUnavailable ? error.message : "Storage temporarily unavailable." }, error instanceof StorageUnavailable ? error.status : 503); } }); + if (guidance) { + routes.get("/storage/history", c => c.json(guidance.history(c.get("principal")!.id))); + routes.get("/storage/guidance", c => c.json(guidance.guidance(c.get("principal")!.id))); + } routes.get("/storage", c => c.json(storage.status(c.get("principal")!.id))); routes.post("/storage/scans", requireJson(), async c => { let body: unknown; try { body = await c.req.json(); } catch { return c.json({ error: "Expected an empty JSON object." }, 400); } diff --git a/packages/ui-server/src/storage/collect.ts b/packages/ui-server/src/storage/collect.ts index b7c15e406f..1875668509 100644 --- a/packages/ui-server/src/storage/collect.ts +++ b/packages/ui-server/src/storage/collect.ts @@ -3,7 +3,7 @@ import { basename, dirname, resolve } from "node:path"; import { z } from "zod"; import type { StorageFileEntry, StorageRoot, StorageSnapshot, StorageOwner, StorageBytes } from "@schlessera/brain-ui-sdk/protocol"; import { SHARE_STAGING_DIR } from "@schlessera/brain-ui-sdk/protocol"; -import { serverMetadata, coreMetadata } from "./process.js"; +import { serverMetadata, coreMetadata, type StorageWorkerLease } from "./process.js"; import type { MetadataEntry, MetadataRoot } from "./metadata.js"; export const STORAGE_SERVER_LIMITS = Object.freeze({ entries: 100_000, page: 200, roots: 32, milliseconds: 35_000, cacheBytes: 96 * 1024 * 1024 }); const bytes = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).nullable(); @@ -19,6 +19,7 @@ export interface StorageCollectOptions { /** Unknown private owner stores are explicit rows, never scans of host /tmp or broker state. */ unavailable?: Array<{ label: string; category: StorageRoot["category"]; owner: StorageOwner; reason: string }>; salt: string; + workerLease?: StorageWorkerLease; } const validPath = (path: string) => path === "" || (!path.startsWith("/") && path.split("/").every(piece => piece !== "." && piece !== ".." && piece !== "") && !path.includes("\0")); export function storageTotals(entries: readonly Pick[], complete = true): StorageBytes { @@ -49,7 +50,7 @@ export async function collectStorage(options: StorageCollectOptions, id: string, const rawEntries: MetadataEntry[] = []; const coreRootMappings = new Map(); try { - const raw = coreInventory.parse(await coreMetadata(options.brainPath, options.gitRoots ?? [], signal)); + const raw = coreInventory.parse(await coreMetadata(options.brainPath, options.gitRoots ?? [], signal, options.workerLease)); if (raw.status !== "complete") fail(brain.id, "core-inventory-partial"); for (const value of raw.errors) fail(brain.id, value.reason); omittedErrors += raw.omittedErrors; @@ -93,7 +94,7 @@ export async function collectStorage(options: StorageCollectOptions, id: string, const capacities: StorageSnapshot["capacities"] = []; const capacityByRoot = new Map(); try { - const measured = await serverMetadata([...appSources, ...probes], signal, Math.max(1, STORAGE_SERVER_LIMITS.entries - rawEntries.length)); + const measured = await serverMetadata([...appSources, ...probes], signal, Math.max(1, STORAGE_SERVER_LIMITS.entries - rawEntries.length), options.workerLease); for (const error of measured.errors) fail(error.rootId, error.reason); omittedErrors += measured.omittedErrors; if (measured.omittedErrors) for (const source of sources.values()) fail(source.root.id, "metadata-errors-omitted"); diff --git a/packages/ui-server/src/storage/coordinator.ts b/packages/ui-server/src/storage/coordinator.ts index 923c44af91..734cb35d6c 100644 --- a/packages/ui-server/src/storage/coordinator.ts +++ b/packages/ui-server/src/storage/coordinator.ts @@ -1,3 +1,6 @@ +import { hashScope, type Source } from "./history-model.js"; +import type { SharedStorageScan, SharedScan } from "./shared-scan.js"; +import type { StorageWriteReport } from "./history-store.js"; import { projectStorageTree, sortStorageRows } from "./tree.js"; import { storageReferences } from "./process.js"; import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; @@ -8,12 +11,17 @@ export class StorageUnavailable extends Error { } export function createStorageCoordinator(options: Omit & { authorized(principalId: string): boolean; + salt?: string; + shared?: SharedStorageScan; + onCollection?(collection: StorageCollection, source: Source): void; + onFailure?(at: number): void; + onWrite?(report: StorageWriteReport): void; /** Host/backend owners resolve exact roots; this callback never receives request data. */ storesNow?(): { stores: StorageCollectOptions["stores"]; unavailable: NonNullable }; }) { - const salt = randomBytes(32).toString("hex"); + const salt = options.salt ?? randomBytes(32).toString("hex"); let current: StorageCollection | null = null, previous: StorageCollection | null = null; - let active: { id: string; principal: string; controller: AbortController; promise: Promise } | null = null; + let active: { id: string; principal: string; controller: AbortController; promise: Promise; lease?: SharedScan } | null = null; let lastAttempt = 0, closed = false; let referenceJob: { controller: AbortController; promise: Promise } | null = null; let lastFailure: { scanId: string; at: number; reason: string } | null = null; @@ -32,28 +40,72 @@ export function createStorageCoordinator(options: Omit createHmac("sha256", salt).update(value).digest("hex").slice(0, 32); + const root = key(`root:${rootIdentity}`), latest = [current, previous].filter(value => value !== null).sort((a, b) => b.summary.observedAt - a.summary.observedAt)[0]; + const capacity = latest?.summary.capacities.find(value => value.rootIds.includes(root)); + const report: StorageWriteReport = { identity: capacity?.id ?? key(`unresolved:${rootIdentity}`), rootScope: hashScope(capacity ? [...capacity.rootIds].sort() : [root]), target: key(`write:${rootIdentity}:${target}`), resource, outcome, at: Date.now() }; + try { options.onWrite?.(report); } catch { /* Telemetry cannot alter the owning operation's result. */ } + }, + start(principal: string, fresh = false, source: Source = 1): StorageStatus { authorized(principal); if (closed) throw new StorageUnavailable(503, "Storage is shutting down."); if (active) return status(principal); if (!fresh && Date.now() - lastAttempt < 5_000) throw new StorageUnavailable(429, "Wait before refreshing Storage again."); + const dynamic = options.storesNow?.() ?? { stores: [], unavailable: [] }; + const scope = hashScope({ brain: options.brainPath, stores: [...options.stores, ...dynamic.stores], git: options.gitRoots ?? [], unavailable: [...options.unavailable ?? [], ...dynamic.unavailable] }); + let lease: SharedScan | undefined; + const requestId = randomUUID(); + if (options.shared) { + try { + const acquired = options.shared.acquire(principal, requestId, scope, source, fresh); + if (!acquired) return status(principal); + lease = acquired; + } catch (error) { + authorized(principal); + const message = error instanceof Error ? error.message : "storage-lease-unavailable"; + throw new StorageUnavailable(message === "storage-scan-cooldown" ? 429 : message === "storage-scan-other-scope-busy" ? 409 : 503, message); + } + } lastAttempt = Date.now(); - const id = randomUUID(), controller = new AbortController(); - const job = { id, principal, controller, promise: Promise.resolve() }; + const id = lease?.id ?? requestId, controller = new AbortController(); + const job = { id, principal, controller, promise: Promise.resolve(), lease }; active = job; const timeout = setTimeout(() => controller.abort(), STORAGE_SERVER_LIMITS.milliseconds); - const revocation = setInterval(() => { if (!options.authorized(principal)) controller.abort(); }, 250); + const revocation = setInterval(() => { + if (!options.authorized(principal)) controller.abort(); + if (lease?.owner) { try { options.shared!.check(lease); } catch { controller.abort(); } } + }, 250); job.promise = (async () => { try { - const dynamic = options.storesNow?.() ?? { stores: [], unavailable: [] }; controller.signal.throwIfAborted(); - const result = await collectStorage({ ...options, stores: [...options.stores, ...dynamic.stores], unavailable: [...options.unavailable ?? [], ...dynamic.unavailable], salt }, id, controller.signal); + if (lease?.owner) await options.shared!.reap(lease); + const result = lease && !lease.owner ? await options.shared!.wait(lease, controller.signal) + : await collectStorage({ ...options, stores: [...options.stores, ...dynamic.stores], unavailable: [...options.unavailable ?? [], ...dynamic.unavailable], salt, + ...(lease ? { workerLease: { register: (pid: number) => options.shared!.register(lease!, pid), unregister: (pid: number) => options.shared!.unregister(lease!, pid) } } : {}) }, id, controller.signal); authorized(principal); controller.signal.throwIfAborted(); + if (lease?.owner) { + let cached = await options.shared!.publish(lease, result); + authorized(principal); controller.signal.throwIfAborted(); + if (!options.shared!.finish(lease, cached, false, result.summary.observedAt)) { + cached = await options.shared!.publish(lease, result); + authorized(principal); controller.signal.throwIfAborted(); + if (!options.shared!.finish(lease, cached, false, result.summary.observedAt)) throw Error("storage-scan-authority-lost"); + } + } + if (!lease || lease.owner) { try { options.onCollection?.(result, source); } catch { /* History failure cannot remove an inventory result. */ } } if (result.summary.coverage === "complete" || !current || current.summary.coverage !== "complete") { previous = current; current = result; } else previous = result; // A partial refresh never erases the last complete observation. if (current && previous && current.bytes + previous.bytes > STORAGE_SERVER_LIMITS.cacheBytes) previous = null; lastFailure = null; - } catch { lastFailure = { scanId: id, at: Date.now(), reason: controller.signal.aborted || !options.authorized(principal) ? "cancelled-or-revoked" : "collection-unavailable" }; } + } catch { + lastFailure = { scanId: id, at: Date.now(), reason: controller.signal.aborted || !options.authorized(principal) ? "cancelled-or-revoked" : "collection-unavailable" }; + if (!lease || lease.owner) { + try { options.onFailure?.(lastFailure.at); } catch { /* Last valid history stays intact. */ } + if (lease) { try { options.shared!.finish(lease, false, true, lastFailure.at); } catch { /* Preserve unresolved lease authority for recovery. */ } } + } + } finally { clearTimeout(timeout); clearInterval(revocation); if (active === job) active = null; } })(); return status(principal); @@ -61,7 +113,9 @@ export function createStorageCoordinator(options: Omit { signal.throwIfAborted(); authorized(principal); - while (active) { await active.promise; signal.throwIfAborted(); } - const id = api.start(principal, true).activeScanId; - const abort = () => { if (active?.id === id) active.controller.abort(); }; + const joined = active !== null; + const id = active?.id ?? api.start(principal, true, 2).activeScanId; + const job = active; + if (!job || !id) return null; + let rejectAbort: (reason: unknown) => void = () => {}; + const cancelled = new Promise((_resolve, reject) => { rejectAbort = reject; }); + const abort = () => { if (!joined && active === job) job.controller.abort(); rejectAbort(signal.reason ?? Error("Storage observation cancelled.")); }; signal.addEventListener("abort", abort, { once: true }); if (signal.aborted) abort(); - try { await api.idle(); signal.throwIfAborted(); authorized(principal); return [current, previous].find(value => value?.summary.id === id)?.summary ?? null; } + try { await Promise.race([job.promise, cancelled]); signal.throwIfAborted(); authorized(principal); return [current, previous].find(value => value?.summary.id === id)?.summary ?? null; } finally { signal.removeEventListener("abort", abort); } }, async idle() { await active?.promise; }, diff --git a/packages/ui-server/src/storage/guidance.ts b/packages/ui-server/src/storage/guidance.ts new file mode 100644 index 0000000000..422ccf4760 --- /dev/null +++ b/packages/ui-server/src/storage/guidance.ts @@ -0,0 +1,52 @@ +import type { Database } from "bun:sqlite"; +import type { StorageCoverage, StorageGuidanceResponse, StorageHistoryResponse } from "@schlessera/brain-ui-sdk/protocol"; +import type { StorageHistory } from "./history-store.js"; +import { capacityEvidence } from "./history-evidence.js"; +import { StorageUnavailable, type StorageCoordinator } from "./coordinator.js"; +import { localHour, noticeTimeZone } from "../inbox/notify.js"; +import type { StorageNotifier } from "./notify.js"; +const coverages: StorageCoverage[] = ["complete", "partial", "unknown"]; +/** Read-only, owner-scoped views. Opening either endpoint never starts a scan. */ +export function createStorageGuidance(db: Database, history: StorageHistory, storage: StorageCoordinator, notices: StorageNotifier) { + function state(principal: string) { + const inventory = storage.status(principal), status = history.status(); + return { inventory, ...status }; + } + return { + history(principal: string): StorageHistoryResponse { + const status = state(principal), payload = status.history; + return { version: 1, state: payload ? "available" : "unavailable", paused: status.paused, pending: status.pending, + lastObservedAt: payload?.metadata.state.lastObservedAt || null, payloadBytes: status.payloadBytes, + limits: { sixHour: 64, daily: 90, weekly: 32, capacities: 4, payloadBytes: 262144 }, + observations: payload?.samples.map(row => ({ tier: (["six-hour", "daily", "weekly"] as const)[row[1]], gap: !!row[2], at: row[3], startedAt: row[4], + source: (["scheduled", "refresh", "operation"] as const)[row[5]], collector: payload.metadata.samples[row[6]]![0], scope: payload.metadata.samples[row[6]]![1], basis: "storage-v1", coverage: coverages[row[7]]!, + reason: (["none", "partial", "capacity", "scope-limit", "budget", "changed"] as const)[row[8]], + capacities: row[9].map(value => ({ identity: payload.metadata.capacities[value[0]]![0], rootScope: payload.metadata.capacities[value[0]]![1], total: value[1], used: value[2], free: value[3], available: value[4], coverage: coverages[value[5]]!, + owners: { brain: { logical: value[6], allocated: value[7] }, app: { logical: value[8], allocated: value[9] }, operator: { logical: value[10], allocated: value[11] } }, union: { logical: value[12], allocated: value[13] } })) })) ?? [] }; + }, + guidance(principal: string, now = Date.now()): StorageGuidanceResponse { + const status = state(principal), payload = status.history, warning = payload?.metadata.state; + const snapshot = [status.inventory.snapshot, status.inventory.previous].filter(value => value !== null).sort((a, b) => b.observedAt - a.observedAt)[0]; + const capacities = warning?.pressures.map(pressure => { + const evidence = capacityEvidence(payload!, pressure.identity, now); + const current = !status.paused && !status.pending && !pressure.scopeChanged && !!evidence?.fresh && evidence.at >= pressure.at; + const roots = snapshot?.capacities.find(value => value.id === pressure.identity)?.rootIds ?? []; + return { identity: pressure.identity, rootScope: pressure.rootScope, severity: pressure.severity, at: pressure.at, + state: current ? "current" as const : "unknown" as const, scopeChanged: pressure.scopeChanged, recoveryAt: pressure.recoveryAt, episode: pressure.episode, + causes: { capacity: pressure.capacity, forecast: pressure.forecast, refusals: pressure.refusals }, + stores: snapshot?.roots.filter(root => roots.includes(root.id)).map(root => ({ label: root.label, owner: root.owner })) ?? [], evidence }; + }) ?? []; + const subscriptions = db.query("SELECT time_zone FROM push_subscriptions WHERE principal_id = ? ORDER BY endpoint LIMIT 101").all(principal) as Array<{ time_zone: string | null }>; + const severity = capacities.some(value => value.severity === "critical") ? "critical" : capacities.some(value => value.severity === "low") ? "low" : capacities.length ? "observed" : null; + const receipts = notices.receipts(principal, now); + const result: StorageGuidanceResponse = { version: 1, state: !payload ? "unavailable" : capacities.length && capacities.every(value => value.state === "current") ? "current" : "unknown", + severity, paused: status.paused, pending: status.pending, observedAt: warning?.lastObservedAt || null, scopeLimit: warning?.scopeLimit ?? false, capacities, + untracked: snapshot?.capacities.filter(value => !capacities.some(row => row.identity === value.id)).map(value => value.id) ?? [], + notifications: { permission: "browser-managed", quietHours: { start: 22, end: 8, unknownZone: "defer" }, subscriptionsTruncated: subscriptions.length > 100, receiptsTruncated: receipts.length > 200, subscriptions: subscriptions.slice(0, 100).map(value => { const zone = noticeTimeZone(value.time_zone); const hour = zone ? localHour(zone, now) : null; return { zoneKnown: !!zone, quiet: hour === null ? null : hour >= 22 || hour < 8 }; }), receipts: receipts.slice(0, 200) } }; + // Recheck the same authority immediately before publishing operational data. + try { storage.status(principal); } catch { throw new StorageUnavailable(403, "Storage requires current human-owner access."); } + return result; + }, + }; +} +export type StorageGuidance = ReturnType; diff --git a/packages/ui-server/src/storage/history-model.ts b/packages/ui-server/src/storage/history-model.ts index 7c0b6ac151..74b3c9dfc6 100644 --- a/packages/ui-server/src/storage/history-model.ts +++ b/packages/ui-server/src/storage/history-model.ts @@ -101,7 +101,7 @@ export function retainAggregate(previous: HistoryPayload, input: Aggregate else if (old < current && bucket(0, row[3]) < bucket(0, bucketStart(row[1], old + 1) - 1)) row[2] = 1; } const lastRecent = samples.filter(row => row[1] === 0).sort((a, b) => b[3] - a[3])[0]; - const changed = lastRecent && (JSON.stringify(metadata.samples[lastRecent[6]]) !== JSON.stringify(input.descriptor) + const changed = lastRecent && (JSON.stringify(metadata.samples[lastRecent[6]]?.slice(1)) !== JSON.stringify(input.descriptor.slice(1)) || JSON.stringify(lastRecent[9].map(row => metadata.capacities[row[0]])) !== JSON.stringify(input.capacities.map(row => row.descriptor))); const missed = !!lastRecent && bucket(0, input.at) > bucket(0, lastRecent[3]) + 1; const incomplete = (input.coverage !== 0 && input.reason !== 3) || input.capacities.some(row => row.values[4] !== 0); diff --git a/packages/ui-server/src/storage/history-store.ts b/packages/ui-server/src/storage/history-store.ts index a0c02a0930..2dc21253a3 100644 --- a/packages/ui-server/src/storage/history-store.ts +++ b/packages/ui-server/src/storage/history-store.ts @@ -3,7 +3,7 @@ import type { Database } from "bun:sqlite"; import { randomBytes } from "node:crypto"; import { realpathSync } from "node:fs"; import { capacityEvidence } from "./history-evidence.js"; -import { newPressure, observePressure, type Pressure } from "./pressure.js"; +import { newPressure, observePressure, ownerWrite, type Pressure, type Refusal } from "./pressure.js"; import { HISTORY_LIMITS, bucket, hashScope, jsonBytes, retainAggregate, validateHistory, type Aggregate, type HistoryPayload } from "./history-model.js"; export interface HistoryState { version: 1; salt: string; @@ -16,6 +16,7 @@ export interface HistoryState { } export type StoredHistory = HistoryPayload; type Pending = { kind: "aggregate"; value: Aggregate } | { kind: "failure"; at: number }; +export interface StorageWriteReport { identity: string; rootScope: string; target: string; resource: Refusal["resource"]; at: number; outcome: "no-space" | "success" } export const storageBrainKey = (brainPath: string) => hashScope(realpathSync(brainPath)); export function emptyHistory(): StoredHistory { return { metadata: { samples: {}, capacities: {}, state: { version: 1, salt: randomBytes(32).toString("hex"), lastPersistedAt: 0, lastObservedAt: 0, lastAttemptBucket: null, paused: null, pressures: [], suppressedEpisodes: [], scopeLimit: false } }, samples: [] }; @@ -48,10 +49,13 @@ export function readStoredHistory(db: Database, brain: string): StoredHistory | } export function createStorageHistory(db: Database, brainPath: string, options: { now?: () => number } = {}) { const brain = storageBrainKey(brainPath), now = options.now ?? Date.now; + const initial = emptyHistory(); let pending: Pending | null = null, unavailable: string | null = null, closed = false; + let pendingWrites: StorageWriteReport[] = [], interrupted = false; // Constructing the store never collects filesystem inventory. Failure here is // a capability result; it cannot prevent existing recovery operations booting. - try { db.query("INSERT OR IGNORE INTO storage_history (brain, payload_json) VALUES (?, ?)").run(brain, JSON.stringify(emptyHistory())); } + const initialize = () => db.query("INSERT OR IGNORE INTO storage_history (brain, payload_json) VALUES (?, ?)").run(brain, JSON.stringify(initial)); + try { initialize(); } catch { unavailable = "history-write-unavailable"; } const read = () => { try { return readStoredHistory(db, brain); } @@ -63,6 +67,7 @@ export function createStorageHistory(db: Database, brainPath: string, options: { const stored = read(); if (stored && at < stored.metadata.state.lastObservedAt) { unavailable = "history-clock-reversal"; + interrupted = true; // Do not overwrite a newer pending observation after a clock reversal. return; } @@ -71,21 +76,26 @@ export function createStorageHistory(db: Database, brainPath: string, options: { flush(); } function flush(): boolean { - if (closed || !pending) return false; - const candidate = pending, at = candidate.kind === "aggregate" ? candidate.value.at : candidate.at, time = now(); + if (closed || !pending && !pendingWrites.length) return false; + const candidate = pending, writes = pendingWrites, time = now(); + const at = Math.max(candidate ? candidate.kind === "aggregate" ? candidate.value.at : candidate.at : 0, ...writes.map(row => row.at)); try { const changed = db.transaction(() => { + initialize(); const previous = readStoredHistory(db, brain); if (!previous) throw Error("history-unavailable"); const state = previous.metadata.state; if (time < state.lastPersistedAt || at < state.lastObservedAt || at > time) throw Error("history-clock-reversal"); if (time - state.lastPersistedAt < HISTORY_LIMITS.writeInterval) return false; let next: StoredHistory; - if (candidate.kind === "aggregate") { + if (interrupted) state.pressures = state.pressures.map(value => ({ ...value, recoveryAt: null, supportAt: null })); + if (candidate?.kind === "aggregate") { next = retainAggregate(previous, candidate.value); + if (interrupted || state.paused) for (const sample of next.samples) if (sample[3] === candidate.value.at) sample[2] = 1; // A partial refresh may preserve the complete stored endpoint. Evaluate // the actual new capacity observation, never give the old total a new time. const evaluation = retainAggregate({ ...structuredClone(next), samples: next.samples.filter(row => row[1] !== 0 || bucket(0, row[3]) !== bucket(0, at)) }, candidate.value); + if (interrupted || state.paused) for (const sample of evaluation.samples) if (sample[3] === candidate.value.at) sample[2] = 1; const incoming = candidate.value.capacities.map(row => row.descriptor); const pressures = state.pressures.filter(value => value.severity !== "observed" || incoming.some(([identity]) => identity === value.identity)) .map(value => value.severity === "observed" ? newPressure(value.identity, incoming.find(([identity]) => identity === value.identity)![1]) : value); @@ -93,34 +103,66 @@ export function createStorageHistory(db: Database, brainPath: string, options: { next.metadata.state.pressures = pressures.map(value => observePressure(value, capacityEvidence(evaluation, value.identity, time), time)); next.metadata.state.scopeLimit = candidate.value.reason === 3 || incoming.some(([identity]) => !pressures.some(value => value.identity === identity)); next.metadata.state.paused = null; - } else { + } else if (candidate) { next = structuredClone(previous); for (const row of next.samples) if (bucket(row[1], row[3]) === bucket(row[1], at)) row[2] = 1; next.metadata.state.pressures = state.pressures.map(value => observePressure(value, null, time)); next.metadata.state.paused = "collection-unavailable"; + } else next = structuredClone(previous); + for (const report of writes) { + let pressure = next.metadata.state.pressures.find(value => value.identity === report.identity); + if (!pressure && report.outcome === "success") continue; + if (!pressure) { + if (next.metadata.state.pressures.length >= 4) throw Error("history-pressure-capacity-budget"); + pressure = newPressure(report.identity, report.rootScope); next.metadata.state.pressures.push(pressure); + } + const index = next.metadata.state.pressures.indexOf(pressure); + // Buffered reports were checked when received. A newer capacity sample + // from this or another process cannot erase an intervening refusal. + if (pressure.refusals.some(value => value.target === report.target && value.resource === report.resource && value.at > report.at)) continue; + const changed = ownerWrite({ ...pressure, at: Math.min(pressure.at, report.at) }, report, time); + changed.at = Math.max(changed.at, pressure.at); + next.metadata.state.pressures[index] = changed; } next.metadata.state.lastPersistedAt = time; - next.metadata.state.lastObservedAt = at; - next.metadata.state.lastAttemptBucket = bucket(0, at); + if (candidate) { + const observedAt = candidate.kind === "aggregate" ? candidate.value.at : candidate.at; + next.metadata.state.lastObservedAt = observedAt; + next.metadata.state.lastAttemptBucket = bucket(0, observedAt); + } const episodes = new Set(next.metadata.state.pressures.flatMap(value => [value.episode, value.resolvedEpisode].filter((id): id is string => id !== null))); next.metadata.state.suppressedEpisodes = next.metadata.state.suppressedEpisodes.filter(value => episodes.has(value)); - validateHistory(next); + validateHistory(next); stateSchema.parse(next.metadata.state); db.query("UPDATE storage_history SET payload_json = ?, revision = revision + 1 WHERE brain = ?").run(JSON.stringify(next), brain); return true; }).immediate(); - if (changed) { if (pending === candidate) pending = null; unavailable = null; } + if (changed) { if (pending === candidate) pending = null; if (pendingWrites === writes) pendingWrites = []; interrupted = false; unavailable = null; } return changed; } catch (error) { + interrupted = true; unavailable = error instanceof Error && error.message.startsWith("history-") ? error.message : "history-write-unavailable"; return false; } } return { - brain, read, flush, + brain, read, flush, salt: read()?.metadata.state.salt ?? initial.metadata.state.salt, offer(value: Aggregate) { offer({ kind: "aggregate", value }); }, - failure(at = now()) { offer({ kind: "failure", at }); }, - status() { const history = read(); return { history, pending: pending !== null, paused: unavailable ?? history?.metadata.state.paused ?? null, payloadBytes: history ? jsonBytes(history) : null }; }, - close() { closed = true; pending = null; }, + failure(at = now()) { interrupted = true; offer({ kind: "failure", at }); }, + ownerWrite(report: StorageWriteReport) { + if (closed) return; + if (![report.identity, report.rootScope, report.target].every(value => /^[a-f0-9]{32}$/.test(value)) || !Number.isSafeInteger(report.at) || report.at < 0 || report.at > now()) throw Error("history-write-report-invalid"); + if (report.at < (read()?.metadata.state.lastPersistedAt ?? 0)) { interrupted = true; unavailable = "history-clock-reversal"; return; } + const next = [...pendingWrites, report]; + if (jsonBytes(next) > HISTORY_LIMITS.metadataBytes) { unavailable = "history-owner-evidence-budget"; return; } + pendingWrites = next; flush(); + }, + status() { + const history = read(); + const scan = db.query("SELECT result_at FROM storage_scan_leases WHERE brain = ?").get(brain) as { result_at: number | null } | null; + const waiting = scan?.result_at !== null && scan?.result_at !== undefined && scan.result_at > (history?.metadata.state.lastObservedAt ?? 0); + return { history, pending: pending !== null || pendingWrites.length > 0 || waiting, paused: unavailable ?? (history && now() < history.metadata.state.lastPersistedAt ? "history-clock-reversal" : history?.metadata.state.paused ?? null), payloadBytes: history ? jsonBytes(history) : null }; + }, + close() { closed = true; pending = null; pendingWrites = []; }, }; } export type StorageHistory = ReturnType; diff --git a/packages/ui-server/src/storage/maintenance.ts b/packages/ui-server/src/storage/maintenance.ts index 0b972a0c20..8419712f63 100644 --- a/packages/ui-server/src/storage/maintenance.ts +++ b/packages/ui-server/src/storage/maintenance.ts @@ -95,6 +95,9 @@ export function createStorageMaintenance(db: Database, options: { if (value.schema_version !== 1 || value.action !== "pack" || !["complete", "partial", "refused"].includes(value.state ?? "") || value.packed !== null && !count(value.packed)) throw Error("packing_report_unavailable"); result.outcome.packed = value.packed as number | null; result.state = value.state as "complete" | "partial" | "refused"; result.reason = value.reason === "no_space" ? "no_space" : result.state === "complete" ? null : "git_maintenance_incomplete"; + // The existing core owner reports its worker result. It does not identify + // a host mount or distinguish byte/inode exhaustion, so neither do we. + if (value.reason === "no_space" || value.state === "complete") options.storage.ownerWrite("unavailable:Native checkpoints and output", "git-packing", "unknown", value.reason === "no_space" ? "no-space" : "success"); } else if (result.operation === "prune-scratch") { let raw: string; try { raw = await options.scratch.run({ signal: controller.signal, assertAuthorized, bounded: true }); } diff --git a/packages/ui-server/src/storage/notify.ts b/packages/ui-server/src/storage/notify.ts index 75d47ff8b9..3654f5b156 100644 --- a/packages/ui-server/src/storage/notify.ts +++ b/packages/ui-server/src/storage/notify.ts @@ -28,30 +28,41 @@ export function createStorageNotifier(db: Database, history: StorageHistory) { db.query("UPDATE storage_notice_receipts SET state = 'unknown', next_at = NULL, pid = NULL, process_start = NULL WHERE brain = ? AND episode = ? AND endpoint = ? AND state = 'sending'").run(brain, row.episode, row.endpoint); } } + function maintain(now: number) { + reap(now); + const state = history.read()?.metadata.state; + if (!state) return; + const retained = new Set(state.pressures.flatMap(row => [row.episode, row.resolvedEpisode].filter((id): id is string => id !== null))); + const active = new Set(state.pressures.filter(row => row.severity === "critical" && row.episode).map(row => row.episode!)); + const rows = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ?").all(brain) as Receipt[]; + for (const row of rows) { + if (!active.has(row.episode) && ["pending", "failed"].includes(row.state)) db.query("UPDATE storage_notice_receipts SET state = 'suppressed', next_at = NULL WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); + if (!retained.has(row.episode) && row.state !== "sending") db.query("DELETE FROM storage_notice_receipts WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); + } + } return { + sweep(now = Date.now()) { db.transaction(() => maintain(now)).immediate(); }, begin(endpoint: string, now = Date.now()): StorageNoticeAttempt | null { if (!Number.isSafeInteger(now) || now < lastClock) return null; lastClock = now; return db.transaction((): StorageNoticeAttempt | null => { if (db.query("SELECT 1 FROM inbox_recovery_state WHERE status = 'pending'").get()) return null; - reap(now); + maintain(now); const status = history.status(), payload = status.history; if (!payload) return null; const state = payload.metadata.state; + // Another process may have a newer observation waiting for the bounded + // history write. Never dispatch using the older durable pressure state. + const scan = db.query("SELECT result_at FROM storage_scan_leases WHERE brain = ?").get(brain) as { result_at: number | null } | null; + if (scan?.result_at !== null && scan?.result_at !== undefined && scan.result_at > state.lastObservedAt) return null; const lastAttempt = db.query("SELECT MAX(attempted_at) AS at FROM storage_notice_receipts WHERE brain = ?").get(brain) as { at: number | null }; if (state.lastPersistedAt > now || lastAttempt.at !== null && lastAttempt.at > now) return null; - const retained = new Set(state.pressures.flatMap(row => [row.episode, row.resolvedEpisode].filter((id): id is string => id !== null))); - const active = new Set(state.pressures.filter(row => row.severity === "critical" && row.episode).map(row => row.episode!)); - const rows = db.query("SELECT * FROM storage_notice_receipts WHERE brain = ?").all(brain) as Receipt[]; - for (const row of rows) { - if (!active.has(row.episode) && ["pending", "failed"].includes(row.state)) db.query("UPDATE storage_notice_receipts SET state = 'suppressed', next_at = NULL WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); - if (!retained.has(row.episode) && row.state !== "sending") db.query("DELETE FROM storage_notice_receipts WHERE brain = ? AND episode = ? AND endpoint = ?").run(brain, row.episode, row.endpoint); - } const sub = db.query("SELECT principal_id, time_zone, p256dh, auth FROM push_subscriptions WHERE endpoint = ?").get(endpoint) as { principal_id: string | null; time_zone: string | null; p256dh: string; auth: string } | null; if (!sub?.principal_id || !owner(sub.principal_id, now)) return null; const eligible = state.pressures.filter(row => row.severity === "critical" && row.episode && !row.scopeChanged && row.supportAt !== null && row.supportAt <= now && now - row.supportAt <= RECENT && !state.suppressedEpisodes.includes(row.episode)); for (const row of eligible) db.query("INSERT OR IGNORE INTO storage_notice_receipts (brain, episode, endpoint, principal_id, state) VALUES (?, ?, ?, ?, 'pending')").run(brain, row.episode!, endpoint, sub.principal_id); - if (status.paused || status.pending) return null; + if (status.pending || status.paused && status.paused !== "collection-unavailable") return null; + if (status.paused === "collection-unavailable" && !eligible.some(row => row.refusals.some(refusal => refusal.at >= state.lastObservedAt))) return null; const zone = noticeTimeZone(sub.time_zone); if (!zone) return null; const hour = localHour(zone, now); @@ -83,7 +94,7 @@ export function createStorageNotifier(db: Database, history: StorageHistory) { }, receipts(principalId: string, now = Date.now()): Array<{ episode: string; state: StorageNoticeState; attempts: number; nextAt: number | null }> { if (!owner(principalId, now)) throw Error("Storage requires current human-owner access."); - return db.query("SELECT r.episode, r.state, r.attempts, r.next_at AS nextAt FROM storage_notice_receipts r JOIN push_subscriptions s ON s.endpoint = r.endpoint WHERE r.brain = ? AND s.principal_id = ? AND r.principal_id = ? ORDER BY r.episode, r.endpoint").all(brain, principalId, principalId) as Array<{ episode: string; state: StorageNoticeState; attempts: number; nextAt: number | null }>; + return db.query("SELECT r.episode, r.state, r.attempts, r.next_at AS nextAt FROM storage_notice_receipts r JOIN push_subscriptions s ON s.endpoint = r.endpoint WHERE r.brain = ? AND s.principal_id = ? AND r.principal_id = ? ORDER BY r.episode, r.endpoint LIMIT 201").all(brain, principalId, principalId) as Array<{ episode: string; state: StorageNoticeState; attempts: number; nextAt: number | null }>; }, }; } diff --git a/packages/ui-server/src/storage/process.ts b/packages/ui-server/src/storage/process.ts index 8164e7be23..a61a8f2fd9 100644 --- a/packages/ui-server/src/storage/process.ts +++ b/packages/ui-server/src/storage/process.ts @@ -1,3 +1,4 @@ +import { processStart } from "../brain/writer-gate.js"; import { fileURLToPath } from "node:url"; import { dirname, relative, isAbsolute } from "node:path"; import { realpathSync, readFileSync } from "node:fs"; @@ -12,14 +13,28 @@ function coreReader(brainPath: string, subpath: "storage" | "queries"): string { if (inside(brain, manifestPath) || inside(brain, entry) || !inside(dirname(manifestPath), entry)) throw Error("storage-reader-untrusted"); return entry; } -export async function coreMetadata(brainPath: string, gitRoots: string[], signal: AbortSignal): Promise { +/** Concrete lease bookkeeping for the two fixed metadata readers. */ +export interface StorageWorkerLease { register(pid: number): void; unregister(pid: number): void } +export function storageReaderSource(source: string): string { + const parent = { pid: process.pid, start: processStart(process.pid) }; + return `import {readFileSync as storageReadParent} from "node:fs"; +const storageParent=${JSON.stringify(parent)}; +const storageGuard=setInterval(()=>{try { + const text=storageReadParent("/proc/"+storageParent.pid+"/stat","utf8"); + const start=text.slice(text.lastIndexOf(")")+2).split(" ")[19]; + if(storageParent.start===null || start===storageParent.start)return; +}catch{}process.kill(process.pid,"SIGKILL");},250);storageGuard.unref(); +const storageDeadline=setTimeout(()=>process.kill(process.pid,"SIGKILL"),35000);storageDeadline.unref(); +${source}`; +} +export async function coreMetadata(brainPath: string, gitRoots: string[], signal: AbortSignal, lease?: StorageWorkerLease): Promise { const source = `import {readStorageInventory} from ${JSON.stringify(coreReader(brainPath, "storage"))}; const controller=new AbortController();process.once("SIGTERM",()=>controller.abort()); console.log(JSON.stringify(await readStorageInventory({...${JSON.stringify({ brainPath, gitRoots })},signal:controller.signal})));`; - return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal }); + return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", storageReaderSource(source)], { signal, lease }); } /** No shell, inherited secrets, repository config or unbounded stdout. Waits for cancellation cleanup. */ -export async function storageProcess(argv: string[], options: { signal: AbortSignal; bytes?: number }): Promise { +export async function storageProcess(argv: string[], options: { signal: AbortSignal; bytes?: number; lease?: StorageWorkerLease }): Promise { options.signal.throwIfAborted(); const child = Bun.spawn(argv, { cwd: "/", env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8" }, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); let reason: string | null = null, force: ReturnType | undefined; @@ -38,6 +53,7 @@ export async function storageProcess(argv: string[], options: { signal: AbortSig return Buffer.concat(chunks).toString("utf8"); }; try { + options.lease?.register(child.pid); const [out, , code] = await Promise.all([read(child.stdout), read(child.stderr), child.exited]); if (reason || code !== 0) throw Error(reason ?? "collector-unavailable"); return JSON.parse(out); @@ -46,13 +62,14 @@ export async function storageProcess(argv: string[], options: { signal: AbortSig options.signal.removeEventListener("abort", abort); signal("SIGKILL"); await child.exited; + options.lease?.unregister(child.pid); } } -export async function serverMetadata(roots: MetadataRoot[], signal: AbortSignal, entries: number): Promise { +export async function serverMetadata(roots: MetadataRoot[], signal: AbortSignal, entries: number, lease?: StorageWorkerLease): Promise { const source = `import {collectMetadata} from ${JSON.stringify(import.meta.resolve("./metadata.js"))}; const controller=new AbortController();process.once("SIGTERM",()=>controller.abort()); console.log(JSON.stringify(await collectMetadata(${JSON.stringify(roots)},{signal:controller.signal,entries:${entries}})));`; - return await storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal }) as MetadataResult; + return await storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", storageReaderSource(source)], { signal, lease }) as MetadataResult; } /** Isolated supported core query, with the same trusted-installation boundary as inventory. */ @@ -61,5 +78,5 @@ export async function storageReferences(brainPath: string, path: string, signal: const source = `import {createCoreQueryAccess} from ${JSON.stringify(import.meta.resolve("../core-queries.js"))}; const feature=createCoreQueryAccess().storage?.(); console.log(JSON.stringify(feature?.ok?feature.queries.readStorageReferences(${JSON.stringify({ brainPath, path })}):{ok:false}));`; - return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", source], { signal, bytes: 1024 * 1024 }); + return storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", storageReaderSource(source)], { signal, bytes: 1024 * 1024 }); } diff --git a/packages/ui-server/src/storage/removal.ts b/packages/ui-server/src/storage/removal.ts index abf12ca3ac..165fa6ebd0 100644 --- a/packages/ui-server/src/storage/removal.ts +++ b/packages/ui-server/src/storage/removal.ts @@ -29,8 +29,15 @@ export function createStorageRemoval(db: Database, options: { authorized(principal: string): boolean; /** Existing operator-owned stores, represented relative to the canonical brain when applicable. */ protectedPaths: readonly string[]; + retainedRoot?: string; }) { const brain = realpathSync(options.brainPath), journal = options.journal; + const record = (root: string, target: string, outcome: "success" | "no-space", error?: unknown) => { + const code = (error as { code?: string; reason?: string } | undefined)?.code; + const quota = code === "EDQUOT" || (error as { reason?: string } | undefined)?.reason === "journal_capacity"; + if (outcome === "success") for (const resource of ["bytes", "quota", "inodes", "unknown"] as const) options.storage.ownerWrite(root, target, resource, outcome); + else if (quota || code === "ENOSPC") options.storage.ownerWrite(root, target, quota ? "quota" : "unknown", outcome); + }; const active = new Map }>(); const reviews = new Set(), pendingReviews = new Set>(); let closed = false; const key = (id: string) => `${brain}:${id}`; @@ -162,7 +169,10 @@ export function createStorageRemoval(db: Database, options: { if (row.action === "remove") { const current = await inspectStorageAttachment(brain, item.path, controller.signal); if (JSON.stringify(current) !== JSON.stringify(privateFile.version)) throw new StorageAttachmentRefusal("changed"); - privateFile.retained = await journal.retainBefore(row.journal_turn_id, item.path, controller.signal); + try { + privateFile.retained = await journal.retainBefore(row.journal_turn_id, item.path, controller.signal); + if (privateFile.retained && options.retainedRoot) record(`store:${options.retainedRoot}`, "retained-attachment", "success"); + } catch (error) { if (options.retainedRoot) record(`store:${options.retainedRoot}`, "retained-attachment", "no-space", error); throw error; } if (!privateFile.retained || privateFile.retained.hash !== current.hash || privateFile.retained.bytes !== current.bytes || privateFile.retained.mode !== current.mode || !journal.hasRetainedVersion(row.journal_turn_id, privateFile.retained)) throw Error("retained_before_unavailable"); } else { const source = select(row.source_id!); @@ -172,7 +182,10 @@ export function createStorageRemoval(db: Database, options: { check(); if (protectedReason(item.path)) throw new StorageAttachmentRefusal("protected"); item.state = "applying"; save(row, result, versions); // Durable retained identity and item intent precede the first file effect. if (row.action === "remove") await removeStorageAttachment(brain, item.path, privateFile.version!, controller.signal, check); - else journal.withRetainedRemoval(select(row.source_id!)!.journal_turn_id, privateFile.retained!, copy => restoreStorageAttachment(brain, item.path, privateFile.version!, copy, check)); + else { + try { journal.withRetainedRemoval(select(row.source_id!)!.journal_turn_id, privateFile.retained!, copy => restoreStorageAttachment(brain, item.path, privateFile.version!, copy, check)); record("brain", item.path, "success"); } + catch (error) { record("brain", item.path, "no-space", error); throw error; } + } effects = true; item.state = row.action === "remove" ? "removed" : "restored"; item.reason = null; save(row, result, versions); // An acknowledgement failure cannot authorize another file. await journal.observe(); diff --git a/packages/ui-server/src/storage/sampling.ts b/packages/ui-server/src/storage/sampling.ts new file mode 100644 index 0000000000..941e9fb4e7 --- /dev/null +++ b/packages/ui-server/src/storage/sampling.ts @@ -0,0 +1,40 @@ +import type { Database } from "bun:sqlite"; +import type { PushSender } from "../activity/push-sender.js"; +import { isUsablePrincipal, resolvePrincipal } from "../db/principals.js"; +import type { StorageCoordinator } from "./coordinator.js"; +import type { StorageHistory } from "./history-store.js"; +import type { StorageNotifier } from "./notify.js"; +import type { SharedStorageScan } from "./shared-scan.js"; +import { bucket } from "./history-model.js"; +/** Background work starts at the next UTC six-hour boundary, never at boot or on a read. */ +export function createStorageSampling(db: Database, storage: StorageCoordinator, history: StorageHistory, shared: SharedStorageScan, notices: StorageNotifier, sender: PushSender, bootAt = Date.now()) { + let lastBucket = bucket(0, bootAt), lastClock = bootAt, lastCleanup = bootAt; + let delivering: Promise | null = null, cleaning: Promise | null = null; + const controller = new AbortController(); + function tick(now = Date.now()) { + if (controller.signal.aborted) return; + // Reversed clocks cannot re-enter a past bucket or renew stale evidence. + if (now < lastClock) return; + lastClock = now; + try { history.flush(); } catch { /* Existing evidence remains readable. */ } + const current = bucket(0, now); + if (current > lastBucket) { + try { + const candidates = db.query("SELECT id FROM principals WHERE kind IN ('owner', 'ambient') AND revoked_at IS NULL AND expires_at > ? ORDER BY created_at DESC LIMIT 100").all(now) as Array<{ id: string }>; + const owner = candidates.find(({ id }) => { const principal = resolvePrincipal(db, id); return principal && isUsablePrincipal(principal, now); }); + if (owner) { lastBucket = current; storage.start(owner.id, true, 0); } + } catch { lastBucket = current; history.failure(now); } + } + try { notices.sweep(now); } catch { /* A failed receipt write cannot authorize transport. */ } + if (!delivering) delivering = sender.deliverStorage(notices, undefined, controller.signal).catch(() => {}).finally(() => { delivering = null; }); + if (!cleaning && now - lastCleanup >= 60_000) { + lastCleanup = now; + cleaning = shared.cleanupCache().catch(() => {}).finally(() => { cleaning = null; }); + } + } + const timer = setInterval(tick, 20_000); timer.unref(); + return { + tick, + async close() { controller.abort(); clearInterval(timer); await Promise.allSettled([delivering, cleaning]); history.flush(); }, + }; +} diff --git a/packages/ui-server/src/storage/shared-scan.ts b/packages/ui-server/src/storage/shared-scan.ts new file mode 100644 index 0000000000..de0983de67 --- /dev/null +++ b/packages/ui-server/src/storage/shared-scan.ts @@ -0,0 +1,177 @@ +import type { Database } from "bun:sqlite"; +import { constants, readFileSync } from "node:fs"; +import { mkdir, open, readdir, rename, rm, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { processAlive, processStart, SELF } from "../brain/writer-gate.js"; +import { bucket, type Source } from "./history-model.js"; +import { storageBrainKey } from "./history-store.js"; +import { STORAGE_SERVER_LIMITS, type StorageCollection } from "./collect.js"; +const CACHE_LIFETIME = 120_000; +const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; +type Worker = { pid: number; start: string }; +type Result = { id: string; scope: string; at: number; cached: boolean; failed: boolean }; +interface Row { brain: string; job_id: string | null; pid: number | null; process_start: string | null; principal_id: string | null; started_at: number | null; scope: string | null; source: Source | null; workers_json: string; shared: number; results_json: string; cancel_requested: number; last_request_at: number | null; last_attempt_bucket: number | null } +export interface SharedScan { + id: string; principal: string; scope: string; source: Source; owner: boolean; inherited: Worker[]; +} +const sleep = (milliseconds: number) => new Promise(resolve => setTimeout(resolve, milliseconds)); +function alive(worker: Worker) { + if (!processAlive(worker.pid, worker.start)) return false; + try { const text = readFileSync(`/proc/${worker.pid}/stat`, "utf8"); return text.slice(text.lastIndexOf(")") + 2).split(" ")[0] !== "Z"; } + catch { return true; } +} +function workers(row: Row): Worker[] { + const value = JSON.parse(row.workers_json) as Worker[]; + if (!Array.isArray(value) || value.length > 2 || value.some(item => !Number.isSafeInteger(item.pid) || item.pid <= 0 || !/^\d+$/.test(item.start))) throw Error("storage-lease-workers-unavailable"); + return value; +} +function results(row: Row): Result[] { + const value = JSON.parse(row.results_json) as Result[]; + if (!Array.isArray(value) || value.length > 2 || value.some(item => !UUID.test(item.id) || !/^[a-f0-9]{32}$/.test(item.scope) || !Number.isSafeInteger(item.at) || typeof item.cached !== "boolean" || typeof item.failed !== "boolean")) throw Error("storage-lease-results-unavailable"); + return value; +} +/** One operational lease for the brain. Only actual cross-process/instance joiners need a disposable file transfer. */ +export function createSharedStorageScan(db: Database, brainPath: string, cacheRoot: string, authorized: (principal: string) => boolean) { + const brain = storageBrainKey(brainPath); + const row = () => db.query("SELECT * FROM storage_scan_leases WHERE brain = ?").get(brain) as Row | null; + function authorize(principal: string) { if (!authorized(principal)) throw Error("storage-scan-owner-revoked"); } + function owned(scan: SharedScan): Row { + authorize(scan.principal); + const value = row(); + if (!scan.owner || value?.job_id !== scan.id || value.pid !== SELF.pid || value.process_start !== SELF.start || value.scope !== scan.scope || value.cancel_requested) throw Error("storage-scan-authority-lost"); + return value; + } + async function cleanupCache(reserveSlot: boolean, now = Date.now()) { + if (reserveSlot) await mkdir(cacheRoot, { recursive: true, mode: 0o700 }); + const directory = await open(cacheRoot, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch(error => { if (!reserveSlot && error.code === "ENOENT") return null; throw error; }); + if (!directory) return; + try { + const names = await readdir(cacheRoot); + if (names.length > 8 || names.some(name => !/^[a-f0-9-]{36}\.(?:json|partial)$/.test(name))) throw Error("storage-shared-cache-unavailable"); + const files = await Promise.all(names.map(async name => ({ name, info: await stat(join(cacheRoot, name)) }))); + if (!reserveSlot && row()?.job_id) return; + for (const file of files) if (file.name.endsWith(".partial") || now - file.info.mtimeMs > CACHE_LIFETIME) await rm(join(cacheRoot, file.name)); + const kept = files.filter(file => !file.name.endsWith(".partial") && now - file.info.mtimeMs <= CACHE_LIFETIME).sort((a, b) => a.info.mtimeMs - b.info.mtimeMs); + while (kept.length > (reserveSlot ? 1 : 2)) await rm(join(cacheRoot, kept.shift()!.name)); + } finally { await directory.close(); } + } + return { + cacheRoot, + acquire(principal: string, id: string, scope: string, source: Source, fresh: boolean, now = Date.now()): SharedScan | null { + authorize(principal); + if (!UUID.test(id) || !/^[a-f0-9]{32}$/.test(scope)) throw Error("storage-scan-identity-invalid"); + return db.transaction(() => { + const current = row(); + if (current?.job_id && current.pid && current.process_start !== null && alive({ pid: current.pid, start: current.process_start })) { + if (current.scope !== scope) throw Error("storage-scan-other-scope-busy"); + db.query("UPDATE storage_scan_leases SET shared = 1 WHERE brain = ? AND job_id = ?").run(brain, current.job_id); + return { id: current.job_id, principal, scope, source: current.source!, owner: false, inherited: [] }; + } + if (current?.last_request_at !== null && current?.last_request_at !== undefined && now < current.last_request_at) throw Error("storage-scan-clock-reversal"); + if (source === 0 && current?.last_attempt_bucket !== null && current?.last_attempt_bucket !== undefined && current.last_attempt_bucket >= bucket(0, now)) return null; + if (!fresh && current?.last_request_at !== null && current?.last_request_at !== undefined && now - current.last_request_at < 5000) throw Error("storage-scan-cooldown"); + const inherited = current ? workers(current) : []; + db.query(`INSERT INTO storage_scan_leases (brain, job_id, pid, process_start, principal_id, started_at, scope, source, last_request_at, last_attempt_bucket) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(brain) DO UPDATE SET job_id = excluded.job_id, pid = excluded.pid, process_start = excluded.process_start, + principal_id = excluded.principal_id, started_at = excluded.started_at, scope = excluded.scope, source = excluded.source, + cancel_requested = 0, shared = 0, last_request_at = excluded.last_request_at, last_attempt_bucket = excluded.last_attempt_bucket`) + .run(brain, id, SELF.pid, SELF.start, principal, now, scope, source, now, bucket(0, now)); + return { id, principal, scope, source, owner: true, inherited }; + }).immediate(); + }, + check: owned, + async reap(scan: SharedScan) { + // A dead coordinator's exact worker identities remain durable until their + // processes stop. Never signal a reused PID or a different owner. + for (const worker of scan.inherited) { + if (!alive(worker)) continue; + try { process.kill(worker.pid, "SIGTERM"); } catch { /* already exited */ } + } + const deadline = Date.now() + 1500; + while (scan.inherited.some(alive) && Date.now() < deadline) await sleep(20); + for (const worker of scan.inherited) if (alive(worker)) { try { process.kill(worker.pid, "SIGKILL"); } catch { /* already exited */ } } + const forceDeadline = Date.now() + 1500; + while (scan.inherited.some(alive) && Date.now() < forceDeadline) await sleep(20); + if (scan.inherited.some(alive)) throw Error("storage-scan-worker-reap-unavailable"); + db.transaction(() => { owned(scan); db.query("UPDATE storage_scan_leases SET workers_json = '[]' WHERE brain = ? AND job_id = ?").run(brain, scan.id); }).immediate(); + }, + register(scan: SharedScan, pid: number) { + const start = processStart(pid); + if (!start) return; // a child which already exited needs no reap authority + db.transaction(() => { + const value = workers(owned(scan)); + if (value.length >= 2) throw Error("storage-scan-worker-budget"); + value.push({ pid, start }); + db.query("UPDATE storage_scan_leases SET workers_json = ? WHERE brain = ? AND job_id = ?").run(JSON.stringify(value), brain, scan.id); + }).immediate(); + }, + unregister(scan: SharedScan, pid: number) { + db.transaction(() => { + const value = row(); + if (value?.job_id !== scan.id || value.pid !== SELF.pid || value.process_start !== SELF.start) return; + db.query("UPDATE storage_scan_leases SET workers_json = ? WHERE brain = ? AND job_id = ?").run(JSON.stringify(workers(value).filter(worker => worker.pid !== pid)), brain, scan.id); + }).immediate(); + }, + cancel(scan: SharedScan) { + authorize(scan.principal); + db.query("UPDATE storage_scan_leases SET cancel_requested = 1 WHERE brain = ? AND job_id = ? AND scope = ?").run(brain, scan.id, scan.scope); + }, + async publish(scan: SharedScan, collection: StorageCollection) { + // An uncontended scan stays in the original bounded memory cache: it does + // not write a second copy of every filename to disk merely for sampling. + const needsCache = db.transaction(() => owned(scan).shared === 1).immediate(); + if (!needsCache) return false; + const serialized = JSON.stringify(collection); + if (Buffer.byteLength(serialized) > STORAGE_SERVER_LIMITS.cacheBytes / 2) throw Error("storage-shared-cache-budget"); + await cleanupCache(true); + owned(scan); + const temporary = join(cacheRoot, `${scan.id}.partial`), destination = join(cacheRoot, `${scan.id}.json`); + const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await file.writeFile(serialized); await file.sync(); owned(scan); await rename(temporary, destination); } + catch (error) { await rm(temporary, { force: true }); throw error; } + finally { await file.close(); } + return true; + }, + finish(scan: SharedScan, cached: boolean, failed: boolean, at = Date.now()): boolean { + return db.transaction(() => { + const current = row(); + if (current?.job_id !== scan.id || current.pid !== SELF.pid || current.process_start !== SELF.start) return false; + if (!failed) owned(scan); + if (workers(current).some(alive)) return false; + // A joiner may arrive just after publish() checked the flag. Keep the + // lease until its full result has been published, then retry finish. + if (!failed && current.shared && !cached) return false; + const completed = [...results(current), { id: scan.id, scope: scan.scope, at, cached, failed }].slice(-2); + db.query(`UPDATE storage_scan_leases SET job_id = NULL, pid = NULL, process_start = NULL, principal_id = NULL, + started_at = NULL, scope = NULL, source = NULL, workers_json = '[]', cancel_requested = 0, shared = 0, + results_json = ?, result_id = ?, result_at = ?, failure = ? WHERE brain = ? AND job_id = ?`) + .run(JSON.stringify(completed), scan.id, at, failed ? "collection-unavailable" : null, brain, scan.id); + return true; + }).immediate(); + }, + async wait(scan: SharedScan, signal: AbortSignal): Promise { + const deadline = Date.now() + STORAGE_SERVER_LIMITS.milliseconds; + while (Date.now() <= deadline) { + signal.throwIfAborted(); authorize(scan.principal); + const current = row(), completed = current && results(current).find(value => value.id === scan.id && value.scope === scan.scope); + if (completed) { + if (completed.failed || !completed.cached || Date.now() - completed.at > CACHE_LIFETIME || completed.at > Date.now()) throw Error("storage-shared-result-unavailable"); + const file = await open(join(cacheRoot, `${scan.id}.json`), constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const info = await file.stat(); + if (!info.isFile() || info.size > STORAGE_SERVER_LIMITS.cacheBytes / 2) throw Error("storage-shared-result-budget"); + const result = JSON.parse(await file.readFile("utf8")) as StorageCollection; + if (result.summary?.id !== scan.id || !Array.isArray(result.entries) || result.entries.length > STORAGE_SERVER_LIMITS.entries || result.bytes > STORAGE_SERVER_LIMITS.cacheBytes / 2) throw Error("storage-shared-result-invalid"); + signal.throwIfAborted(); authorize(scan.principal); return result; + } finally { await file.close(); } + } + if (!current?.job_id || current.job_id !== scan.id || !current.pid || current.process_start === null || !alive({ pid: current.pid, start: current.process_start })) throw Error("storage-shared-scan-interrupted"); + await sleep(100); + } + throw Error("storage-shared-scan-time-budget"); + }, + cleanupCache() { return cleanupCache(false); }, + }; +} +export type SharedStorageScan = ReturnType; diff --git a/packages/ui-server/tests/fixtures/storage-notify-process.ts b/packages/ui-server/tests/fixtures/storage-notify-process.ts new file mode 100644 index 0000000000..14b7124bf2 --- /dev/null +++ b/packages/ui-server/tests/fixtures/storage-notify-process.ts @@ -0,0 +1,15 @@ +import { appendFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { createUiDb } from "../../src/db/client.js"; +import { createStorageHistory } from "../../src/storage/history-store.js"; +import { createStorageNotifier } from "../../src/storage/notify.js"; +import { createPushSender } from "../../src/activity/push-sender.js"; +const [root, mode, marker, rawAt] = process.argv.slice(2) as [string, string, string, string]; +const at = Number(rawAt), db = createUiDb(join(root, "operations.sqlite")); +const history = createStorageHistory(db, root, { now: () => at }), notices = createStorageNotifier(db, history); +const sender = createPushSender(db, { send: async () => { + appendFileSync(join(root, "transport-attempts.txt"), "attempt\n"); + if (mode === "crash") { writeFileSync(marker, "DISPATCHED\n"); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000); } +} }); +console.log("READY"); await Bun.stdin.text(); +console.log(JSON.stringify({ attempts: await sender.deliverStorage(notices, at) })); db.close(); diff --git a/packages/ui-server/tests/fixtures/storage-shared-process.ts b/packages/ui-server/tests/fixtures/storage-shared-process.ts new file mode 100644 index 0000000000..3cb845c766 --- /dev/null +++ b/packages/ui-server/tests/fixtures/storage-shared-process.ts @@ -0,0 +1,41 @@ +import { join } from "node:path"; +import { writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { createUiDb } from "../../src/db/client.js"; +import { createStorageHistory } from "../../src/storage/history-store.js"; +import { createSharedStorageScan } from "../../src/storage/shared-scan.js"; +import { createStorageCoordinator } from "../../src/storage/coordinator.js"; +import { storageProcess, storageReaderSource } from "../../src/storage/process.js"; +import { hashScope } from "../../src/storage/history-model.js"; +import { processStart } from "../../src/brain/writer-gate.js"; +const [root, mode, marker] = process.argv.slice(2) as [string, string, string]; +const brain = join(root, "brain"), db = createUiDb(join(root, "operations.sqlite")), history = createStorageHistory(db, brain); +const shared = createSharedStorageScan(db, brain, join(root, "cache"), id => id === "odysseus-owner"); +if (mode === "collect") { + let collections = 0; + const coordinator = createStorageCoordinator({ brainPath: brain, stores: [], salt: history.read()!.metadata.state.salt, shared, + authorized: id => id === "odysseus-owner", onCollection() { collections++; } }); + console.log("READY"); + await Bun.stdin.text(); + const result = coordinator.start("odysseus-owner", true); + await coordinator.idle(); + const status = coordinator.status("odysseus-owner"), snapshot = status.snapshot ?? status.previous; + if (!snapshot) throw Error(JSON.stringify(status.lastFailure)); + const page = coordinator.page("odysseus-owner", snapshot.id, {}); + console.log(JSON.stringify({ requested: result.activeScanId, id: snapshot.id, collections, names: page.entries.map(row => row.name), failure: status.lastFailure })); + await coordinator.close(); db.close(); +} else { + const lease = shared.acquire("odysseus-owner", randomUUID(), hashScope("Odysseus guard fixture"), 1, true)!; + await shared.reap(lease); + await storageProcess([process.execPath, "--config=/dev/null", "--no-env-file", "-e", storageReaderSource("setInterval(()=>{},1000); await new Promise(()=>{});")], { + signal: new AbortController().signal, + lease: { + register(pid) { + if (mode !== "before-register") shared.register(lease, pid); + writeFileSync(marker, JSON.stringify({ pid, start: processStart(pid), lease: lease.id }) + "\n"); + if (mode === "before-register") Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000); + }, + unregister(pid) { shared.unregister(lease, pid); }, + }, + }); +} diff --git a/packages/ui-server/tests/http-supported-mounts.test.ts b/packages/ui-server/tests/http-supported-mounts.test.ts index 2f1ea05843..7a85b9029a 100644 --- a/packages/ui-server/tests/http-supported-mounts.test.ts +++ b/packages/ui-server/tests/http-supported-mounts.test.ts @@ -16,6 +16,8 @@ type Check = { path?: string; init?: RequestInit; status?: number; body?: object // More detailed existing behavior checks are linked by the coverage matrix; // this map is cross-checked with the spec and the actual Hono mounts. const checks: Record = { + "GET /api/storage/history": { body: { version: 1, state: "available", observations: [] } }, + "GET /api/storage/guidance": { body: { version: 1, state: "unknown", severity: null, capacities: [] } }, "GET /api/storage/removals": { verify: async r => { const value = await r.json(); expect(value).toHaveProperty("available"); expect(value.limits.items).toBe(32); } }, "POST /api/storage/removals/inspection": { init: json("POST", {paths: []}), status: 400, body: {error:"Select bounded attachment paths for inspection."} }, "POST /api/storage/removals/reviews": { init: json("POST", { id: "odysseus-mounted-review", paths: [] }), status: 400, body: {error:"Select bounded attachment paths and a new review identity."} }, diff --git a/packages/ui-server/tests/storage-guidance.test.ts b/packages/ui-server/tests/storage-guidance.test.ts new file mode 100644 index 0000000000..261946f001 --- /dev/null +++ b/packages/ui-server/tests/storage-guidance.test.ts @@ -0,0 +1,100 @@ +import { afterEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { generateSignedCookie } from "hono/cookie"; +import { createUiDb } from "../src/db/client.js"; +import { createPrincipal, revokePrincipal } from "../src/db/principals.js"; +import { createStorageHistory } from "../src/storage/history-store.js"; +import { createSharedStorageScan } from "../src/storage/shared-scan.js"; +import { createStorageCoordinator } from "../src/storage/coordinator.js"; +import { createStorageNotifier } from "../src/storage/notify.js"; +import { createStorageSampling } from "../src/storage/sampling.js"; +import { createPushSender } from "../src/activity/push-sender.js"; +import { aggregateCollection, RECENT, hashScope, type Aggregate } from "../src/storage/history-model.js"; +import { exportInboxSnapshot, restoreInboxSnapshot } from "../src/inbox/snapshot.js"; +import { httpContractApp } from "./helpers/http-contract-app.js"; +const cleanup: Array<() => void | Promise> = []; +afterEach(async () => { for (const close of cleanup.splice(0).reverse()) await close(); }); + +test("mounted named history and guidance are read-only and require current human-owner authority", async () => { + const secret = "storage-odysseus-history-secret-0123456789"; + const f = await httpContractApp({ env: { AUTH_MODE: "password", BRAIN_UI_PASSWORD_HASH: "unused-fixture-hash", COOKIE_SECRET: secret, BRAIN_UI_ALLOW_LOOPBACK_ORIGIN: "1" } }); cleanup.push(() => f.close()); + const owner = createPrincipal(f.app.db, { authMethod: "password", label: "Odysseus", ttlSeconds: 3600 }), agent = createPrincipal(f.app.db, { authMethod: "delegated", createdBy: owner.id, label: "Athena", ttlSeconds: 3600 }); + const cookie = async (id: string) => (await generateSignedCookie("brain_ui_session", id, secret)).split(";")[0]!; + const ownerHeaders = { cookie: await cookie(owner.id) }, agentHeaders = { cookie: await cookie(agent.id) }; + const count = () => f.app.db.query("SELECT COUNT(*) AS count FROM storage_scan_leases").get(); + for (const path of ["/api/storage/history", "/api/storage/guidance"]) { + expect((await f.fetch(path)).status).toBe(401); + expect((await f.fetch(path, { headers: agentHeaders })).status, "STORAGE_HISTORY_AGENT_DENIED").toBe(403); + const response = await f.fetch(path, { headers: ownerHeaders }); + expect(response.status, "STORAGE_NAMED_GUIDANCE_MOUNTED").toBe(200); expect(response.headers.get("cache-control")).toBe("private, no-store"); + expect(await response.json()).toMatchObject({ version: 1, pending: false }); + } + expect(count(), "STORAGE_HISTORY_READ_NEVER_SCANS").toEqual({ count: 0 }); + expect((await f.fetch("/api/storage/scans", { method: "POST", headers: { ...ownerHeaders, "content-type": "application/json" }, body: "{}" })).status).toBe(202); + let status: any; + for (let i = 0; i < 100; i++) { status = await (await f.fetch("/api/storage", { headers: ownerHeaders })).json(); if (status.state === "idle") break; await Bun.sleep(20); } + expect(status.snapshot).not.toBeNull(); + const history = await (await f.fetch("/api/storage/history", { headers: ownerHeaders })).json(); + expect(history.observations).toHaveLength(3); + expect(history.observations[0]).toMatchObject({ tier: "six-hour", source: "refresh", basis: "storage-v1", at: status.snapshot.observedAt }); + expect(history.observations[0].capacities[0]).toHaveProperty("owners.app.logical"); + const text = JSON.stringify(history); expect(text).not.toContain(f.brainPath); expect(text).not.toContain("notes/arrival.md"); expect(text).not.toContain("salt"); + revokePrincipal(f.app.db, owner.id, Date.now()); + for (const path of ["/api/storage/history", "/api/storage/guidance"]) expect((await f.fetch(path, { headers: ownerHeaders })).status, "STORAGE_GUIDANCE_REVOKED").toBe(401); +}); + +test("real sampler waits for a future six-hour boundary, coalesces owners and does no outage catch-up", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-sampling-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); + const brain = join(root, "brain"), cache = join(root, "cache"); mkdirSync(join(brain, ".git/objects"), { recursive: true }); mkdirSync(join(brain, ".git/refs/heads"), { recursive: true }); writeFileSync(join(brain, ".git/HEAD"), "ref: refs/heads/main\n"); writeFileSync(join(brain, "Odysseus.md"), "# Odysseus\n"); + const db = createUiDb(join(root, "operations.sqlite")); cleanup.push(() => db.close()); + let owner = createPrincipal(db, { authMethod: "password", label: "Odysseus", ttlSeconds: 365 * 24 * 3600 }); + const history = createStorageHistory(db, brain), shared = createSharedStorageScan(db, brain, cache, id => id === owner.id); + let count = 0; + const storage = createStorageCoordinator({ brainPath: brain, stores: [], shared, salt: history.salt, authorized: id => id === owner.id, + onCollection(value, source) { count++; history.offer(aggregateCollection(value, source, "0.44.0")); } }); cleanup.push(() => storage.close()); + const notices = createStorageNotifier(db, history), sender = createPushSender(db, { send: async () => { throw Error("No destination belongs to this fixture"); } }); + const boot = Date.now(), sampling = createStorageSampling(db, storage, history, shared, notices, sender, boot); cleanup.push(() => sampling.close()); + sampling.tick(boot); await storage.idle(); expect(count, "STORAGE_NO_BOOT_SCAN").toBe(0); expect(existsSync(cache)).toBe(false); + await sampling.close(); + const started = spyOn(storage, "start"); cleanup.push(() => started.mockRestore()); + // A sleeping process wakes after thirty missed buckets. Its current tick + // reaches the real coordinator once, without replaying those old intervals. + const waking = createStorageSampling(db, storage, history, shared, notices, sender, boot - 30 * RECENT); cleanup.push(() => waking.close()); + revokePrincipal(db, owner.id, boot); + waking.tick(boot); await storage.idle(); expect(count).toBe(0); + owner = createPrincipal(db, { authMethod: "password", label: "Odysseus", ttlSeconds: 365 * 24 * 3600 }); + waking.tick(boot + 1); waking.tick(boot + 2); await storage.idle(); + expect(count, "STORAGE_SCHEDULE_SINGLE_COLLECTION").toBe(1); + expect(started, "STORAGE_NO_OUTAGE_CATCHUP_REQUESTS").toHaveBeenCalledTimes(1); + expect(started).toHaveBeenCalledWith(owner.id, true, 0); + waking.tick(boot - 1); await storage.idle(); expect(count).toBe(1); + expect(history.read()!.samples).toHaveLength(3); + await waking.close(); waking.tick(boot + RECENT); expect(started).toHaveBeenCalledTimes(1); +}); + +test("operational export/restore carries history but suppresses old pending, accepted and uncertain episode authority", async () => { + const root = mkdtempSync(join(tmpdir(), "odysseus-storage-restore-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); + const brain = join(root, "brain"); mkdirSync(brain); const path = join(root, "source.sqlite"), db = createUiDb(path); cleanup.push(() => db.close()); + const at = Date.now(), history = createStorageHistory(db, brain, { now: () => at }), scope = hashScope("Odysseus stores"); + const value: Aggregate = { at, startedAt: at, source: 1, descriptor: ["0.44.0", scope, 1], coverage: 0, reason: 0, capacities: [{ descriptor: [hashScope("Odysseus volume"), scope], values: [1000, 960, 40, 40, 0, 1, 4096, 1, 4096, 0, 0, 2, 8192] }] }; + history.offer(value); const episode = history.read()!.metadata.state.pressures[0]!.episode!; + const owner = createPrincipal(db, { authMethod: "password", label: "Odysseus", ttlSeconds: 3600 }); + const sender = createPushSender(db, { send: async () => {} }); + for (const state of ["pending", "accepted", "unknown"] as const) { + const endpoint = `https://push.example.test/${state}`; sender.subscribe({ endpoint, keys: { p256dh: "opaque", auth: "opaque" } }, owner.id); + db.query("INSERT INTO storage_notice_receipts (brain, episode, endpoint, principal_id, state, attempts) VALUES (?, ?, ?, ?, ?, ?)").run(history.brain, episode, endpoint, owner.id, state, state === "pending" ? 0 : 1); + } + const snapshot = await exportInboxSnapshot(db, brain, at), destination = join(root, "restored.sqlite"); + await restoreInboxSnapshot(snapshot, destination, brain, at + 60_000); + const restoredDb = createUiDb(destination); cleanup.push(() => restoredDb.close()); + const restored = createStorageHistory(restoredDb, brain, { now: () => at + 120_000 }); + expect(restored.read()!.samples).toEqual(history.read()!.samples); + expect(restored.read()!.metadata.state.suppressedEpisodes, "STORAGE_ACTUAL_RESTORE_SUPPRESSES_EPISODE").toContain(episode); + expect(restoredDb.query("SELECT state FROM storage_notice_receipts ORDER BY endpoint").all()).toEqual([{ state: "accepted" }, { state: "suppressed" }, { state: "unknown" }]); + restored.offer({ ...value, at: at + 120_000, startedAt: at + 120_000 }); + restoredDb.query("UPDATE push_subscriptions SET time_zone = 'UTC'").run(); + const notices = createStorageNotifier(restoredDb, restored); + for (const name of ["pending", "accepted", "unknown"]) expect(notices.begin(`https://push.example.test/${name}`, at + 120_000), "STORAGE_RESTORED_HISTORY_HAS_NO_SEND_AUTHORITY").toBeNull(); +}); diff --git a/packages/ui-server/tests/storage-history-model.test.ts b/packages/ui-server/tests/storage-history-model.test.ts index 3c68f76dd4..3afcbe51c2 100644 --- a/packages/ui-server/tests/storage-history-model.test.ts +++ b/packages/ui-server/tests/storage-history-model.test.ts @@ -127,3 +127,11 @@ test("changed capacity, roots, unknown allocation and unrounded zero horizon pre const zero = steady(30, i => (120 - i) * 1_000_000); expect(capacityEvidence(zero, identity, at + 30 * DAY)?.short.days).toBe(0); }); + +test("collector patch versions keep comparable scopes without manufacturing gaps", () => { + const payload = steady(3), latest = payload.samples.filter(row => row[1] === 0).at(-1)![3]; + const next = observation(latest + RECENT, 887_000_000); next.descriptor[0] = "0.44.1"; + const changed = retainAggregate(payload, next); + expect(changed.samples.find(row => row[1] === 0 && row[3] === next.at)?.[2], "STORAGE_PATCH_VERSION_SAME_BASIS").toBe(0); + expect(capacityEvidence(changed, identity, next.at)?.short.state).toBe("finite"); +}); diff --git a/packages/ui-server/tests/storage-history-store.test.ts b/packages/ui-server/tests/storage-history-store.test.ts index f7e2e03c7a..891ff9b52b 100644 --- a/packages/ui-server/tests/storage-history-store.test.ts +++ b/packages/ui-server/tests/storage-history-store.test.ts @@ -95,3 +95,47 @@ test("malformed persisted warning authority is rejected instead of interpreted a (payload.metadata.state.pressures[0] as unknown as { severity: string }).severity = "unrecognised"; expect(() => parseStoredHistory(JSON.stringify(payload))).toThrow(); }); + +test("an intervening buffered failure resets recovery even when the newest pending observation succeeds", () => { + const f = fixture(); f.history.offer(aggregate(start, 50_000_000)); + f.advance(start + 60_000); f.history.offer(aggregate(start + 60_000, 80_000_000)); + f.advance(start + 70_000); f.history.failure(); + f.advance(start + 80_000); f.history.offer(aggregate(start + 80_000, 80_000_000)); + f.advance(start + 120_000); f.history.flush(); + const result = f.history.read()!; + expect(result.metadata.state.pressures[0], "STORAGE_PENDING_FAILURE_RESETS_PAIR").toMatchObject({ severity: "critical", recoveryAt: start + 80_000 }); + expect(result.samples.every(row => row[2] === 1)).toBe(true); +}); + +test("buffered owning-operation refusals survive a newer capacity sample and require matching writes", () => { + const f = fixture(); f.history.offer(aggregate(start)); + const report = { identity, rootScope: scope, target: hashScope("Odysseus attachment"), resource: "bytes" as const, outcome: "no-space" as const, at: start + 10_000 }; + f.advance(report.at); f.history.ownerWrite(report); + f.advance(start + 20_000); f.history.offer(aggregate(start + 20_000)); + f.advance(start + 60_000); f.history.flush(); + let pressure = f.history.read()!.metadata.state.pressures[0]!; + expect(pressure, "STORAGE_OWNER_REFUSAL_SURVIVES_BUFFER").toMatchObject({ severity: "critical", refusals: [{ target: report.target, at: report.at, successAt: null }] }); + f.advance(start + 120_000); f.history.ownerWrite({ ...report, at: start + 120_000, outcome: "success" }); + f.advance(start + 180_000); f.history.offer(aggregate(start + 180_000)); + f.advance(start + 240_000); f.history.offer(aggregate(start + 240_000)); + pressure = f.history.read()!.metadata.state.pressures[0]!; + expect(pressure.severity).toBe("low"); expect(pressure.refusals).toHaveLength(0); +}); + +test("initial history admission can recover while keeping the already selected opaque identity salt", () => { + const f = fixture(); f.db.query("DELETE FROM storage_history").run(); + f.db.exec("CREATE TRIGGER refuse_history_seed BEFORE INSERT ON storage_history BEGIN SELECT RAISE(ABORT, 'history storage unavailable'); END"); + const history = createStorageHistory(f.db, f.root, { now: () => start }), salt = history.salt; + expect(history.status().history).toBeNull(); expect(history.status().paused).toBe("history-write-unavailable"); + f.db.exec("DROP TRIGGER refuse_history_seed"); history.offer(aggregate(start)); + expect(history.read()?.metadata.state.salt, "STORAGE_INITIAL_ADMISSION_RECOVERS_STABLE_SALT").toBe(salt); + expect(history.status().paused).toBeNull(); expect(history.read()?.samples).toHaveLength(3); +}); + +test("a failed bucket stays a comparison gap after history reopens before its first successful endpoint", () => { + const f = fixture(); f.history.offer(aggregate(start)); + f.advance(start + RECENT); f.history.failure(start + RECENT); f.history.close(); + const recovered = createStorageHistory(f.db, f.root, { now: () => start + RECENT + 60_000 }); + recovered.offer(aggregate(start + RECENT + 60_000)); + expect(recovered.read()!.samples.find(row => row[1] === 0 && row[3] === start + RECENT + 60_000)?.[2], "STORAGE_DURABLE_FAILURE_BREAKS_COMPARISON").toBe(1); +}); diff --git a/packages/ui-server/tests/storage-notify.test.ts b/packages/ui-server/tests/storage-notify.test.ts index 766ab3158a..ebd4c38991 100644 --- a/packages/ui-server/tests/storage-notify.test.ts +++ b/packages/ui-server/tests/storage-notify.test.ts @@ -120,3 +120,66 @@ test("delegated principals cannot dispatch or read Storage notice receipts", asy expect(() => f.notices.receipts(agent.id, start)).toThrow(); expect(f.db.query("SELECT COUNT(*) AS count FROM storage_notice_receipts").get()).toEqual({ count: 0 }); }); + +test("native concurrent senders submit once; a SIGKILL after transport entry is terminal unknown on reopen", async () => { + const { readFileSync, existsSync } = await import("node:fs"); + const { resolve } = await import("node:path"); + const driver = resolve(import.meta.dir, "fixtures/storage-notify-process.ts"), preload = resolve(import.meta.dir, "../../../scripts/test-network-child-preload.ts"); + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + for (const mode of ["accepted", "crash"]) { + const f = fixture(); f.device("Ithaca"); f.observe(start); + const marker = join(f.root, "dispatch.txt"), children = Array.from({ length: mode === "accepted" ? 2 : 1 }, () => Bun.spawn([process.execPath, "--preload", preload, driver, f.root, mode, marker, String(start)], { stdin: "pipe", stdout: "pipe", stderr: "pipe" })); + const outputs = children.map(() => ""), readers = children.map((child, index) => (async () => { for await (const bytes of child.stdout) outputs[index] += Buffer.from(bytes).toString(); })()); + const errors = children.map(child => new Response(child.stderr).text()); + try { + const deadline = Date.now() + 5000; + while (!outputs.every(value => value.includes("READY\n")) && Date.now() < deadline) await sleep(10); + expect(outputs.every(value => value.includes("READY\n"))).toBe(true); + for (const child of children) { child.stdin.write("deliver\n"); child.stdin.end(); } + if (mode === "crash") { + while (!existsSync(marker) && Date.now() < deadline) await sleep(10); + expect(existsSync(marker), "STORAGE_PUSH_ACTUAL_DISPATCH_ENTERED").toBe(true); + expect(readFileSync(marker, "utf8")).toBe("DISPATCHED\n"); + children[0]!.kill("SIGKILL"); expect(await children[0]!.exited).not.toBe(0); + } else expect(await Promise.all(children.map(child => child.exited)), (await Promise.all(errors)).join("\n")).toEqual([0, 0]); + await Promise.all(readers); + expect(readFileSync(join(f.root, "transport-attempts.txt"), "utf8"), "STORAGE_PUSH_NATIVE_EXCLUSIVE_INTENT").toBe("attempt\n"); + expect(await f.deliver(), "STORAGE_PUSH_SIGKILL_NEVER_REPLAYS").toBe(0); + expect(f.notices.receipts(f.owner.id, start)[0]).toMatchObject({ state: mode === "crash" ? "unknown" : "accepted", attempts: 1 }); + } finally { + for (const child of children) if (child.exitCode === null) child.kill("SIGKILL"); + await Promise.all(children.map(child => child.exited)); await Promise.allSettled([...readers, ...errors]); + } + } +}); + +test("a newer cross-process collection defers old Critical delivery until its history commits", async () => { + const f = fixture(); f.device("Ithaca"); f.observe(start); + f.db.query("INSERT INTO storage_scan_leases (brain, result_at) VALUES (?, ?)").run(f.history.brain, start + 60_000); + expect(await f.deliver(), "STORAGE_PUSH_PENDING_SHARED_OBSERVATION").toBe(0); + f.observe(start + 60_000); expect(await f.deliver()).toBe(1); +}); + +test("fresh owner refusal supports Critical delivery independently of unavailable capacity", async () => { + const f = fixture(); f.device("Ithaca"); f.observe(start, 50_000); + f.at(start + 60_000); f.history.failure(start + 60_000); + f.at(start + 120_000); + const pressure = f.history.read()!.metadata.state.pressures[0]!; + f.history.ownerWrite({ identity: pressure.identity, rootScope: pressure.rootScope, target: hashScope("retained attachment"), resource: "unknown", at: start + 120_000, outcome: "no-space" }); + expect(f.history.status().paused).toBe("collection-unavailable"); + expect(await f.deliver(), "STORAGE_REFUSAL_INDEPENDENT_PUSH_EVIDENCE").toBe(1); + expect(f.notices.receipts(f.owner.id, start + 120_000)[0]?.state).toBe("accepted"); +}); + +test("retired episodes are purged and receipt rows cannot outlive subscriptions", async () => { + const f = fixture(), endpoint = f.device("Ithaca", null); + let time = start; + for (let episode = 0; episode < 8; episode++) { + f.observe(time, 5000, 4); await f.deliver(); + for (let step = 1; step <= 4; step++) { f.observe(time + step * 60_000, 50_000, 4); f.notices.sweep(time + step * 60_000); } + expect((f.db.query("SELECT COUNT(*) AS count FROM storage_notice_receipts").get() as { count: number }).count, "STORAGE_RECEIPT_ACTIVE_LATEST_BOUND").toBeLessThanOrEqual(8); + time += 5 * 60_000; + } + f.sender.unsubscribe(endpoint, f.owner.id); + expect(f.db.query("SELECT COUNT(*) AS count FROM storage_notice_receipts").get()).toEqual({ count: 0 }); +}); diff --git a/packages/ui-server/tests/storage-pressure.test.ts b/packages/ui-server/tests/storage-pressure.test.ts index 6d32e88c28..1b419cec44 100644 --- a/packages/ui-server/tests/storage-pressure.test.ts +++ b/packages/ui-server/tests/storage-pressure.test.ts @@ -16,7 +16,7 @@ test("capacity exact entry/recovery boundaries require two additional observatio const episode = value.episode; expect(episode).toBeTruthy(); value = observePressure(value, evidence(at + 60_002, 7000), at + 60_002); expect(value.recoveryAt).toBeNull(); value = observePressure(value, evidence(at + 120_002, 7001), at + 120_002); expect(value.severity).toBe("critical"); - value = observePressure(value, evidence(at + 180_001, 7001), at + 180_001); expect(value.severity).toBe("critical"); + value = observePressure(value, evidence(at + 180_001, 7001), at + 180_001); expect(value.severity, "STORAGE_RECOVERY_REQUIRES_SIXTY_SECONDS").toBe("critical"); value = observePressure(value, evidence(at + 180_002, 7001), at + 180_002); expect(value.severity, "STORAGE_PRESSURE_TWO_FRESH_OBSERVATIONS").toBe("low"); expect(value.resolvedEpisode).toBe(episode); expect(value.episode).toBeNull(); value = observePressure(value, evidence(at + 240_002, 12000), at + 240_002); expect(value.recoveryAt).toBeNull(); diff --git a/packages/ui-server/tests/storage-routes.test.ts b/packages/ui-server/tests/storage-routes.test.ts index add23c6136..8e89099e36 100644 --- a/packages/ui-server/tests/storage-routes.test.ts +++ b/packages/ui-server/tests/storage-routes.test.ts @@ -20,7 +20,7 @@ test("mounted Storage routes reject delegated and revoked principals and arbitra const agent = createPrincipal(fixture.app.db, { authMethod: "delegated", createdBy: owner.id, label: "Athena", ttlSeconds: 3600 }); const cookie = async (id: string) => (await generateSignedCookie("brain_ui_session", id, secret)).split(";")[0]!; const ownerCookie = await cookie(owner.id), agentCookie = await cookie(agent.id); - for (const [method, path] of [["GET", "/api/storage"], ["POST", "/api/storage/scans"], ["DELETE", "/api/storage/scans/unknown"], ["GET", "/api/storage/scans/unknown/entries"], ["GET", "/api/storage/scans/unknown/tree"], ["GET", "/api/storage/scans/unknown/entries/unknown/references"]]) { + for (const [method, path] of [["GET", "/api/storage/history"], ["GET", "/api/storage/guidance"], ["GET", "/api/storage"], ["POST", "/api/storage/scans"], ["DELETE", "/api/storage/scans/unknown"], ["GET", "/api/storage/scans/unknown/entries"], ["GET", "/api/storage/scans/unknown/tree"], ["GET", "/api/storage/scans/unknown/entries/unknown/references"]]) { const response = await fixture.fetch(path!, { method, headers: { cookie: agentCookie, "content-type": "application/json" }, ...(method === "POST" ? { body: "{}" } : {}) }); expect(response.status, "STORAGE_DELEGATED_DENIED").toBe(403); expect((await fixture.fetch(path!, { method, ...(method === "POST" ? { headers: { "content-type": "application/json" }, body: "{}" } : {}) })).status).toBe(401); diff --git a/packages/ui-server/tests/storage-shared-scan.test.ts b/packages/ui-server/tests/storage-shared-scan.test.ts new file mode 100644 index 0000000000..9045a96f1b --- /dev/null +++ b/packages/ui-server/tests/storage-shared-scan.test.ts @@ -0,0 +1,132 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { createUiDb } from "../src/db/client.js"; +import { createSharedStorageScan } from "../src/storage/shared-scan.js"; +import { createStorageCoordinator } from "../src/storage/coordinator.js"; +import { createStorageHistory } from "../src/storage/history-store.js"; +import { hashScope } from "../src/storage/history-model.js"; +import { processAlive, processStart } from "../src/brain/writer-gate.js"; +const cleanup: Array<() => void> = []; +afterEach(() => { for (const close of cleanup.splice(0).reverse()) close(); }); +const driver = resolve(import.meta.dir, "fixtures/storage-shared-process.ts"); +const preload = resolve(import.meta.dir, "../../../scripts/test-network-child-preload.ts"); +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "odysseus-shared-scan-")), brain = join(root, "brain"), cache = join(root, "cache"); + cleanup.push(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(brain, ".git/objects"), { recursive: true }); mkdirSync(join(brain, ".git/refs/heads"), { recursive: true }); + writeFileSync(join(brain, ".git/HEAD"), "ref: refs/heads/main\n"); + writeFileSync(join(brain, "Odysseus.md"), "# Odysseus\n\nPrepare the return to Ithaca.\n"); + const db = createUiDb(join(root, "operations.sqlite")); cleanup.push(() => db.close()); + const history = createStorageHistory(db, brain), shared = createSharedStorageScan(db, brain, cache, id => id === "odysseus-owner"); + return { root, brain, cache, db, history, shared }; +} +function child(root: string, mode: string, marker = "") { + return Bun.spawn([process.execPath, "--preload", preload, driver, root, mode, marker], { stdin: "pipe", stdout: "pipe", stderr: "pipe" }); +} +async function marker(path: string, process: ReturnType) { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && process.exitCode === null) { + try { const value = JSON.parse(readFileSync(path, "utf8")); if (Number.isSafeInteger(value.pid) && value.pid > 0 && /^\d+$/.test(value.start)) return value as { pid: number; start: string; lease: string }; } catch { /* publication not complete */ } + await sleep(10); + } + throw Error("STORAGE_SHARED_CHILD_DID_NOT_PUBLISH_PID"); +} +function running(pid: number, start: string) { + if (!processAlive(pid, start)) return false; + try { const value = readFileSync(`/proc/${pid}/stat`, "utf8"); return value.slice(value.lastIndexOf(")") + 2).split(" ")[0] !== "Z"; } catch { return false; } +} +test("two native processes share one collection and the same complete cached pages", async () => { + const f = fixture(), children = [child(f.root, "collect"), child(f.root, "collect")]; + const outputs: string[] = ["", ""], readers = children.map((value, index) => (async () => { for await (const bytes of value.stdout) outputs[index] += Buffer.from(bytes).toString("utf8"); })()); + const errors = children.map(value => new Response(value.stderr).text()); + try { + const deadline = Date.now() + 10_000; + while (!outputs.every(value => value.includes("READY\n")) && Date.now() < deadline) await sleep(10); + expect(outputs.every(value => value.includes("READY\n")), "STORAGE_SHARED_BOTH_CALLERS_READY").toBe(true); + for (const value of children) { value.stdin.write("collect\n"); value.stdin.end(); } + const exits = await Promise.all(children.map(value => value.exited)); await Promise.all(readers); + expect(exits, (await Promise.all(errors)).join("\n")).toEqual([0, 0]); + const results = outputs.map(value => JSON.parse(value.trim().split("\n").at(-1)!)); + expect(results[0].id, "STORAGE_SHARED_ONE_COLLECTION").toBe(results[1].id); + expect(results.map(value => value.collections).reduce((sum, value) => sum + value, 0)).toBe(1); + expect(results[0].names).toEqual(results[1].names); expect(results[0].names).toContain("Odysseus.md"); + expect(f.db.query("SELECT job_id, workers_json FROM storage_scan_leases").get()).toEqual({ job_id: null, workers_json: "[]" }); + } finally { + for (const value of children) if (value.exitCode === null) value.kill("SIGKILL"); + await Promise.all(children.map(value => value.exited)); await Promise.allSettled([...readers, ...errors]); + } +}); +test("uncontended and joined in-process operation requests take one scan without a transfer cache", async () => { + const f = fixture(); let collected = 0; + const coordinator = createStorageCoordinator({ brainPath: f.brain, stores: [], shared: f.shared, salt: f.history.read()!.metadata.state.salt, authorized: id => id === "odysseus-owner", onCollection() { collected++; } }); + try { + const first = coordinator.start("odysseus-owner", true); + const measurement = await coordinator.measure("odysseus-owner", new AbortController().signal); + expect(measurement?.id ?? null, "STORAGE_OPERATION_JOINS_ACTIVE_SCAN").toBe(first.activeScanId); + expect(collected).toBe(1); expect(existsSync(f.cache)).toBe(false); + } finally { await coordinator.close(); } +}); +for (const phase of ["registered", "before-register"]) test(`native collector exits after parent SIGKILL ${phase}`, async () => { + const f = fixture(), path = join(f.root, "ready.json"), process = child(f.root, phase, path); + let worker: { pid: number; start: string; lease: string } | null = null; + const output = new Response(process.stdout).text(), errors = new Response(process.stderr).text(); + try { + worker = await marker(path, process); expect(running(worker.pid, worker.start)).toBe(true); + process.kill("SIGKILL"); expect(await process.exited).not.toBe(0); + const deadline = Date.now() + 3500; + while (running(worker.pid, worker.start) && Date.now() < deadline) await sleep(20); + expect(running(worker.pid, worker.start), "STORAGE_PARENT_DEATH_REAPS_READER").toBe(false); + const lease = f.shared.acquire("odysseus-owner", randomUUID(), hashScope("Odysseus guard fixture"), 1, true)!; + await f.shared.reap(lease); expect(f.shared.finish(lease, false, true)).toBe(true); + expect(f.db.query("SELECT job_id, workers_json FROM storage_scan_leases").get()).toEqual({ job_id: null, workers_json: "[]" }); + } finally { + if (process.exitCode === null) process.kill("SIGKILL"); + await process.exited; + if (worker && processStart(worker.pid) === worker.start && running(worker.pid, worker.start)) globalThis.process.kill(worker.pid, "SIGKILL"); + await Promise.allSettled([output, errors]); + } +}); +test("lease identity, cancellation, different scopes and a shared completed result retain owner checks", async () => { + const f = fixture(), scope = hashScope("Odysseus scope"); + const owner = f.shared.acquire("odysseus-owner", randomUUID(), scope, 1, true)!; + expect(() => f.shared.acquire("odysseus-owner", randomUUID(), hashScope("Troy scope"), 1, true)).toThrow("storage-scan-other-scope-busy"); + expect(() => f.shared.acquire("agent", randomUUID(), scope, 1, true)).toThrow("storage-scan-owner-revoked"); + const joined = f.shared.acquire("odysseus-owner", randomUUID(), scope, 1, true)!; + expect(joined.id, "STORAGE_SHARED_ONE_LEASE").toBe(owner.id); expect(joined.owner).toBe(false); + f.shared.cancel(joined); expect(() => f.shared.check(owner)).toThrow("storage-scan-authority-lost"); + expect(f.shared.finish(owner, false, true)).toBe(true); + expect(f.shared.acquire("odysseus-owner", randomUUID(), scope, 0, true)).toBeNull(); +}); + +test("failed shared-cache publication keeps the previous snapshot and releases the native scan lease", async () => { + const f = fixture(); let failures = 0; + const settings = { brainPath: f.brain, stores: [], shared: f.shared, salt: f.history.salt, authorized: (id: string) => id === "odysseus-owner" }; + const first = createStorageCoordinator({ ...settings, onFailure() { failures++; } }), other = createStorageCoordinator(settings); + try { + first.start("odysseus-owner", true); await first.idle(); const before = first.status("odysseus-owner").snapshot!; expect(before).not.toBeNull(); + writeFileSync(f.cache, "The cache destination is unavailable."); + const owner = first.start("odysseus-owner", true), joined = other.start("odysseus-owner", true); + expect(joined.activeScanId).toBe(owner.activeScanId); await Promise.all([first.idle(), other.idle()]); + expect(first.status("odysseus-owner").snapshot?.id, "STORAGE_CACHE_FAILURE_RETAINS_PREVIOUS").toBe(before.id); + expect(first.status("odysseus-owner").lastFailure?.reason).toBe("collection-unavailable"); + expect(other.status("odysseus-owner").lastFailure?.reason).toBe("collection-unavailable"); expect(failures).toBe(1); + expect(f.db.query("SELECT job_id, workers_json FROM storage_scan_leases").get(), "STORAGE_FAILED_SHARED_SCAN_RELEASES_LEASE").toEqual({ job_id: null, workers_json: "[]" }); + rmSync(f.cache); other.start("odysseus-owner", true); await other.idle(); expect(other.status("odysseus-owner").snapshot).not.toBeNull(); + } finally { await Promise.all([first.close(), other.close()]); } +}); + +test("joined cancellation stops publication and leaves no reader ownership behind", async () => { + const f = fixture(), settings = { brainPath: f.brain, stores: [], shared: f.shared, salt: f.history.salt, authorized: (id: string) => id === "odysseus-owner" }; + const first = createStorageCoordinator(settings), joined = createStorageCoordinator(settings); + try { + first.start("odysseus-owner", true); const status = joined.start("odysseus-owner", true); joined.cancel("odysseus-owner", status.activeScanId!); + await Promise.all([first.idle(), joined.idle()]); + expect(first.status("odysseus-owner").snapshot, "STORAGE_JOINED_CANCEL_PREVENTS_PUBLICATION").toBeNull(); + expect(joined.status("odysseus-owner").snapshot).toBeNull(); + expect(f.db.query("SELECT job_id, workers_json FROM storage_scan_leases").get()).toEqual({ job_id: null, workers_json: "[]" }); + } finally { await Promise.all([first.close(), joined.close()]); } +}); From 20c71f68601acb3fb97c6e3a7f05db3d8ede740b Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 14:24:25 +0200 Subject: [PATCH 14/44] feat(storage): prepare proportional map and root-bound transport --- packages/ui-kit/src/index.ts | 2 + packages/ui-kit/src/storage/StorageMap.tsx | 44 ++++++++++++++++++++++ packages/ui-react/src/lib/storage-api.ts | 30 +++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 packages/ui-kit/src/storage/StorageMap.tsx create mode 100644 packages/ui-react/src/lib/storage-api.ts diff --git a/packages/ui-kit/src/index.ts b/packages/ui-kit/src/index.ts index d9e9a8f416..1a646a46ba 100644 --- a/packages/ui-kit/src/index.ts +++ b/packages/ui-kit/src/index.ts @@ -285,3 +285,5 @@ export { Overlay, type OverlayProps, type OverlayVariant, type OverlayCloseReaso export { IconButton, type IconButtonProps } from "./primitives/IconButton.js"; export { TextButton, type TextButtonProps } from "./primitives/TextButton.js"; + +export { StorageMap, storageRectangles, type StorageMapItem, type StorageMapProps } from "./storage/StorageMap.js"; diff --git a/packages/ui-kit/src/storage/StorageMap.tsx b/packages/ui-kit/src/storage/StorageMap.tsx new file mode 100644 index 0000000000..2a7651eab4 --- /dev/null +++ b/packages/ui-kit/src/storage/StorageMap.tsx @@ -0,0 +1,44 @@ +import { useId, useRef, useState, type KeyboardEvent } from "react"; +import { color, font } from "../tokens.js"; + +export interface StorageMapItem { id: string; name: string; owner: "brain" | "app" | "operator"; bytes: number; label: string; smaller?: boolean } +export interface StorageMapProps { + items: readonly StorageMapItem[]; width: number; height: number; label: string; descriptionId: string; + onOpen: (id: string) => void; onUp: () => void; +} +/** Bounded binary subdivision preserves area without a chart runtime. Input is already a disjoint union. */ +export function storageRectangles(items: readonly StorageMapItem[], width: number, height: number) { + const result: Array<{ item: StorageMapItem; x: number; y: number; width: number; height: number }> = []; + function split(rows: readonly StorageMapItem[], x: number, y: number, w: number, h: number) { + if (!rows.length || w <= 0 || h <= 0) return; + if (rows.length === 1) { result.push({item: rows[0]!,x,y,width:w,height:h}); return; } + const total=rows.reduce((sum,row)=>sum+row.bytes,0);let cut=1, first=rows[0]!.bytes; + while(cut=h) { split(rows.slice(0,cut),x,y,w*ratio,h);split(rows.slice(cut),x+w*ratio,y,w*(1-ratio),h); } + else { split(rows.slice(0,cut),x,y,w,h*ratio);split(rows.slice(cut),x,y+h*ratio,w,h*(1-ratio)); } + } + split(items.filter(item=>Number.isFinite(item.bytes)&&item.bytes>0),0,0,Math.max(0,width),Math.max(0,height)); + return result; +} +/** One roving tab stop; the equivalent list remains the complete interaction surface. */ +export function StorageMap({items,width,height,label,descriptionId,onOpen,onUp}:StorageMapProps) { + const prefix=useId(), group=useRef(null), [active,setActive]=useState(null); + const rectangles=storageRectangles(items,width,height), targets=rectangles.filter(rect=>rect.width>=44 && rect.height>=44).sort((a,b)=>a.y-b.y||a.x-b.x); + const selected=targets.some(rect=>rect.item.id===active)?active:targets[0]?.item.id; + function key(event:KeyboardEvent) { + if(event.key==="Backspace" || event.altKey&&event.key==="ArrowUp") {event.preventDefault();onUp();return;} + if(!["ArrowLeft","ArrowRight","ArrowUp","ArrowDown","Home","End"].includes(event.key)||!targets.length)return; + event.preventDefault();const current=targets.findIndex(rect=>rect.item.id===selected); + const next=event.key==="Home"?0:event.key==="End"?targets.length-1:(current+(["ArrowLeft","ArrowUp"].includes(event.key)?-1:1)+targets.length)%targets.length; + const id=targets[next]!.item.id;setActive(id);group.current?.querySelector(`[data-storage-map-index="${next}"]`)?.focus(); + } + return
+ {rectangles.map(rect=>{ + const {item}=rect,target=targets.findIndex(value=>value.item.id===item.id),labelled=rect.width>=62&&rect.height>=40; + const skin={position:"absolute" as const,left:rect.x,top:rect.y,width:rect.width,height:rect.height,boxSizing:"border-box" as const,padding:4,overflow:"hidden",border:`1px ${item.owner==="operator"||item.smaller?"dashed":"solid"} ${color.edge}`,background:item.owner==="app"?`repeating-linear-gradient(135deg, transparent, transparent 6px, var(--bk-color-edge) 6px, var(--bk-color-edge) 7px)` : "var(--bk-color-raised)",color:color.ink,textAlign:"left" as const,fontSize:12,lineHeight:1.35,borderRadius:3}; + const contents=labelled?<>{item.name}{item.label}{item.smaller?"Open list":item.owner==="brain"?"Brain":item.owner==="app"?"App":"Operator"}:null; + return target>=0?:; + })} +
; +} diff --git a/packages/ui-react/src/lib/storage-api.ts b/packages/ui-react/src/lib/storage-api.ts new file mode 100644 index 0000000000..ed0492389d --- /dev/null +++ b/packages/ui-react/src/lib/storage-api.ts @@ -0,0 +1,30 @@ +import type { StorageStatus, StorageFilter, StorageTreeFilter, StoragePage, StorageTreePage, StorageReferenceEvidence, StorageMaintenanceRow, StorageMaintenanceReceipt, StorageMaintenanceKind, StorageRemovalCapability, StorageRemovalReceipt } from "@schlessera/brain-ui-sdk/protocol"; +import type { BrainUiServices } from "../root.js"; +import { ApiRequestError } from "./api-client.js"; +/** Uses the root's account-lifetime guarded request; no ambient transport or shared cache. */ +export function storageApi(root: Pick) { + async function call(path:string,signal?:AbortSignal,method="GET",body?:unknown):Promise { + const response=await root.request(root.apiBase()+"/storage"+path,{method,signal,credentials:"include",...(body===undefined?{}:{headers:{"content-type":"application/json"},body:JSON.stringify(body)})}); + if(!response.ok)throw new ApiRequestError(`Storage request failed (${response.status}).`,response.status); + return await response.json() as T; + } + const query=(filter:StorageFilter|StorageTreeFilter)=>new URLSearchParams(Object.entries(filter).filter(([,value])=>value!==undefined).map(([key,value])=>[key,String(value)])).toString(); + return { + status:(signal?:AbortSignal)=>call("",signal), + scan:(signal?:AbortSignal)=>call("/scans",signal,"POST",{}), + cancelScan:(id:string,signal?:AbortSignal)=>call(`/scans/${encodeURIComponent(id)}`,signal,"DELETE"), + tree:(id:string,filter:StorageTreeFilter,signal?:AbortSignal)=>call(`/scans/${encodeURIComponent(id)}/tree?${query(filter)}`,signal), + files:(id:string,filter:StorageFilter,signal?:AbortSignal)=>call(`/scans/${encodeURIComponent(id)}/entries?${query(filter)}`,signal), + references:(scan:string,entry:string,signal?:AbortSignal)=>call(`/scans/${encodeURIComponent(scan)}/entries/${encodeURIComponent(entry)}/references`,signal), + maintenance:(signal?:AbortSignal)=>call<{operations:StorageMaintenanceRow[]}>("/maintenance",signal), + maintain:(id:string,operation:StorageMaintenanceKind,signal?:AbortSignal)=>call("/maintenance",signal,"POST",{id,operation}), + maintenanceReceipt:(id:string,signal?:AbortSignal)=>call(`/maintenance/${encodeURIComponent(id)}`,signal), + cancelMaintenance:(id:string,signal?:AbortSignal)=>call(`/maintenance/${encodeURIComponent(id)}`,signal,"DELETE"), + removal:(signal?:AbortSignal)=>call("/removals",signal), + review:(id:string,paths:string[],signal?:AbortSignal)=>call("/removals/reviews",signal,"POST",{id,paths}), + undo:(id:string,sourceId:string,paths:string[],signal?:AbortSignal)=>call("/removals/undo-reviews",signal,"POST",{id,sourceId,paths}), + remove:(id:string,fingerprint:string,signal?:AbortSignal)=>call("/removals",signal,"POST",{id,fingerprint}), + receipt:(id:string,signal?:AbortSignal)=>call(`/removals/${encodeURIComponent(id)}`,signal), + cancelRemoval:(id:string,signal?:AbortSignal)=>call(`/removals/${encodeURIComponent(id)}`,signal,"DELETE"), + }; +} From 20eb8570b51e85e314b406fb0e6e36f9fb1d90aa Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 15:11:05 +0200 Subject: [PATCH 15/44] CONTRACT: feat(storage): implement complete Settings inspection and removal journey --- .changeset/complete-storage-pane.md | 6 + api-report/ui-kit.txt | 34 + docs/README.md | 1 + docs/integration-contract/package-api.md | 16 + docs/storage.md | 80 ++ packages/ui-kit/src/index.ts | 2 +- packages/ui-kit/src/storage/StorageMap.tsx | 256 ++++- .../conversation/StorageTheme.stories.tsx | 117 +++ packages/ui-kit/vitest.config.ts | 3 +- packages/ui-react/README.md | 9 +- .../src/components/layout/desktop-palette.tsx | 1 + .../components/settings/settings-panel.tsx | 5 +- .../src/components/settings/storage-tab.tsx | 912 ++++++++++++++++++ .../settings/storage/storage-dialogs.tsx | 423 ++++++++ .../settings/storage/storage-model.ts | 89 ++ .../settings/storage/storage-observations.tsx | 146 +++ .../settings/storage/use-storage.ts | 649 +++++++++++++ packages/ui-react/src/lib/storage-api.ts | 144 ++- packages/ui-react/src/stores/ui-state.ts | 7 +- .../browser/settings-structure.layout.tsx | 10 +- .../tests/browser/storage-pane.layout.tsx | 339 +++++++ .../tests/browser/storage-routes.layout.tsx | 118 +++ .../tests/browser/storage-server-command.ts | 96 ++ .../tests/fixtures/storage-browser-server.ts | 65 ++ .../ui-react/tests/fixtures/storage-view.ts | 589 +++++++++++ 25 files changed, 4050 insertions(+), 67 deletions(-) create mode 100644 .changeset/complete-storage-pane.md create mode 100644 docs/storage.md create mode 100644 packages/ui-kit/stories/conversation/StorageTheme.stories.tsx create mode 100644 packages/ui-react/src/components/settings/storage-tab.tsx create mode 100644 packages/ui-react/src/components/settings/storage/storage-dialogs.tsx create mode 100644 packages/ui-react/src/components/settings/storage/storage-model.ts create mode 100644 packages/ui-react/src/components/settings/storage/storage-observations.tsx create mode 100644 packages/ui-react/src/components/settings/storage/use-storage.ts create mode 100644 packages/ui-react/tests/browser/storage-pane.layout.tsx create mode 100644 packages/ui-react/tests/browser/storage-routes.layout.tsx create mode 100644 packages/ui-react/tests/browser/storage-server-command.ts create mode 100644 packages/ui-react/tests/fixtures/storage-browser-server.ts create mode 100644 packages/ui-react/tests/fixtures/storage-view.ts diff --git a/.changeset/complete-storage-pane.md b/.changeset/complete-storage-pane.md new file mode 100644 index 0000000000..4b75e42c32 --- /dev/null +++ b/.changeset/complete-storage-pane.md @@ -0,0 +1,6 @@ +--- +"@schlessera/brain-ui-react": minor +"@schlessera/brain-ui-kit": minor +--- + +Add the complete Storage Settings pane: separate capacity observations, owned-store maps and equivalent lists, large-file inspection, owner maintenance, fresh attachment removal review, durable per-file outcomes and retained Undo. Preserve root/account boundaries, Settings navigation, current-file versions and explicit unknown evidence. diff --git a/api-report/ui-kit.txt b/api-report/ui-kit.txt index 1e85b0086f..9a600c9694 100644 --- a/api-report/ui-kit.txt +++ b/api-report/ui-kit.txt @@ -239,6 +239,9 @@ export "." (./src/index.ts) type StepListProps type StepListVariant type StepState + StorageMap + type StorageMapItem + type StorageMapProps type StreamPhase StreamingAnswer type StreamingAnswerProps @@ -288,6 +291,7 @@ export "." (./src/index.ts) effectLineCount followUpLabel font + storageRectangles z export "./brand" (./src/brand.ts) @@ -2085,6 +2089,28 @@ signatures (every public export and the types it is made of) export type StepListVariant = "numbered" | "checklist" | "progress"; StepState (packages/ui-kit/src/types.ts) export type StepState = "done" | "current" | "todo"; + StorageMap (packages/ui-kit/src/storage/StorageMap.tsx) + export function StorageMap({ items, width, height, label, descriptionId, onOpen, onUp }: StorageMapProps): import("react").JSX.Element; + StorageMapItem (packages/ui-kit/src/storage/StorageMap.tsx) + export interface StorageMapItem { + id: string; + name: string; + owner: "brain" | "app" | "operator"; + bytes: number; + label: string; + smaller?: boolean; + interactive?: boolean; + } + StorageMapProps (packages/ui-kit/src/storage/StorageMap.tsx) + export interface StorageMapProps { + items: readonly StorageMapItem[]; + width: number; + height: number; + label: string; + descriptionId: string; + onOpen: (id: string) => void; + onUp: () => void; + } StreamPhase (packages/ui-kit/src/types.ts) export type StreamPhase = "thinking" | "searching" | "reading" | "fetching" | "writing"; StreamingAnswer (packages/ui-kit/src/conversation/StreamingAnswer.tsx) @@ -2387,6 +2413,14 @@ signatures (every public export and the types it is made of) }; highlyRestrictive (packages/render-template/src/links.ts) recorded in api-report/render-template.txt refusalSentence (packages/render-template/src/links.ts) recorded in api-report/render-template.txt + storageRectangles (packages/ui-kit/src/storage/StorageMap.tsx) + export function storageRectangles(items: readonly StorageMapItem[], width: number, height: number): { + height: number; + item: StorageMapItem; + width: number; + x: number; + y: number; + }[]; z (packages/ui-kit/src/tokens.ts) const z: { readonly banner: "var(--bk-z-banner)"; diff --git a/docs/README.md b/docs/README.md index 4ca2489d65..ed1d9f2451 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ question. This Markdown index is also the navigation source for public docs. | Guide | Use it to | | --- | --- | | [Modules](modules.md) | Enable domain workflows or author a local module; follow links to each module's full reference. | +| [Storage](storage.md) | Inspect capacity, find large files, run owner-supported maintenance, and review recoverable attachment removal. | | [Media](media.md) | Decide which assets to keep in git and how large files behave during sync. | | [Evaluating search](evaluating-search.md) | Build a query set and measure retrieval without mistaking an invalid set for a useful score. | | [Hosting](hosting/README.md) | Back up a local brain and understand the separate, not-yet-published hosting-template path. | diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index f65238b26b..0556183c48 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3283,3 +3283,19 @@ and bounds are defined by the [removal HTTP contract](../http-api.md#recoverable `StorageRemovalReview`, `StorageRemovalItem`, `StorageRemovalReceipt` and `StorageRemovalCapability`. They contain relative paths and opaque identities, never private recovery roots, executable choices or credentials. + +### Storage presentation (additive, #2013) + +`@schlessera/brain-ui-kit` exports `StorageMap`, `StorageMapItem` and +`StorageMapProps`. Items supply opaque identities, names, ownership +(`brain`, `app`, `operator`), known disjoint bytes, a size/share label and optional +`smaller`/`interactive` flags. Props supply those items, CSS-pixel width/height, +an accessible group label and description ID, and `onOpen(id)`/`onUp()` callbacks. +The component performs no I/O; callers own the bounded, deduplicated inventory +and equivalent complete list. Unknown and nonpositive sizes have no tile area; +small tiles have no activation target. It preserves one roving tab stop. + +The ui-react Settings selection accepts `storage`. `openSettings("storage")` +and the existing palette open the same lazy pane after Modules; Models remains +the default. This adds no main destination or keyboard shortcut. Requests and +local selections remain scoped to the current root and authentication lifetime. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000000..046d3d9dab --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,80 @@ +# Inspect and manage Storage + +Open **Settings → Storage**, after Modules, or choose **Jump to → Storage** +in the command palette. Storage is available to the human owner. The pane +includes Brain files, observable app stores and configured backups; it does +not infer remote usage or unobserved host storage. + +## Read the measurements + +Choose **Measure** for the first observation and **Refresh** to request another. +Opening the pane does not start a scan. During collection the previous result +stays visible; **Show new measurement** adopts the new result and clears old +selections. Measurements older than a minute say **Out of date**. Partial results +show known subtotals and their missing evidence; unknown is never zero. + +Each capacity has its own bar and write-target caption. Available filesystem +space is not a quota guarantee. Shared or aliased capacities are not added, +and memory-backed temporary storage is labelled separately from disk. + +**On disk (estimate)** is the default. It uses reported allocated bytes and +counts a known hard-linked object once; it does not promise exclusive or +reclaimable space. **File size** uses logical file bytes. Sizes use decimal +units: 1 MB means 1,000,000 bytes. Inspect a file for exact bytes. + +**All**, **Brain** and **App** are views over the inventory. They are not values +to add. Open map tiles or list rows to explore folders, then use breadcrumbs to +go back. The list supports name/size sorting, search and pages of 200 entries. +**Use list only** provides the same inspection and navigation without the map. +Small tiles remain accessible through the list; unknown sizes have no tile area. + +## Find and review large files + +**Find large files** starts with PDF filenames strictly larger than 5 MB in Brain +files. The extension comparison ignores case; it makes no claim about MIME type +or whether a document is safe. Exactly 5,000,000 bytes does not match. Entry totals +may count multiple names of a shared object; the separate object union does not. + +1. **Inspect** a file to see its current Git state, reference evidence, eligibility + and recovery coverage. Inspection does not reserve a removal receipt. +2. Select eligible files and choose **Review selected files**. Protected, shared, + ignored or uncovered files remain unselectable. Unknown references do not mean + there are no links. +3. Read the fresh review, including exact versions, broken links and unchecked + consumers. **Keep files** dismisses it without removal. Changed or expired + reviews require fresh inspection and review. +4. Confirm the eligible selection. Originals must be retained before removal. + The result lists each file independently, including changed, protected, + cancelled and unconfirmed outcomes. **Cancel remaining files** stops new + admissions; an already acknowledged file can remain removed. + +A request with an unconfirmed response keeps its original identity. Use +**Check original receipt**; do not submit another removal to guess the outcome. +A refreshed or reopened pane can recover the latest receipt for the same owner. + +**Undo removal** is offered only for a confirmed removal whose original is still +retained. Undo has its own fresh review. It never replaces a file that has since +appeared at the target, and it is subject to existing journal retention. Keep +backups for recovery beyond that retention. Hosts without qualified recovery +still allow inspection, Preview, Open in Files and copying the visible file list. + +## Maintenance and storage deltas + +Maintenance uses existing owners: packing Git, pruning eligible scratch files, +and cleaning eligible upload staging. Each row explains its effect, headroom +and latest receipt. Packing preserves refs, history and reflogs; it can need +additional temporary space. Scratch keeps pinned answer files and the existing +seven-day / 1,073,741,824-byte policy. Staging cleanup respects active uploads. + +The result separates working files, retained journal bytes, Git storage and +filesystem availability. Removing a file may increase journal use and leave Git +history or backups unchanged. Other writers can affect before/after values. +Index refresh, a deletion commit and remote sync are separate outcomes; removal +does not automatically commit or sync. + +Scans and operations have finite bounds. A scan can stop with partial evidence. +Removal admits at most 32 paths, 64 MiB per file and 128 MiB per operation; +review lasts ten minutes. Durable receipt capacity can refuse new operations +while existing receipts remain readable. The pane never erases recovery data +or expands retention policy to make room. If protected stores dominate, capacity +provisioning or the store owner's existing controls may be needed. diff --git a/packages/ui-kit/src/index.ts b/packages/ui-kit/src/index.ts index 1a646a46ba..7b2f1d8b3c 100644 --- a/packages/ui-kit/src/index.ts +++ b/packages/ui-kit/src/index.ts @@ -286,4 +286,4 @@ export { Overlay, type OverlayProps, type OverlayVariant, type OverlayCloseReaso export { IconButton, type IconButtonProps } from "./primitives/IconButton.js"; export { TextButton, type TextButtonProps } from "./primitives/TextButton.js"; -export { StorageMap, storageRectangles, type StorageMapItem, type StorageMapProps } from "./storage/StorageMap.js"; +export { StorageMap, type StorageMapItem, type StorageMapProps } from "./storage/StorageMap.js"; diff --git a/packages/ui-kit/src/storage/StorageMap.tsx b/packages/ui-kit/src/storage/StorageMap.tsx index 2a7651eab4..261436358a 100644 --- a/packages/ui-kit/src/storage/StorageMap.tsx +++ b/packages/ui-kit/src/storage/StorageMap.tsx @@ -1,44 +1,238 @@ import { useId, useRef, useState, type KeyboardEvent } from "react"; import { color, font } from "../tokens.js"; -export interface StorageMapItem { id: string; name: string; owner: "brain" | "app" | "operator"; bytes: number; label: string; smaller?: boolean } +export interface StorageMapItem { + id: string; + name: string; + owner: "brain" | "app" | "operator"; + bytes: number; + label: string; + smaller?: boolean; + interactive?: boolean; +} export interface StorageMapProps { - items: readonly StorageMapItem[]; width: number; height: number; label: string; descriptionId: string; - onOpen: (id: string) => void; onUp: () => void; + items: readonly StorageMapItem[]; + width: number; + height: number; + label: string; + descriptionId: string; + onOpen: (id: string) => void; + onUp: () => void; } /** Bounded binary subdivision preserves area without a chart runtime. Input is already a disjoint union. */ -export function storageRectangles(items: readonly StorageMapItem[], width: number, height: number) { - const result: Array<{ item: StorageMapItem; x: number; y: number; width: number; height: number }> = []; - function split(rows: readonly StorageMapItem[], x: number, y: number, w: number, h: number) { +function storageRectangles( + items: readonly StorageMapItem[], + width: number, + height: number +) { + const result: Array<{ + item: StorageMapItem; + x: number; + y: number; + width: number; + height: number; + }> = []; + function split( + rows: readonly StorageMapItem[], + x: number, + y: number, + w: number, + h: number + ) { if (!rows.length || w <= 0 || h <= 0) return; - if (rows.length === 1) { result.push({item: rows[0]!,x,y,width:w,height:h}); return; } - const total=rows.reduce((sum,row)=>sum+row.bytes,0);let cut=1, first=rows[0]!.bytes; - while(cut=h) { split(rows.slice(0,cut),x,y,w*ratio,h);split(rows.slice(cut),x+w*ratio,y,w*(1-ratio),h); } - else { split(rows.slice(0,cut),x,y,w,h*ratio);split(rows.slice(cut),x,y+h*ratio,w,h*(1-ratio)); } + if (rows.length === 1) { + result.push({ item: rows[0]!, x, y, width: w, height: h }); + return; + } + const total = rows.reduce((sum, row) => sum + row.bytes, 0); + let cut = 1, + first = rows[0]!.bytes; + while (cut < rows.length - 1 && first + rows[cut]!.bytes <= total / 2) + first += rows[cut++]!.bytes; + const ratio = first / total; + if (w >= h) { + split(rows.slice(0, cut), x, y, w * ratio, h); + split(rows.slice(cut), x + w * ratio, y, w * (1 - ratio), h); + } else { + split(rows.slice(0, cut), x, y, w, h * ratio); + split(rows.slice(cut), x, y + h * ratio, w, h * (1 - ratio)); + } } - split(items.filter(item=>Number.isFinite(item.bytes)&&item.bytes>0),0,0,Math.max(0,width),Math.max(0,height)); + split( + items.filter((item) => Number.isFinite(item.bytes) && item.bytes > 0), + 0, + 0, + Math.max(0, width), + Math.max(0, height) + ); return result; } /** One roving tab stop; the equivalent list remains the complete interaction surface. */ -export function StorageMap({items,width,height,label,descriptionId,onOpen,onUp}:StorageMapProps) { - const prefix=useId(), group=useRef(null), [active,setActive]=useState(null); - const rectangles=storageRectangles(items,width,height), targets=rectangles.filter(rect=>rect.width>=44 && rect.height>=44).sort((a,b)=>a.y-b.y||a.x-b.x); - const selected=targets.some(rect=>rect.item.id===active)?active:targets[0]?.item.id; - function key(event:KeyboardEvent) { - if(event.key==="Backspace" || event.altKey&&event.key==="ArrowUp") {event.preventDefault();onUp();return;} - if(!["ArrowLeft","ArrowRight","ArrowUp","ArrowDown","Home","End"].includes(event.key)||!targets.length)return; - event.preventDefault();const current=targets.findIndex(rect=>rect.item.id===selected); - const next=event.key==="Home"?0:event.key==="End"?targets.length-1:(current+(["ArrowLeft","ArrowUp"].includes(event.key)?-1:1)+targets.length)%targets.length; - const id=targets[next]!.item.id;setActive(id);group.current?.querySelector(`[data-storage-map-index="${next}"]`)?.focus(); +export function StorageMap({ + items, + width, + height, + label, + descriptionId, + onOpen, + onUp, +}: StorageMapProps) { + const prefix = useId(), + group = useRef(null), + [active, setActive] = useState(null); + const rectangles = storageRectangles(items, width, height), + targets = rectangles + .filter( + (rect) => + rect.item.interactive !== false && + rect.width >= 44 && + rect.height >= 44 + ) + .sort((a, b) => a.y - b.y || a.x - b.x); + const selected = targets.some((rect) => rect.item.id === active) + ? active + : targets[0]?.item.id; + function key(event: KeyboardEvent) { + if ( + event.key === "Backspace" || + (event.altKey && event.key === "ArrowUp") + ) { + event.preventDefault(); + onUp(); + return; + } + if ( + ![ + "ArrowLeft", + "ArrowRight", + "ArrowUp", + "ArrowDown", + "Home", + "End", + ].includes(event.key) || + !targets.length + ) + return; + event.preventDefault(); + const current = targets.findIndex((rect) => rect.item.id === selected); + const next = + event.key === "Home" + ? 0 + : event.key === "End" + ? targets.length - 1 + : (current + + (["ArrowLeft", "ArrowUp"].includes(event.key) ? -1 : 1) + + targets.length) % + targets.length; + const id = targets[next]!.item.id; + setActive(id); + group.current + ?.querySelector(`[data-storage-map-index="${next}"]`) + ?.focus(); } - return
- {rectangles.map(rect=>{ - const {item}=rect,target=targets.findIndex(value=>value.item.id===item.id),labelled=rect.width>=62&&rect.height>=40; - const skin={position:"absolute" as const,left:rect.x,top:rect.y,width:rect.width,height:rect.height,boxSizing:"border-box" as const,padding:4,overflow:"hidden",border:`1px ${item.owner==="operator"||item.smaller?"dashed":"solid"} ${color.edge}`,background:item.owner==="app"?`repeating-linear-gradient(135deg, transparent, transparent 6px, var(--bk-color-edge) 6px, var(--bk-color-edge) 7px)` : "var(--bk-color-raised)",color:color.ink,textAlign:"left" as const,fontSize:12,lineHeight:1.35,borderRadius:3}; - const contents=labelled?<>{item.name}{item.label}{item.smaller?"Open list":item.owner==="brain"?"Brain":item.owner==="app"?"App":"Operator"}:null; - return target>=0?:; - })} -
; + return ( +
+ {rectangles.map((rect) => { + const { item } = rect, + target = targets.findIndex((value) => value.item.id === item.id), + labelled = rect.width >= 62 && rect.height >= 40; + const skin = { + position: "absolute" as const, + left: rect.x, + top: rect.y, + width: rect.width, + height: rect.height, + boxSizing: "border-box" as const, + padding: 4, + overflow: "hidden", + border: `1px ${ + item.owner === "operator" || item.smaller ? "dashed" : "solid" + } ${color.edge}`, + background: + item.owner === "app" + ? `repeating-linear-gradient(135deg, transparent, transparent 6px, var(--bk-color-edge) 6px, var(--bk-color-edge) 7px)` + : "var(--bk-color-raised)", + color: color.ink, + textAlign: "left" as const, + fontSize: 12, + lineHeight: 1.35, + borderRadius: 3, + }; + const contents = labelled ? ( + <> + + {item.name} + + + {item.label} + + {rect.height >= 60 && ( + + {item.smaller + ? "Open list" + : item.owner === "brain" + ? "Brain" + : item.owner === "app" + ? "App" + : "Operator"} + + )} + + ) : null; + return target >= 0 ? ( + + ) : ( + + ); + })} +
+ ); } diff --git a/packages/ui-kit/stories/conversation/StorageTheme.stories.tsx b/packages/ui-kit/stories/conversation/StorageTheme.stories.tsx new file mode 100644 index 0000000000..15d143c977 --- /dev/null +++ b/packages/ui-kit/stories/conversation/StorageTheme.stories.tsx @@ -0,0 +1,117 @@ +import preview from "#.storybook/preview"; +import { useEffect, useState } from "react"; +import { expect, waitFor, within } from "storybook/test"; +import { BrainUiProvider } from "../../../ui-react/src/root-context.js"; +import { createBrainUiRoot } from "../../../ui-react/src/root.js"; +import { StorageTab } from "../../../ui-react/src/components/settings/storage-tab.js"; +import { + storageViewFixture, + type StorageScene, +} from "../../../ui-react/tests/fixtures/storage-view.js"; +import { + ConsumerDocument, + consumerViewport, + painted, + accessible, +} from "./_consumer-parity.js"; +function StorageTheme({ scene }: { width: number; scene: StorageScene }) { + const [root] = useState(() => + createBrainUiRoot({ + storage: null, + request: storageViewFixture(scene).request, + }) + ); + useEffect(() => () => root.dispose(), [root]); + return ( + +
+ + + +
+ + ); +} +const meta = preview.meta({ + title: "Settings/Storage", + component: StorageTheme, + parameters: { layout: "fullscreen" }, + beforeEach: consumerViewport, + args: { width: 320, scene: "dense" as StorageScene }, +}); +const overview = async ({ + args, + globals, +}: { + args: { width: number }; + globals: { theme?: string }; +}) => { + await painted(args.width, globals.theme); + const screen = within(document.body); + await screen.findByRole("group", { name: /Storage map/ }); + await accessible(document.querySelector("[data-storage-pane]")!); +}; +export const Phone = meta.story({ play: overview }); +export const Desktop = meta.story({ args: { width: 1280 }, play: overview }); +export const SeparateTemporary = meta.story({ + args: { scene: "separate" }, + play: overview, +}); +export const Partial = meta.story({ + args: { scene: "partial" }, + play: overview, +}); +export const Empty = meta.story({ args: { scene: "empty" } }); +export const Unsupported = meta.story({ args: { scene: "unsupported" } }); +export const NeverMeasured = meta.story({ args: { scene: "never" } }); +export const RemovalAndUndo = meta.story({ + play: async ({ args, globals, userEvent }) => { + await overview({ args, globals }); + const screen = within(document.body); + await userEvent.click( + screen.getByRole("button", { name: "Find large files", exact: true }) + ); + for (const name of ["Ithaca-harbor.pdf", "Troy-charts.PDF"]) { + await userEvent.click( + await screen.findByRole("button", { + name: `Inspect ${name}`, + exact: true, + }) + ); + const select = await screen.findByRole("button", { + name: "Select file", + exact: true, + }); + await waitFor(() => + expect(select).not.toHaveAttribute("aria-disabled", "true") + ); + await userEvent.click(select); + } + await userEvent.click( + screen.getByRole("button", { name: "Review selected files", exact: true }) + ); + await userEvent.click( + await screen.findByRole("button", { name: "Remove 2 files", exact: true }) + ); + await screen.findByText("1 removed · 0 restored · 1 other outcomes"); + await accessible(screen.getByRole("dialog")); + await userEvent.click( + screen.getByRole("button", { + name: "Undo removal of attachments/Ithaca-harbor.pdf", + exact: true, + }) + ); + await userEvent.click( + await screen.findByRole("button", { name: "Restore 1 file", exact: true }) + ); + await screen.findByText("0 removed · 1 restored · 0 other outcomes"); + await accessible(screen.getByRole("dialog")); + }, +}); diff --git a/packages/ui-kit/vitest.config.ts b/packages/ui-kit/vitest.config.ts index c3e14b76c9..d5b3180d56 100644 --- a/packages/ui-kit/vitest.config.ts +++ b/packages/ui-kit/vitest.config.ts @@ -1,4 +1,5 @@ import { consumerParityViewport } from "./tests/visual/consumer-parity-viewport.ts"; +import { storageServer } from "../ui-react/tests/browser/storage-server-command.ts"; import { policyServer } from "../ui-react/tests/browser/policy-server-command.ts"; import { hygieneLoop } from "../ui-react/tests/browser/hygiene-loop-command.ts"; import { hygieneServer } from "../ui-react/tests/browser/hygiene-server-command.ts"; @@ -131,7 +132,7 @@ export default mergeConfig( browser: { enabled: true, screenshotFailures: false, - commands: { buttonCapture, overlayMouse, dictationMotion, policyServer, hygieneLoop, hygieneServer, dispatchActions, sheetInput, focusCapture, formViewport, formConsumerStyles, rankFooterFonts, codeHighlightFailure, codeHighlightFailureCount, offlineScene, coldCapture, shareReview, transcriptionHttp, startRequestLog, requestLog, grantMicrophone }, + commands: { buttonCapture, overlayMouse, dictationMotion, policyServer, storageServer, hygieneLoop, hygieneServer, dispatchActions, sheetInput, focusCapture, formViewport, formConsumerStyles, rankFooterFonts, codeHighlightFailure, codeHighlightFailureCount, offlineScene, coldCapture, shareReview, transcriptionHttp, startRequestLog, requestLog, grantMicrophone }, // The fake microphone plays the generated 10-second fixture // (#1016). Chromium reads the file at launch, so it is written // when this config loads; the fake UI grants the permission. diff --git a/packages/ui-react/README.md b/packages/ui-react/README.md index c9c764603d..b6a590d30f 100644 --- a/packages/ui-react/README.md +++ b/packages/ui-react/README.md @@ -700,8 +700,8 @@ offers **Refresh and retry**. Sync and reset use immediate layout updates. ## Settings organization -Settings exposes the same seven panes on phone, tablet and desktop: Appearance -& input, Models, Accounts, Skills, Modules, Security, and Devices & agents. +Settings exposes the same eight panes on phone, tablet and desktop: Appearance +& input, Models, Accounts, Skills, Modules, Storage, Security, and Devices & agents. Theme and the single-key shortcut switch apply only on this device and live in Appearance & input. Accounts holds provider sign-in and web-search settings; Security holds passkeys, always-allowed tool revocation and sign-out. @@ -713,3 +713,8 @@ selection survives resize. Module drafts and pending credentials keep their existing navigation and dismissal protections. Close and Escape return focus to the opener, including after a resize; choosing another destination preserves that destination's focus. + +Storage presents bounded inventory, per-capacity availability, a proportional map +and equivalent list, current file inspection, owner maintenance, and recoverable +attachment removal. See the [Storage guide](../../docs/storage.md) for the +measurement basis, limits, review flow and Undo conditions. diff --git a/packages/ui-react/src/components/layout/desktop-palette.tsx b/packages/ui-react/src/components/layout/desktop-palette.tsx index b174c4fb56..4f4ce50306 100644 --- a/packages/ui-react/src/components/layout/desktop-palette.tsx +++ b/packages/ui-react/src/components/layout/desktop-palette.tsx @@ -108,6 +108,7 @@ export function DesktopPalette() { ...(finePointer ? { shortcut: d.key } : {}), onClick: run(d.go), })), + { icon: "ledger", label: "Storage", tone: "neutral", onClick: run(() => root.stores.ui.getState().openSettings("storage")) }, { icon: "graph", label: "Graph", tone: "purple", onClick: run(routes.graph) }, { icon: "compose", label: "New chat", tone: "amber", onClick: run(routes.inChat(clearMessages)) }, ]; diff --git a/packages/ui-react/src/components/settings/settings-panel.tsx b/packages/ui-react/src/components/settings/settings-panel.tsx index 38df6f49f0..74c2549670 100644 --- a/packages/ui-react/src/components/settings/settings-panel.tsx +++ b/packages/ui-react/src/components/settings/settings-panel.tsx @@ -1,5 +1,5 @@ import { Fragment, Suspense, useEffect, useLayoutEffect, useId, useMemo, useRef, useState, type CSSProperties, type KeyboardEvent } from "react"; -import { ChevronRight, KeyRound, Laptop, Puzzle, SlidersHorizontal, UserRound, Workflow } from "lucide-react"; +import { ChevronRight, HardDrive, KeyRound, Laptop, Puzzle, SlidersHorizontal, UserRound, Workflow } from "lucide-react"; import { useUIStore, type SettingsTab } from "../../stores/ui-store.js"; import { SlidePanel } from "../layout/slide-panel.js"; import { cn } from "../../lib/utils.js"; @@ -21,6 +21,7 @@ const PasskeyTab = lazyChunk(() => import("./passkey-tab.js").then((m) => ({ def const DevicesAgentsTab = lazyChunk(() => import("./devices-agents-tab.js").then((m) => ({ default: m.DevicesAgentsTab }))); const SkillsTab = lazyChunk(() => import("./skills-tab.js").then((m) => ({ default: m.SkillsTab }))); const ModulesTab = lazyChunk(() => import("./modules-tab.js").then((m) => ({ default: m.ModulesTab }))); +const StorageTab = lazyChunk(() => import("./storage-tab.js").then((m) => ({ default: m.StorageTab }))); /** One logical inventory at every width; the groups label the desktop column. */ const SECTIONS: Array<{ id: SettingsTab; label: string; meta: string; group: string; icon: IconName; glyph: typeof KeyRound }> = [ @@ -29,6 +30,7 @@ const SECTIONS: Array<{ id: SettingsTab; label: string; meta: string; group: str { id: "accounts", label: "Accounts", meta: "providers", group: "This brain", icon: "wallet", glyph: UserRound }, { id: "skills", label: "Skills", meta: "catalog", group: "This brain", icon: "capability", glyph: Puzzle }, { id: "modules", label: "Modules", meta: "workflows", group: "This brain", icon: "steps", glyph: Workflow }, + { id: "storage", label: "Storage", meta: "capacity", group: "This brain", icon: "ledger", glyph: HardDrive }, { id: "security", label: "Security", meta: "passkeys", group: "Access", icon: "passkey", glyph: KeyRound }, { id: "devices", label: "Devices & agents", meta: "credentials", group: "Access", icon: "agent", glyph: Laptop }, ]; @@ -236,6 +238,7 @@ function SectionBody({ tab, open, moduleSession }: { tab: SettingsTab; open: boo : tab === "accounts" ? : tab === "skills" ? : tab === "modules" ? + : tab === "storage" ? : tab === "security" ? : } ; diff --git a/packages/ui-react/src/components/settings/storage-tab.tsx b/packages/ui-react/src/components/settings/storage-tab.tsx new file mode 100644 index 0000000000..157353fd2b --- /dev/null +++ b/packages/ui-react/src/components/settings/storage-tab.tsx @@ -0,0 +1,912 @@ +import { + useEffect, + useId, + useMemo, + useRef, + useState, + useSyncExternalStore, +} from "react"; +import { Button, StorageMap } from "@schlessera/brain-ui-kit"; +import type { StorageFileEntry } from "@schlessera/brain-ui-sdk/protocol"; +import { useBrainUiRoot } from "../../root-context.js"; +import type { BrainUiRoot } from "../../root.js"; +import { useMediaQuery } from "../../hooks/use-media-query.js"; +import { + bytes, + exactBytes, + humanReason, + mapItems, + ownerName, + percent, + time, +} from "./storage/storage-model.js"; +import { + StorageCapacities, + StorageDelta, +} from "./storage/storage-observations.js"; +import { StorageDialogs } from "./storage/storage-dialogs.js"; +import { running, useStorage } from "./storage/use-storage.js"; +const touch = { minHeight: 44 }; +/** Account and root changes discard pending selections and local response ownership synchronously. */ +export function StorageTab({ active }: { active: boolean }) { + const root = useBrainUiRoot(), + rootKey = useMemo(() => crypto.randomUUID(), [root]); + const epoch = useSyncExternalStore( + root.authLock.state.subscribe, + () => `${root.authLock.epoch()}:${root.authLock.state.getState().phase}`, + () => `${root.authLock.epoch()}:${root.authLock.state.getState().phase}` + ); + return active && root.authLock.state.getState().phase === "active" ? ( + + ) : null; +} +function StorageSession({ root }: { root: BrainUiRoot }) { + const view = useStorage(root), + phone = useMediaQuery("(max-width: 599px)"), + id = useId(); + const heading = useRef(null), + breadcrumb = useRef(null), + listHeading = useRef(null), + container = useRef(null), + trigger = useRef(null), + pendingFocus = useRef(null); + const [width, setWidth] = useState(296), + [now, setNow] = useState(Date.now()), + [listOnly, setListOnly] = useState(false); + useEffect(() => { + const element = container.current; + if (!element) return; + const observer = new ResizeObserver(() => + setWidth(Math.max(1, element.clientWidth)) + ); + observer.observe(element); + setWidth(element.clientWidth); + return () => observer.disconnect(); + }, []); + useEffect(() => { + if (!view.snapshot) return; + const timer = setTimeout( + () => setNow(Date.now()), + Math.max(1, view.snapshot.staleAfter - Date.now()) + ); + return () => clearTimeout(timer); + }, [view.snapshot]); + useEffect(() => { + if ( + view.loading || + !pendingFocus.current || + view.tree?.parentId !== (view.parent ?? null) + ) + return; + const target = pendingFocus.current; + pendingFocus.current = null; + if (target === "breadcrumb") breadcrumb.current?.focus(); + else + ( + container.current?.querySelector( + `[data-storage-open="${target}"] [role="button"]` + ) ?? breadcrumb.current + )?.focus(); + }, [view.loading, view.tree, view.parent]); + const snapshot = view.snapshot, + tree = view.tree, + page = view.filtering ? view.files : tree, + total = page?.totals[view.metric] ?? null; + const tiles = view.chart + ? mapItems(view.chart, view.metric, width < 600) + : [], + shown = new Set( + tiles.filter((tile) => !tile.smaller).map((tile) => tile.id) + ); + const nodes = (tree?.nodes ?? []).filter( + (node) => !view.smaller || !shown.has(node.id) + ); + function open(entry: StorageFileEntry) { + trigger.current = + document.activeElement instanceof HTMLElement + ? document.activeElement + : null; + void view.inspect(entry); + } + function drill(nodeId: string) { + pendingFocus.current = "breadcrumb"; + view.setParent(nodeId); + view.setSearch(""); + view.setCursor(undefined); + } + function goUp(target: string | null) { + const index = + view.tree?.breadcrumbs.findIndex((crumb) => crumb.id === target) ?? -1; + pendingFocus.current = + view.tree?.breadcrumbs[index + 1]?.id ?? "breadcrumb"; + view.setParent(target ?? undefined); + view.setSearch(""); + view.setCursor(undefined); + } + const returning = () => + trigger.current?.isConnected ? trigger.current : heading.current; + const unavailable = !view.capability?.available, + operationPending = + running(view.receipt?.state) || view.receipt?.state === "unknown"; + const selectReason = (entry: StorageFileEntry) => { + const file = view.evidence[entry.id]; + return unavailable + ? "This host can't remove attachments yet" + : file?.facts?.eligible + ? "Eligible after fresh review" + : file?.facts + ? humanReason(file.facts.reason) + : "Inspect to verify eligibility and Undo"; + }; + function inspectControl(entry: StorageFileEntry) { + return ( + + + ); +} diff --git a/packages/ui-react/src/components/settings/storage-tab.tsx b/packages/ui-react/src/components/settings/storage-tab.tsx index 5263568b67..07ed3058d1 100644 --- a/packages/ui-react/src/components/settings/storage-tab.tsx +++ b/packages/ui-react/src/components/settings/storage-tab.tsx @@ -25,6 +25,8 @@ import { StorageDelta, } from "./storage/storage-observations.js"; import { StorageDialogs } from "./storage/storage-dialogs.js"; +import { useStorageGuidance } from "./storage/use-storage-guidance.js"; +import { StorageGuidance } from "./storage/storage-guidance.js"; import { running, useStorage } from "./storage/use-storage.js"; const touch = { minHeight: 44 }; /** Account and root changes discard pending selections and local response ownership synchronously. */ @@ -41,6 +43,7 @@ export function StorageTab({ active }: { active: boolean }) { ) : null; } function StorageSession({ root }: { root: BrainUiRoot }) { + const guidance = useStorageGuidance(root, true); const view = useStorage(root), phone = useMediaQuery("(max-width: 599px)"), id = useId(); @@ -88,6 +91,8 @@ function StorageSession({ root }: { root: BrainUiRoot }) { ) ?? breadcrumb.current )?.focus(); }, [view.loading, view.tree, view.parent]); + const observed = Math.max(view.status?.snapshot?.observedAt ?? 0, view.status?.previous?.observedAt ?? 0, view.status?.lastFailure?.at ?? 0); + useEffect(() => { if (observed) void guidance.refresh(); }, [observed, guidance.refresh]); const snapshot = view.snapshot, tree = view.tree, page = view.filtering ? view.files : tree, @@ -252,6 +257,7 @@ function StorageSession({ root }: { root: BrainUiRoot }) { )} + {snapshot && ( <> Owner: {row.owner}. {row.effect}

-

{row.headroom.note}

+

Needs headroom. {row.headroom.note}

{row.id === "prune-scratch" && (

Existing seven-day / 1.07 GB policy (1,073,741,824 bytes); diff --git a/packages/ui-react/src/components/settings/storage/storage-guidance.tsx b/packages/ui-react/src/components/settings/storage/storage-guidance.tsx new file mode 100644 index 0000000000..4674a12842 --- /dev/null +++ b/packages/ui-react/src/components/settings/storage/storage-guidance.tsx @@ -0,0 +1,608 @@ +import { useId, useMemo, useState } from "react"; +import { CircleHelp, TriangleAlert } from "lucide-react"; +import type { + StorageGuidanceResponse, + StorageHistoryResponse, + StorageHistoryObservation, + StorageHistoryTier, + StoragePressureView, + StorageScenario, + StorageObservedRate, +} from "@schlessera/brain-ui-sdk/protocol"; +import { PushToggle } from "../../activity/push-toggle.js"; +import { + bytes, + exactBytes, + humanReason, + time, + type StorageMetric, +} from "./storage-model.js"; +const DAY = 86_400_000; +const number = (value: number) => + new Intl.NumberFormat("en", { maximumSignificantDigits: 3 }).format(value); +const interval = (milliseconds: number) => { + const [value, unit] = + milliseconds < 60_000 + ? ([Math.floor(milliseconds / 1000), "second"] as const) + : milliseconds < 3_600_000 + ? ([Math.floor(milliseconds / 60_000), "minute"] as const) + : milliseconds < DAY + ? ([milliseconds / 3_600_000, "hour"] as const) + : ([milliseconds / DAY, "day"] as const); + return `${number(value)} ${unit}${value === 1 ? "" : "s"}`; +}; +const age = (at: number, now: number) => + at > now ? "Clock ordering unknown" : `${interval(now - at)} ago`; +function Rate({ + value, + label, +}: { + value: StorageObservedRate | null; + label: string; +}) { + return value ? ( +

+ {label}: {value.bytesPerDay > 0 ? "+" : ""} + {bytes(value.bytesPerDay)}/day observed +
+ + {time(value.start)} → {time(value.end)} ·{" "} + {interval(value.end - value.start)} · {value.samples} observations + +

+ ) : ( +

{label}: rate unavailable · need comparable observations.

+ ); +} +function Scenario({ value, label }: { value: StorageScenario; label: string }) { + return ( +
+

+ {label}:{" "} + {value.state === "finite" && value.days !== null + ? `${number(value.days)} days, if observed depletion continues` + : value.state === "non-depleting" + ? "No depletion observed in this window; future capacity is unknown" + : `Unavailable · ${humanReason(value.reason)}`} +

+ {value.start !== null && value.end !== null && ( +

+ {time(value.start)} → {time(value.end)} ·{" "} + {interval(value.end - value.start)} · {value.samples} observations +

+ )} +
+ ); +} +function PressureCard({ + value, + index, + metric, + stale, +}: { + value: StoragePressureView; + index: number; + metric: StorageMetric; + stale: boolean; +}) { + const id = useId(), + evidence = value.scopeChanged ? null : value.evidence; + const uncertain = stale || value.state !== "current" || value.scopeChanged; + const label = + value.severity === "critical" + ? "Very low space" + : value.severity === "low" + ? "Low space" + : uncertain + ? "Capacity unavailable" + : "Capacity observed"; + const Glyph = value.severity === "observed" ? CircleHelp : TriangleAlert; + const finite = + evidence?.short.state === "finite" && + evidence.long.state === "finite" && + evidence.short.days !== null && + evidence.long.days !== null; + const name = value.scopeChanged + ? "Earlier capacity scope" + : value.stores[0]?.label ?? `Capacity ${index + 1}`; + return ( +
+

+ Capacity · {name} +

+

+

+ {value.scopeChanged ? ( +

+ The observed scope changed. The earlier warning requires inspection; + its measurements do not describe the new scope. +

+ ) : ( +

+ Writes:{" "} + {value.stores.map((store) => store.label).join(", ") || + "Current target labels unavailable; refresh to inspect."} +

+ )} +

+ Scope {value.rootScope.slice(0, 8)} · {time(value.at)} ·{" "} + {age(value.at, Date.now())} +

+
    + {value.causes.capacity && ( +
  • Available-capacity warning remains active.
    • + )} + {value.causes.forecast && ( +
  • Qualified capacity-scenario warning remains active.
    • + )} + {value.causes.refusals.map((refusal) => ( +
  • + Owner refused a write ·{" "} + {refusal.resource === "unknown" + ? "resource unknown" + : refusal.resource}{" "} + · {time(refusal.at)}. + {refusal.successAt !== null + ? ` A matching write succeeded ${time( + refusal.successAt + )}; recovery evidence is still required.` + : " Matching successful write has not been reported."} +
    • + ))} +
+ {value.recoveryAt !== null && ( +

+ Recovery observation recorded {time(value.recoveryAt)}. Waiting for + another qualifying observation; every active cause must recover. +

+ )} + {evidence ? ( + <> +

+ {bytes(evidence.available)} available of {bytes(evidence.total)} · + measured {time(evidence.at)} + {!evidence.fresh ? " · Out of date" : ""} +

+
+

+ Growth ·{" "} + {metric === "allocated" ? "On disk estimate" : "File size"} +

+ + +

+ Other writers can change capacity while App usage stays flat. + Shared extents may overlap; negative change is not proof of space + reclaimed. +

+
+
+

Capacity scenarios

+ {finite && ( +

+ {number(Math.min(evidence.short.days!, evidence.long.days!))}– + {number(Math.max(evidence.short.days!, evidence.long.days!))}{" "} + days if observed rates continue. These are two scenarios, not a + confidence interval or deadline. +

+ )} + + +
+ + ) : ( +

+ Available bytes, growth and capacity scenarios are unknown for this + scope. +

+ )} +
+ ); +} +function provisional(value: StorageHistoryObservation, now: number) { + const bucket = (at: number) => + value.tier === "six-hour" + ? Math.floor(at / (6 * 3_600_000)) + : value.tier === "daily" + ? Math.floor(at / DAY) + : Math.floor((at - 4 * DAY) / (7 * DAY)); + return value.at <= now && bucket(value.at) === bucket(now); +} +function HistoryTable({ + history, + guidance, +}: { + history: StorageHistoryResponse; + guidance: StorageGuidanceResponse; +}) { + const id = useId(), + [tier, setTier] = useState("six-hour"), + [chosen, setChosen] = useState(""); + const identities = useMemo( + () => [ + ...new Set( + [...history.observations] + .sort((a, b) => b.at - a.at) + .flatMap((row) => row.capacities.map((capacity) => capacity.identity)) + ), + ], + [history] + ); + const identity = identities.includes(chosen) ? chosen : identities[0] ?? ""; + const rows = history.observations + .filter( + (row) => + row.tier === tier && + row.capacities.some((capacity) => capacity.identity === identity) + ) + .sort((a, b) => a.at - b.at); + const now = Date.now(); + return ( +
+ + View observations · Recent, Daily and Weekly + +
+

+ Actual endpoints, never averages. Current buckets are provisional; + gaps are not zeros. Daily and weekly points are not interpolated. + Weekly points do not supply scenarios. +

+
+ + +
+ {!rows.length ? ( +

No retained observations in this tier.

+ ) : ( + + + + + + + + + + {rows.map((row, index) => { + const capacity = row.capacities.find( + (value) => value.identity === identity + )!, + previous = rows[index - 1]; + const same = + previous?.scope === row.scope && + previous.capacities.find( + (value) => value.identity === identity + )?.rootScope === capacity.rootScope; + return ( + + + + + + ); + })} + +
Endpoint and intervalObserved quantitiesScope and evidence
+

{time(row.at)}

+

{age(row.at, now)}

+

Started {time(row.startedAt)}

+

+ {previous + ? same + ? `${interval( + row.at - previous.at + )} since previous endpoint` + : "Changed scope · no comparable interval" + : "First retained endpoint"} +

+ {provisional(row, now) && ( +

Current bucket · provisional

+ )} + 		+ {(["brain", "app", "operator"] as const).map((owner) => ( +

+ {owner === "brain" + ? "Brain" + : owner === "app" + ? "App" + : "Operator"} + : file size{" "} + + {bytes(capacity.owners[owner].logical)} + + ; on disk{" "} + + {bytes(capacity.owners[owner].allocated)} + +

+ ))} +

+ Object union: file size {bytes(capacity.union.logical)}; + on disk {bytes(capacity.union.allocated)} +

+

+ Available {bytes(capacity.available)} · free{" "} + {bytes(capacity.free)} · used {bytes(capacity.used)} · + total {bytes(capacity.total)} +

+ 		+

+ {row.gap ? "Gap in this bucket" : "No recorded gap"} +

+

+ Collection {row.coverage}; capacity {capacity.coverage} +

+

+ {row.reason === "none" + ? "No collection exception" + : humanReason(row.reason)} +

+

+ Basis {row.basis} · collector {row.collector} +

+

{row.source} observation

+

+ Collection scope {row.scope.slice(0, 8)} +

+

+ Root scope {capacity.rootScope.slice(0, 8)} +

+
+ )} +

+ {history.observations.length} retained endpoints ·{" "} + {history.payloadBytes === null + ? "Payload charge unknown" + : `${exactBytes(history.payloadBytes)} serialized history`} + . This explanatory charge is already inside operational database + usage; SQLite/WAL and delivery receipts cost additional physical + storage. +

+
+
+ ); +} +function Notices({ + value, +}: { + value: StorageGuidanceResponse["notifications"]; +}) { + return ( +
+ + Critical notifications + +
+

+ Optional, through this app's existing push subscription. Only Critical + Storage episodes can submit a notice; no recurring reminders. Opening + this view never asks for browser permission. +

+
+ +
+

+ Quiet hours: 22:00–08:00 in the device's reported timezone. An unknown + timezone defers delivery. Stale or unresolved evidence can remain + visible without sending a notice. +

+ {!value.subscriptions.length ? ( +

No bound push subscription was reported for this account.

+ ) : ( +
    + {value.subscriptions.map((row, index) => ( +
  • + Device {index + 1}:{" "} + {!row.zoneKnown + ? "timezone unknown · delivery pending" + : row.quiet + ? "quiet hours · delivery deferred" + : "outside quiet hours; other eligibility checks still apply"} + . +
    • + ))} +
+ )} + {value.subscriptionsTruncated && ( +

Additional subscription summaries are not shown.

+ )} + {!value.receipts.length ? ( +

No Storage delivery attempt recorded.

+ ) : ( +
    + {value.receipts.map((row, index) => ( +
  • + Episode {row.episode.slice(0, 8)}:{" "} + {row.state === "accepted" + ? "Accepted by push service; device display unconfirmed" + : row.state === "unknown" + ? "Unknown delivery outcome; no automatic retry" + : row.state === "failed" + ? "Known delivery failure" + : row.state === "suppressed" + ? "Suppressed · no longer eligible" + : row.state === "sending" + ? "Submission in progress; outcome unknown" + : "Pending eligibility checks"}{" "} + · {row.attempts} of 3 attempts. + {row.nextAt !== null && + ` Earliest retry ${time( + row.nextAt + )}; quiet hours and fresh evidence still apply.`} +
    • + ))} +
+ )} + {value.receiptsTruncated && ( +

Additional delivery receipts are not shown.

+ )} +
+
+ ); +} +export function StorageGuidance({ + guidance, + history, + unavailable, + metric, +}: { + guidance: StorageGuidanceResponse | null; + history: StorageHistoryResponse | null; + unavailable: boolean; + metric: StorageMetric; +}) { + if (!guidance) + return ( +
+

+ {unavailable + ? "History and guidance are unavailable on this host. Inventory and recovery actions remain separate." + : "Loading capacity guidance…"} +

+
+ ); + return ( +
+

Pressure and growth

+ {unavailable && ( +

+ Current guidance could not be checked. The retained warning is out of + date; refresh before relying on its measurements. +

+ )} + {guidance.paused && ( +

+ History paused · {humanReason(guidance.paused)}. Last valid evidence + is retained. +

+ )} + {guidance.pending && ( +

+ A new observation is awaiting history storage. Existing warnings + remain until recovery is confirmed. +

+ )} + {!guidance.capacities.length && ( +

+ Capacity guidance not measured yet. Use Measure to inspect current + storage. +

+ )} +
1 + ? "@[600px]/storage-guidance:grid-cols-2" + : "" + }`} + > + {guidance.capacities.map((value, index) => ( + + ))} +
+ {(guidance.scopeLimit || guidance.untracked.length > 0) && ( +

+ History is limited to four capacity scopes. History is not recorded + for additional capacities; inspect their current inventory separately. +

+ )} +

+ Cleanup follows existing ownership and retention rules. If no eligible + cleanup is found, add capacity through your host. Packing, index + rebuilds, retained removal, Undo and backups need headroom; owner + preflight remains final. +

+ {history ? ( + + ) : ( +

Observation history is unavailable.

+ )} + +
+ ); +} diff --git a/packages/ui-react/src/components/settings/storage/use-storage-guidance.ts b/packages/ui-react/src/components/settings/storage/use-storage-guidance.ts new file mode 100644 index 0000000000..b2c52a87ad --- /dev/null +++ b/packages/ui-react/src/components/settings/storage/use-storage-guidance.ts @@ -0,0 +1,187 @@ +import { useCallback, useMemo, useSyncExternalStore } from "react"; +import { createStore } from "zustand/vanilla"; +import type { + StorageGuidanceResponse, + StorageHistoryResponse, +} from "@schlessera/brain-ui-sdk/protocol"; +import type { BrainUiRoot } from "../../../root.js"; +import { ApiRequestError } from "../../../lib/api-client.js"; +import { storageApi } from "../../../lib/storage-api.js"; +interface EvidenceState { + epoch: number; + guidance: StorageGuidanceResponse | null; + history: StorageHistoryResponse | null; + error: "unavailable" | "forbidden" | null; + loading: boolean; +} +const empty = (epoch: number): EvidenceState => ({ + epoch, + guidance: null, + history: null, + error: null, + loading: false, +}); +/** One read coordinator per mounted root. It retains no account data in browser storage. */ +function createEvidence(root: BrainUiRoot) { + const api = storageApi(root), + state = createStore(() => empty(root.authLock.epoch())); + let users = 0, + histories = 0, + generation = 0; + let controller: AbortController | null = null, + active: Promise | null = null; + let timer: ReturnType | null = null, + unwatch: (() => void) | null = null, + disconnect: (() => void) | null = null; + function clear() { + generation++; + controller?.abort(); + controller = null; + active = null; + state.setState(empty(root.authLock.epoch()), true); + } + function refresh(): Promise { + if (!users || root.authLock.state.getState().phase !== "active") + return Promise.resolve(); + if (active) return active; + const request = new AbortController(), + epoch = root.authLock.epoch(), + account = root.stores.connection.getState().accountKey, + owner = ++generation, + withHistory = histories > 0; + controller = request; + const deadline = setTimeout(() => request.abort(), 10_000); + state.setState({ loading: true, epoch }); + active = Promise.allSettled([ + api.guidance(request.signal), + withHistory ? api.history(request.signal) : Promise.resolve(null), + ]) + .then(([guidance, history]) => { + if ( + owner !== generation || + account !== root.stores.connection.getState().accountKey || + epoch !== root.authLock.epoch() || + root.authLock.state.getState().phase !== "active" + ) + return; + const denied = [guidance, history].some( + (value) => + value.status === "rejected" && + value.reason instanceof ApiRequestError && + [401, 403].includes(value.reason.status) + ); + if (denied) { + state.setState({ ...empty(epoch), error: "forbidden" }, true); + return; + } + if ( + guidance.status === "fulfilled" && + guidance.value.version === 1 && + Array.isArray(guidance.value.capacities) + ) + state.setState({ guidance: guidance.value, error: null }); + else state.setState({ error: "unavailable" }); + if (withHistory) + state.setState({ + history: + history.status === "fulfilled" && + history.value?.version === 1 && + Array.isArray(history.value.observations) + ? history.value + : null, + }); + }) + .finally(() => { + clearTimeout(deadline); + if (owner !== generation) return; + controller = null; + active = null; + state.setState({ loading: false }); + // The pane may have subscribed while an indicator-only read was in flight. + if (!withHistory && histories && state.getState().error === null) + void refresh(); + }); + return active; + } + function foreground() { + if (document.visibilityState !== "hidden") void refresh(); + } + return { + state, + refresh, + subscribe(listener: () => void, history: boolean) { + const stop = state.subscribe(listener); + users++; + if (history) histories++; + if (users === 1) { + if (state.getState().epoch !== root.authLock.epoch()) clear(); + unwatch = root.authLock.state.subscribe(() => { + clear(); + if (root.authLock.state.getState().phase === "active") void refresh(); + }); + disconnect = root.stores.connection.subscribe((next, previous) => { + if (next.accountKey !== previous.accountKey) { + clear(); + void refresh(); + } + }); + // This only refreshes server evidence. There is no browser warning or reminder policy. + timer = setInterval(foreground, 60_000); + window.addEventListener("focus", foreground); + document.addEventListener("visibilitychange", foreground); + } + void refresh(); + return () => { + stop(); + users--; + if (history) histories--; + if (!users) { + clear(); + if (timer) clearInterval(timer); + timer = null; + unwatch?.(); + unwatch = null; + disconnect?.(); + disconnect = null; + window.removeEventListener("focus", foreground); + document.removeEventListener("visibilitychange", foreground); + } + }; + }, + }; +} +const evidence = new WeakMap>(); +export function useStorageGuidance( + root: BrainUiRoot, + history = false, + enabled = true +) { + const shared = useMemo(() => { + let value = evidence.get(root); + if (!value) { + value = createEvidence(root); + evidence.set(root, value); + } + return value; + }, [root]); + const auth = useSyncExternalStore( + root.authLock.state.subscribe, + () => `${root.authLock.epoch()}:${root.authLock.state.getState().phase}`, + () => `${root.authLock.epoch()}:${root.authLock.state.getState().phase}` + ); + const subscribe = useCallback( + (listener: () => void) => + enabled ? shared.subscribe(listener, history) : () => {}, + [shared, history, enabled] + ); + const value = useSyncExternalStore( + subscribe, + shared.state.getState, + shared.state.getState + ); + const visible = auth === `${value.epoch}:active`; + return { + ...(visible ? value : empty(root.authLock.epoch())), + refresh: shared.refresh, + }; +} diff --git a/packages/ui-react/src/components/settings/storage/use-storage.ts b/packages/ui-react/src/components/settings/storage/use-storage.ts index cd7e516a5c..c6de418dc1 100644 --- a/packages/ui-react/src/components/settings/storage/use-storage.ts +++ b/packages/ui-react/src/components/settings/storage/use-storage.ts @@ -65,9 +65,15 @@ export function useStorage(root: BrainUiRoot) { [maintenanceId, setMaintenanceId] = useState(null); const [announcement, setAnnouncement] = useState(""); const refreshedReceipts = useRef(new Set()); + const requestedScan = useRef(null); const epoch = root.authLock.epoch(); const acceptStatus = (next: StorageStatus) => { setStatus(next); + if (next.state === "idle" && requestedScan.current && (next.snapshot?.id === requestedScan.current || next.previous?.id === requestedScan.current || next.lastFailure?.scanId === requestedScan.current)) { + const requestedScanId = requestedScan.current; + requestedScan.current = null; + setAnnouncement(next.lastFailure?.scanId === requestedScanId ? "Storage measurement did not complete. Previous observations remain available." : "Storage measurement updated. Existing file selections remain unchanged."); + } setSnapshot((previous) => previous ?? next.snapshot); }; function current(controller: AbortController) { @@ -575,7 +581,7 @@ export function useStorage(root: BrainUiRoot) { refresh: () => action(async (signal) => { const next = await api.scan(signal); - if (guarded(signal)) acceptStatus(next); + if (guarded(signal)) { requestedScan.current = next.activeScanId; setAnnouncement(""); acceptStatus(next); } }), copyList: async () => { try { diff --git a/packages/ui-react/src/hooks/use-hash-routes.ts b/packages/ui-react/src/hooks/use-hash-routes.ts index 0fd1ecb491..b8d035ad51 100644 --- a/packages/ui-react/src/hooks/use-hash-routes.ts +++ b/packages/ui-react/src/hooks/use-hash-routes.ts @@ -1,4 +1,5 @@ -import { useBrainUiRoot } from "../root-context.js"; +import { useStorageGuidance } from "../components/settings/storage/use-storage-guidance.js"; +import { useBrainUiRoot, useRootStore } from "../root-context.js"; import { useEffect, useRef, useState } from "react"; import { useFileStore } from "../stores/file-store.js"; import { useUIStore, type ActiveView } from "../stores/ui-store.js"; @@ -35,6 +36,69 @@ export function useHashRoutes(): void { const root = useBrainUiRoot(); const [hash, setHash] = useState(() => window.location.hash); const syncingFromHash = useRef(false); + const storage = useStorageGuidance( + root, + false, + hash === "#/settings/storage" + ); + const account = useRootStore("connection", (state) => state.accountKey); + const openedStorage = useRef<{ + root: typeof root; + epoch: number; + account: string | null; + hash: string; + } | null>(null); + const storageWasOpen = useRef(false); + useEffect(() => { + if (hash !== "#/settings/storage") { + openedStorage.current = null; + return; + } + if ( + !storage.guidance || + storage.error || + root.authLock.state.getState().phase !== "active" + ) + return; + const epoch = root.authLock.epoch(); + if ( + openedStorage.current?.root === root && + openedStorage.current.epoch === epoch && + openedStorage.current.account === account && + openedStorage.current.hash === hash + ) + return; + openedStorage.current = { root, epoch, account, hash }; + root.stores.ui.getState().openSettings("storage"); + }, [hash, storage.guidance, storage.error, root, account]); + const storageOpen = useUIStore( + (state) => state.settingsPanelOpen && state.settingsTab === "storage" + ); + useEffect(() => { + if (hash !== "#/settings/storage") { + storageWasOpen.current = false; + return; + } + if (storageOpen) { + storageWasOpen.current = true; + return; + } + if ( + !storageWasOpen.current || + root.authLock.state.getState().phase !== "active" + ) + return; + storageWasOpen.current = false; + const view = root.stores.ui.getState().activeView; + history.replaceState( + null, + "", + `${window.location.pathname}${window.location.search}${ + view === "chat" ? "" : `#/${view}` + }` + ); + setHash(window.location.hash); + }, [storageOpen, hash, root]); const activeView = useUIStore((state) => state.activeView); const setActiveView = useUIStore((state) => state.setActiveView); const setFilePanelOpen = useUIStore((state) => state.setFilePanelOpen); @@ -89,6 +153,8 @@ export function useHashRoutes(): void { return; } + // Preserve the authenticated Settings deep link while its authority read settles. + if (window.location.hash === "#/settings/storage") return; if (activeView === "chat") { if (viewForHash(window.location.hash)) { history.replaceState(null, "", window.location.pathname); @@ -96,8 +162,12 @@ export function useHashRoutes(): void { return; } - const config = VIEW_ROUTES.find((candidate) => candidate.view === activeView)!; - if (!matchesViewRoute(window.location.hash, config.view, config.prefixMatch)) { + const config = VIEW_ROUTES.find( + (candidate) => candidate.view === activeView + )!; + if ( + !matchesViewRoute(window.location.hash, config.view, config.prefixMatch) + ) { history.replaceState(null, "", `#/${activeView}`); } }, [activeView]); diff --git a/packages/ui-react/src/lib/storage-api.ts b/packages/ui-react/src/lib/storage-api.ts index 4dda1a8d9f..3e20ca51e0 100644 --- a/packages/ui-react/src/lib/storage-api.ts +++ b/packages/ui-react/src/lib/storage-api.ts @@ -1,5 +1,7 @@ import type { StorageStatus, + StorageHistoryResponse, + StorageGuidanceResponse, StorageFilter, StorageTreeFilter, StoragePage, @@ -47,6 +49,8 @@ export function storageApi(root: Pick) { .map(([key, value]) => [key, String(value)]) ).toString(); return { + history: (signal?: AbortSignal) => call("/history", signal), + guidance: (signal?: AbortSignal) => call("/guidance", signal), status: (signal?: AbortSignal) => call("", signal), scan: (signal?: AbortSignal) => call("/scans", signal, "POST", {}), diff --git a/packages/ui-react/tests/browser/storage-guidance.layout.tsx b/packages/ui-react/tests/browser/storage-guidance.layout.tsx new file mode 100644 index 0000000000..3c94d62df0 --- /dev/null +++ b/packages/ui-react/tests/browser/storage-guidance.layout.tsx @@ -0,0 +1,453 @@ +import axe from "axe-core"; +import { afterEach, expect, test, vi } from "vitest"; +import { commands, page, userEvent } from "vitest/browser"; +import { flushSync } from "react-dom"; +import { createRoot, type Root } from "react-dom/client"; +import { createBrainUiRoot, type BrainUiRoot } from "../../src/root.js"; +import { BrainUiProvider } from "../../src/root-context.js"; +import { StorageTab } from "../../src/components/settings/storage-tab.js"; +import { StorageIndicator } from "../../src/components/layout/storage-indicator.js"; +import { useHashRoutes } from "../../src/hooks/use-hash-routes.js"; +import { + storageGuidanceFixture, + guidanceTime, +} from "../fixtures/storage-guidance.js"; +let ui: BrainUiRoot, renderer: Root, host: HTMLElement, style: HTMLElement; +const original = { + width: innerWidth, + height: innerHeight, + theme: document.documentElement.dataset.theme, + zoom: document.documentElement.style.zoom, + hash: location.hash, +}; +afterEach(async () => { + if (renderer) flushSync(() => renderer.unmount()); + ui?.dispose(); + host?.remove(); + style?.remove(); + vi.restoreAllMocks(); + vi.useRealTimers(); + delete document.body.dataset.buttonGallery; + document.documentElement.style.zoom = original.zoom; + if (original.theme === undefined) + delete document.documentElement.dataset.theme; + else document.documentElement.dataset.theme = original.theme; + history.replaceState( + null, + "", + `${location.pathname}${location.search}${original.hash}` + ); + await page.viewport(original.width, original.height); +}); +function Routes() { + useHashRoutes(); + return ; +} +async function scene({ + width = 320, + theme = "dark", + zoom = 1, + route = false, + fixture = storageGuidanceFixture(), +} = {}) { + await page.viewport(width * zoom, 900 * zoom); + document.documentElement.style.zoom = String(zoom); + document.documentElement.dataset.theme = theme; + document.body.dataset.buttonGallery = ""; + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(guidanceTime); + style = document.createElement("style"); + style.textContent = await commands.formConsumerStyles(); + document.head.append(style); + host = document.createElement("div"); + host.style.cssText = `width:${width}px;height:900px;display:flex;flex-direction:column;background:var(--bk-color-canvas);color:var(--bk-color-ink)`; + document.body.append(host); + ui = createBrainUiRoot({ storage: null, request: fixture.request }); + renderer = createRoot(host); + flushSync(() => + renderer.render( + + {route ? ( + + ) : ( + <> + + + + )} + + ) + ); + await expect.poll(() => fixture.reads.length).toBeGreaterThan(0); + return fixture; +} +const button = (name: string | RegExp) => + page.getByRole("button", { name, exact: typeof name === "string" }); +const refresh = () => window.dispatchEvent(new Event("focus")); +const warning = () => host.querySelector("[data-storage-indicator]"); +async function settled() { + await expect.poll(() => warning()?.textContent).toContain("Very low space"); +} +async function historyOpen() { + await page + .getByText("View observations · Recent, Daily and Weekly", { exact: true }) + .click(); +} +for (const width of [320, 1280]) + for (const theme of ["dark", "light"]) + for (const zoom of [1, 2]) + test(`History evidence ${width} ${theme} ${zoom * 100}%`, async () => { + const f = await scene({ width, theme, zoom }); + await settled(); + await commands.buttonCapture( + `guidance-${width}-${theme}-${zoom}-overview` + ); + await historyOpen(); + const table = page.getByRole("table", { + name: "Storage observation history", + }); + await expect + .poll(() => table.element().textContent) + .toContain("1 day since previous endpoint"); + expect(table.element().textContent).toContain( + "Current bucket · provisional" + ); + expect(table.element().textContent).toContain("Gap in this bucket"); + expect(table.element().textContent).toContain("on disk Unknown"); + expect(table.element().textContent).toContain( + "Collection partial; capacity partial" + ); + expect(host.textContent, "STORAGE_DIVERGENT_SCENARIOS").toContain( + "4–80 days if observed rates continue" + ); + expect(host.textContent).toContain("Measured App stores: 0 bytes/day"); + expect(host.textContent).toContain( + "Capacity available change: -10 MB/day" + ); + await page + .getByRole("combobox", { name: "History tier" }) + .selectOptions("daily"); + expect(table.element().textContent).toContain( + "3 days since previous endpoint" + ); + await page + .getByRole("combobox", { name: "History tier" }) + .selectOptions("weekly"); + expect(table.element().textContent).toContain( + "14 days since previous endpoint" + ); + if (width === 320) + expect( + getComputedStyle(table.element().querySelector("tbody tr")!) + .display, + "STORAGE_HISTORY_NARROW_STACKED_ROWS" + ).toBe("block"); + expect( + host.scrollWidth, + "STORAGE_HISTORY_NO_OVERFLOW" + ).toBeLessThanOrEqual(host.clientWidth + 1); + for (const target of host.querySelectorAll( + "summary,select,[data-storage-indicator] button" + )) + expect( + target.getBoundingClientRect().height / zoom, + "STORAGE_GUIDANCE_TARGET_HEIGHT" + ).toBeGreaterThanOrEqual(44); + const result = await axe.run(host, { + runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "wcag21aa"] }, + }); + expect( + result.violations.map((row) => ({ + id: row.id, + nodes: row.nodes.map((node) => node.target), + })), + "STORAGE_GUIDANCE_ACCESSIBLE" + ).toEqual([]); + const tier = page + .getByRole("combobox", { name: "History tier" }) + .element() as HTMLElement; + tier.focus(); + await userEvent.keyboard("{Tab}"); + expect(document.activeElement).toBe( + page.getByRole("combobox", { name: "Capacity scope" }).element() + ); + await commands.buttonCapture(`guidance-${width}-${theme}-${zoom}`); + if (zoom === 1) { + host.style.filter = "grayscale(1)"; + await commands.buttonCapture(`guidance-${width}-${theme}-grey`); + host.style.filter = ""; + } + expect( + f.inventory.requests.filter((row) => row.method === "POST"), + "STORAGE_GUIDANCE_NO_SCAN_OR_PERMISSION" + ).toEqual([]); + }); +test("Missing scenarios never hide refusal pressure or invent a range", async () => { + const f = storageGuidanceFixture(), + capacity = f.guidance.capacities[0]!; + capacity.causes.capacity = false; + capacity.causes.forecast = false; + capacity.causes.refusals[0]!.resource = "quota"; + capacity.evidence!.short = { + state: "unavailable", + reason: "incomplete", + samples: 0, + start: null, + end: null, + depletionPerDay: null, + days: null, + }; + capacity.evidence!.long = { + ...capacity.evidence!.short, + state: "non-depleting", + reason: null, + }; + await scene({ fixture: f }); + await settled(); + expect( + host.querySelector("[data-storage-scenario-range]"), + "STORAGE_NO_UNQUALIFIED_RANGE" + ).toBeNull(); + expect(host.textContent).toContain("Owner refused a write · quota"); + expect(host.textContent).toContain("Short window: Unavailable"); + expect(host.textContent).toContain("No depletion observed"); + expect(host.textContent).not.toMatch(/Infinity|NaN/); + f.guidance.state = "unknown"; + capacity.scopeChanged = true; + capacity.state = "unknown"; + refresh(); + await expect.poll(() => host.textContent).toContain("Earlier capacity scope"); + expect(host.textContent).toContain( + "measurements do not describe the new scope" + ); + expect(host.textContent).not.toContain("4–80"); + expect(warning()?.textContent).toContain("Very low space"); +}); +test("History clocks, changed scopes and nullable quantities stay explicit", async () => { + const f = storageGuidanceFixture(); + f.history.observations[1]!.scope = "d".repeat(32); + f.history.observations[1]!.at += 60_000; + await scene({ fixture: f }); + await settled(); + await historyOpen(); + const text = page + .getByRole("table", { name: "Storage observation history" }) + .element().textContent; + expect(text).toContain("Changed scope · no comparable interval"); + expect(text).toContain("Clock ordering unknown"); + expect(text).toContain("Available 40 MB"); + expect(text).toContain("Object union: file size Unknown"); +}); +test("Permission is never requested on opening; receipts distinguish acceptance and uncertainty", async () => { + const permission = vi.spyOn(Notification, "requestPermission"); + await scene(); + await settled(); + await page.getByText("Critical notifications", { exact: true }).click(); + expect( + permission, + "STORAGE_NO_AUTO_PERMISSION_PROMPT" + ).not.toHaveBeenCalled(); + expect(host.textContent).toContain("timezone unknown · delivery pending"); + expect(host.textContent).toContain("quiet hours · delivery deferred"); + expect(host.textContent).toContain( + "Accepted by push service; device display unconfirmed" + ); + expect(host.textContent).toContain( + "Unknown delivery outcome; no automatic retry" + ); + expect(host.querySelectorAll('[aria-live="assertive"]')).toHaveLength(0); +}); +test("Unavailable evidence retains an explicitly stale warning; forbidden removes it", async () => { + const f = await scene(); + await settled(); + f.setStatus(503); + refresh(); + await expect + .poll(() => warning()?.textContent) + .toContain("Check current evidence"); + expect(host.textContent).toContain("out of date; refresh"); + f.setStatus(403); + refresh(); + await expect + .poll(warning, { message: "STORAGE_FORBIDDEN_HIDES_INDICATOR" }) + .toBeNull(); + expect(host.textContent).not.toContain("App journal: 4"); +}); +test("A late response from the prior account cannot replace current evidence", async () => { + const f = await scene({ route: true }); + await settled(); + let release!: () => void; + f.hold( + new Promise((resolve) => { + release = resolve; + }) + ); + const before = f.reads.length; + refresh(); + await expect.poll(() => f.reads.length).toBeGreaterThan(before); + f.hold(null); + f.guidance.severity = "low"; + f.guidance.capacities[0]!.severity = "low"; + ui.stores.connection.getState().setVpnStatus("connected", "athena"); + await expect.poll(() => warning()?.textContent).toContain("Low space"); + release(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect( + warning()?.textContent, + "STORAGE_ACCOUNT_RESPONSE_OWNERSHIP" + ).not.toContain("Very low space"); + await ui.authLock.expire("signed out", true); + await expect + .poll(warning, { message: "STORAGE_LOCK_HIDES_GUIDANCE" }) + .toBeNull(); +}); +test("Indicator and authenticated tap-through honor Settings leave guards and clean up the route", async () => { + const f = await scene({ route: true }); + await settled(); + let leave: (() => void) | undefined; + ui.stores.ui.getState().openSettings("modules"); + ui.stores.ui.getState().setSettingsNavigationGuard((next) => { + leave = next; + }); + await button("Open Storage: Very low space").click(); + expect( + ui.stores.ui.getState().settingsTab, + "STORAGE_INDICATOR_LEAVE_GUARD" + ).toBe("modules"); + leave!(); + expect(ui.stores.ui.getState().settingsTab).toBe("storage"); + ui.stores.ui.getState().setSettingsNavigationGuard(null); + ui.stores.ui.getState().closeAllPanels(); + f.setStatus(403); + refresh(); + await expect.poll(warning).toBeNull(); + history.replaceState(null, "", "#/settings/storage"); + window.dispatchEvent(new HashChangeEvent("hashchange")); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect( + ui.stores.ui.getState().settingsPanelOpen, + "STORAGE_TAP_REQUIRES_AUTHORITY" + ).toBe(false); + f.setStatus(200); + refresh(); + await expect.poll(() => ui.stores.ui.getState().settingsPanelOpen).toBe(true); + expect(ui.stores.ui.getState().settingsTab).toBe("storage"); + ui.stores.ui.getState().closeAllPanels(); + await expect + .poll(() => location.hash, { message: "STORAGE_CLOSED_ROUTE_CLEARED" }) + .toBe(""); +}); +test("Refreshing guidance and measurement preserves selection, review and focus with one polite completion", async () => { + const f = await scene({ width: 1280 }); + await settled(); + await button("Find large files").click(); + await button("Inspect Ithaca-harbor.pdf").click(); + await expect + .poll(() => button("Select file").element().getAttribute("aria-disabled")) + .not.toBe("true"); + await button("Select file").click(); + const measure = button("Refresh"); + await measure.click(); + const changes: string[] = []; + const live = host.querySelector('[aria-live="polite"]')!; + const observer = new MutationObserver(() => { + if (live.textContent?.includes("Storage measurement updated")) + changes.push(live.textContent); + }); + observer.observe(live, { + childList: true, + subtree: true, + characterData: true, + }); + try { + await expect + .poll(() => live.textContent, { timeout: 5000 }) + .toContain("Storage measurement updated"); + expect(changes, "STORAGE_ONE_POLITE_REFRESH").toHaveLength(1); + expect(document.activeElement).toBe(measure.element()); + expect(host.textContent).toContain("1 selected"); + } finally { + observer.disconnect(); + } + await button("Review selected files").click(); + await expect.poll(() => button("Remove 1 file").elements().length).toBe(1); + f.guidance.pending = true; + refresh(); + await expect + .poll(() => warning()?.textContent) + .toContain("Check current evidence"); + expect(button("Remove 1 file").elements()).toHaveLength(1); +}); + +test("Browser-blocked push is labelled and the existing opt-in switch keeps its touch target", async () => { + const permission = vi + .spyOn(Notification, "permission", "get") + .mockReturnValue("default"); + await scene(); + await settled(); + await page.getByText("Critical notifications", { exact: true }).click(); + const toggle = page.getByRole("switch", { name: "Push notifications" }); + await expect.poll(() => toggle.elements().length).toBe(1); + const r = toggle.element().getBoundingClientRect(), + pseudo = getComputedStyle(toggle.element(), "::before"); + expect( + r.height - parseFloat(pseudo.top) - parseFloat(pseudo.bottom) + ).toBeGreaterThanOrEqual(44); + expect( + r.width - parseFloat(pseudo.left) - parseFloat(pseudo.right) + ).toBeGreaterThanOrEqual(44); + expect(matchMedia("(prefers-reduced-motion: reduce)").matches).toBe(true); + for (const node of host.querySelectorAll( + "[data-storage-history], [data-storage-pressure], [data-storage-map-index]" + )) { + expect( + getComputedStyle(node).animationName, + "STORAGE_STATIC_HISTORY_REDUCED_MOTION" + ).toBe("none"); + } + permission.mockReturnValue("denied"); + const old = ui; + ui = createBrainUiRoot({ + storage: null, + request: storageGuidanceFixture().request, + }); + flushSync(() => + renderer.render( + + + + ) + ); + old.dispose(); + await page.getByText("Critical notifications", { exact: true }).click(); + await expect + .poll(() => host.textContent) + .toContain("Blocked in browser settings"); + expect(toggle.elements()).toHaveLength(0); +}); +test("A replaced root cannot publish its late operational response", async () => { + const oldFixture = storageGuidanceFixture(); + let release!: () => void; + oldFixture.hold( + new Promise((resolve) => { + release = resolve; + }) + ); + await scene({ route: true, fixture: oldFixture }); + const old = ui, + next = storageGuidanceFixture(); + next.setStatus(403); + ui = createBrainUiRoot({ storage: null, request: next.request }); + flushSync(() => + renderer.render( + + + + ) + ); + await expect.poll(() => next.reads.length).toBeGreaterThan(0); + release(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(warning(), "STORAGE_REPLACED_ROOT_HIDES_OLD_GUIDANCE").toBeNull(); + old.dispose(); +}); diff --git a/packages/ui-react/tests/browser/storage-routes.layout.tsx b/packages/ui-react/tests/browser/storage-routes.layout.tsx index 6fad48c5b1..5082a84872 100644 --- a/packages/ui-react/tests/browser/storage-routes.layout.tsx +++ b/packages/ui-react/tests/browser/storage-routes.layout.tsx @@ -59,6 +59,8 @@ async function scene(width: number) { ) ); await expect.poll(() => page.getByRole("table").elements().length).toBe(1); + await expect.poll(() => host!.textContent).toContain("Very low space"); + expect(host!.querySelector("[data-storage-scenario-range]"), "STORAGE_HTTP_INSUFFICIENT_HISTORY_NO_RANGE").toBeNull(); await button("Find large files").click(); await expect .poll(() => button("Inspect chart.pdf").elements().length) diff --git a/packages/ui-react/tests/fixtures/storage-browser-server.ts b/packages/ui-react/tests/fixtures/storage-browser-server.ts index 9e73f6b529..b9a265e008 100644 --- a/packages/ui-react/tests/fixtures/storage-browser-server.ts +++ b/packages/ui-react/tests/fixtures/storage-browser-server.ts @@ -10,6 +10,11 @@ import { storageRemovalFixture } from "../../../ui-server/tests/helpers/storage- import { testPrincipal } from "../../../ui-server/tests/helpers/principal.js"; import { createStorageRoutes } from "../../../ui-server/src/routes/storage.js"; import { createStorageRemovalRoutes } from "../../../ui-server/src/routes/storage-removal.js"; +import { createPrincipal } from "../../../ui-server/src/db/principals.js"; +import { createStorageHistory } from "../../../ui-server/src/storage/history-store.js"; +import { createStorageNotifier } from "../../../ui-server/src/storage/notify.js"; +import { createStorageGuidance } from "../../../ui-server/src/storage/guidance.js"; +import { hashScope } from "../../../ui-server/src/storage/history-model.js"; const fixture = storageRemovalFixture(), target = join(fixture.brain, "attachments/chart.pdf"), original = Buffer.alloc(5_100_000, 65); @@ -20,6 +25,13 @@ const digest = (bytes: Uint8Array) => originalHash = digest(original); fixture.storage.start("odysseus"); await fixture.storage.idle(); +const owner = createPrincipal(fixture.db, { authMethod: "password", label: "Odysseus", ttlSeconds: 3600 }); +fixture.db.query("UPDATE principals SET id = ? WHERE id = ?").run("odysseus", owner.id); +const history = createStorageHistory(fixture.db, fixture.brain), now = Date.now(), capacity = fixture.storage.status("odysseus").snapshot!.capacities[0]!; +// Deterministic pressure on the real HTTP projection, independent of host free space. +history.offer({ at: now, startedAt: now - 1000, source: 1, descriptor: ["0.44.0", hashScope("Ithaca"), 1], coverage: 0, reason: 0, + capacities: [{ descriptor: [capacity.id, hashScope("Ithaca roots")], values: [1_000_000_000, 960_000_000, 40_000_000, 40_000_000, 0, 5_100_000, 5_103_616, 100_000, 102_400, 0, 0, 5_200_000, 5_206_016] }] }); +const guidance = createStorageGuidance(fixture.db, history, fixture.storage, createStorageNotifier(fixture.db, history)); let drop = false, confirms = 0; const app = new Hono(); @@ -39,7 +51,7 @@ app.use("*", async (c, next) => { ); } }); -app.route("/api", createStorageRoutes(fixture.storage)); +app.route("/api", createStorageRoutes(fixture.storage, guidance)); app.route("/api", createStorageRemovalRoutes(fixture.removal)); app.get("/api/storage/maintenance", (c) => c.json({ operations: [] })); app.post("/fixture", async (c) => { @@ -60,6 +72,7 @@ const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: app.fetch }); console.log(JSON.stringify({ url: `http://127.0.0.1:${server.port}` })); process.on("SIGTERM", async () => { server.stop(true); + history.close(); await fixture.close(); process.exit(0); }); diff --git a/packages/ui-react/tests/fixtures/storage-guidance.ts b/packages/ui-react/tests/fixtures/storage-guidance.ts new file mode 100644 index 0000000000..af50686f2f --- /dev/null +++ b/packages/ui-react/tests/fixtures/storage-guidance.ts @@ -0,0 +1,199 @@ +import type { + StorageGuidanceResponse, + StorageHistoryResponse, + StorageHistoryObservation, +} from "@schlessera/brain-ui-sdk/protocol"; +import { storageViewFixture } from "./storage-view.js"; +export const guidanceTime = Date.UTC(2026, 6, 12, 9, 41), + day = 86_400_000; +/** Odysseus-only evidence, including a retained warning with no usable forecast. */ +export function storageGuidanceFixture() { + const inventory = storageViewFixture(), + at = guidanceTime, + identity = "a".repeat(32), + scope = "b".repeat(32); + const scenario = (days: number) => ({ + state: "finite" as const, + reason: null, + start: at - 15 * day, + end: at, + samples: 16, + depletionPerDay: 1_000_000, + days, + }); + const guidance: StorageGuidanceResponse = { + version: 1, + state: "current", + severity: "critical", + paused: null, + pending: false, + observedAt: at, + scopeLimit: false, + untracked: [], + capacities: [ + { + identity, + rootScope: scope, + severity: "critical", + at, + state: "current", + scopeChanged: false, + recoveryAt: at - 60_000, + episode: "ithaca-episode", + causes: { + capacity: true, + forecast: true, + refusals: [ + { + target: "c".repeat(32), + resource: "bytes", + at: at - 120_000, + successAt: at - 60_000, + }, + ], + }, + stores: [{ label: "App journal", owner: "app" }], + evidence: { + identity, + rootScope: scope, + at, + total: 1_000_000_000, + available: 40_000_000, + fresh: true, + short: { ...scenario(4), start: at - 2 * day, samples: 9 }, + long: scenario(80), + appAllocated: { + start: at - 2 * day, + end: at, + samples: 9, + bytesPerDay: 0, + }, + appLogical: { + start: at - 2 * day, + end: at, + samples: 9, + bytesPerDay: 20_000, + }, + availableChange: { + start: at - 2 * day, + end: at, + samples: 9, + bytesPerDay: -10_000_000, + }, + }, + }, + ], + notifications: { + permission: "browser-managed", + quietHours: { start: 22, end: 8, unknownZone: "defer" }, + subscriptionsTruncated: false, + receiptsTruncated: false, + subscriptions: [ + { zoneKnown: false, quiet: null }, + { zoneKnown: true, quiet: true }, + ], + receipts: [ + { + episode: "ithaca-accepted", + state: "accepted", + attempts: 1, + nextAt: null, + }, + { + episode: "ithaca-unknown", + state: "unknown", + attempts: 1, + nextAt: null, + }, + ], + }, + }; + const observation = ( + tier: StorageHistoryObservation["tier"], + offset: number, + gap = false + ): StorageHistoryObservation => ({ + tier, + at: at - offset, + startedAt: at - offset - 1000, + gap, + source: "scheduled", + collector: "0.44.0", + scope, + basis: "storage-v1", + coverage: "partial", + reason: "partial", + capacities: [ + { + identity, + rootScope: scope, + coverage: "partial", + total: 1_000_000_000, + used: 900_000_000, + free: 100_000_000, + available: 40_000_000, + owners: { + brain: { logical: 5_100_000, allocated: 5_103_616 }, + app: { logical: 100_000, allocated: null }, + operator: { logical: null, allocated: null }, + }, + union: { logical: null, allocated: null }, + }, + ], + }); + const history: StorageHistoryResponse = { + version: 1, + state: "available", + paused: null, + pending: false, + lastObservedAt: at, + payloadBytes: 12345, + limits: { + sixHour: 64, + daily: 90, + weekly: 32, + capacities: 4, + payloadBytes: 262144, + }, + observations: [ + observation("six-hour", day), + observation("six-hour", 0, true), + observation("daily", 3 * day), + observation("daily", 0), + observation("weekly", 14 * day), + observation("weekly", 0, true), + ], + }; + let status = 200, + hold: Promise | null = null; + const reads: string[] = []; + const request = async (url: string, init?: RequestInit) => { + const path = new URL(url, "http://fixture.invalid").pathname; + if (!path.endsWith("/guidance") && !path.endsWith("/history")) + return inventory.request(url, init); + reads.push(path); + const body = structuredClone( + path.endsWith("/guidance") ? guidance : history + ), + responseStatus = status, + wait = hold; + if (wait) await wait; // Intentionally ignores abort: exercise the consumer's response ownership guard. + return Response.json( + responseStatus === 200 ? body : { error: "Synthetic authority response" }, + { status: responseStatus } + ); + }; + return { + inventory, + guidance, + history, + request, + reads, + setStatus(value: number) { + status = value; + }, + hold(value: Promise | null) { + hold = value; + }, + }; +} diff --git a/packages/ui-react/tests/fixtures/storage-view.ts b/packages/ui-react/tests/fixtures/storage-view.ts index 4183b4fd32..b364eb4ed3 100644 --- a/packages/ui-react/tests/fixtures/storage-view.ts +++ b/packages/ui-react/tests/fixtures/storage-view.ts @@ -433,7 +433,7 @@ export function storageViewFixture(scene: StorageScene = "dense") { if (scanning && ++scanReads > 1) { scanning = false; measured = true; - snapshot.id += "-new"; + snapshot.id = "odysseus-active"; snapshot.observedAt += 1000; snapshot.staleAfter += 1000; } From 4644d16b87694a08a2aa72d96e7b64ed3c9a6456 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 18:30:47 +0200 Subject: [PATCH 18/44] fix(storage): use shared indicator control and verify navigation entries --- .../components/layout/storage-indicator.tsx | 41 ++++++++++--------- .../src/components/settings/storage-tab.tsx | 3 +- .../browser/desktop-navigation.pointer.tsx | 4 +- .../browser/navigation-reach.pointer.tsx | 6 +-- .../storage-guidance-navigation.layout.tsx | 40 ++++++++++++++++++ .../tests/browser/storage-guidance.layout.tsx | 2 +- .../tests/render/render-smoke.test.tsx | 8 ++-- 7 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 packages/ui-react/tests/browser/storage-guidance-navigation.layout.tsx diff --git a/packages/ui-react/src/components/layout/storage-indicator.tsx b/packages/ui-react/src/components/layout/storage-indicator.tsx index 5597ec4ca3..5aa642a131 100644 --- a/packages/ui-react/src/components/layout/storage-indicator.tsx +++ b/packages/ui-react/src/components/layout/storage-indicator.tsx @@ -1,4 +1,4 @@ -import { CircleHelp, TriangleAlert, HardDrive } from "lucide-react"; +import { Button, type IconName } from "@schlessera/brain-ui-kit"; import { useBrainUiRoot } from "../../root-context.js"; import { useStorageGuidance } from "../settings/storage/use-storage-guidance.js"; /** A persistent status control opens the existing Settings pane; it adds no destination. */ @@ -17,36 +17,39 @@ export function StorageIndicator() { : uncertain ? "Capacity unknown" : "Capacity observed"; - const Glyph = + const icon: IconName = value.severity === "critical" || value.severity === "low" - ? TriangleAlert + ? "failed" : uncertain - ? CircleHelp - : HardDrive; + ? "unverified" + : "ledger"; return (
- + : "" + }`} + onClick={() => root.stores.ui.getState().openSettings("storage")} + />
); } diff --git a/packages/ui-react/src/components/settings/storage-tab.tsx b/packages/ui-react/src/components/settings/storage-tab.tsx index 07ed3058d1..1cfa230bcf 100644 --- a/packages/ui-react/src/components/settings/storage-tab.tsx +++ b/packages/ui-react/src/components/settings/storage-tab.tsx @@ -91,8 +91,9 @@ function StorageSession({ root }: { root: BrainUiRoot }) { ) ?? breadcrumb.current )?.focus(); }, [view.loading, view.tree, view.parent]); + const refreshGuidance = guidance.refresh; const observed = Math.max(view.status?.snapshot?.observedAt ?? 0, view.status?.previous?.observedAt ?? 0, view.status?.lastFailure?.at ?? 0); - useEffect(() => { if (observed) void guidance.refresh(); }, [observed, guidance.refresh]); + useEffect(() => { if (observed) void refreshGuidance(); }, [observed, refreshGuidance]); const snapshot = view.snapshot, tree = view.tree, page = view.filtering ? view.files : tree, diff --git a/packages/ui-react/tests/browser/desktop-navigation.pointer.tsx b/packages/ui-react/tests/browser/desktop-navigation.pointer.tsx index 021acc8062..8aef1f2d22 100644 --- a/packages/ui-react/tests/browser/desktop-navigation.pointer.tsx +++ b/packages/ui-react/tests/browser/desktop-navigation.pointer.tsx @@ -312,8 +312,8 @@ for (const theme of ["dark", "light"] as const) { await expect.poll(() => document.activeElement, { message: "focus enters the query" }).toBe(input); const names = [...dialog()!.querySelectorAll('[role="option"]')].map((o) => o.textContent!.replace("⏎", "")); const fine = mode !== "coarse"; - expect(names.slice(0, 7), "Jump to, remapped, with Graph keyless").toEqual([ - ...DESTINATIONS.map((d, i) => (fine ? `${d}⌘${i + 1}` : d)), "Graph", "New chat", + expect(names.slice(0, 8), "Jump to, remapped, with Storage and Graph keyless").toEqual([ + ...DESTINATIONS.map((d, i) => (fine ? `${d}⌘${i + 1}` : d)), "Storage", "Graph", "New chat", ]); expect(names.filter((n) => n.startsWith("Sessions")), "Sessions once").toHaveLength(1); await userEvent.keyboard("{Escape}"); diff --git a/packages/ui-react/tests/browser/navigation-reach.pointer.tsx b/packages/ui-react/tests/browser/navigation-reach.pointer.tsx index 188038df22..f1078a5c7b 100644 --- a/packages/ui-react/tests/browser/navigation-reach.pointer.tsx +++ b/packages/ui-react/tests/browser/navigation-reach.pointer.tsx @@ -400,9 +400,9 @@ function paletteRows(s: Scene): string[] { return [...s.host.querySelectorAll('[role="dialog"][aria-label="Command palette"] [role="option"]')] .map((o) => (o.textContent ?? "").replace(/⏎/g, "").replace(/⌘\d$/, "").replace(/(spends|sync|a turn is running|needs the host)+$/g, "").trim()); } -// Hygiene's authenticated review effect is proved against a real server in -// hygiene-review.layout.tsx; this shell fixture still inventories its row. -const PALETTE_ROWS = [...Object.values(targets).flatMap((t) => (t.palette ? [t.palette] : [])), "Start hygiene review"]; +// Specialized Storage and Hygiene flows have their own real-browser proof; +// this D52 destination matrix still inventories both keyless palette rows. +const PALETTE_ROWS = [...Object.values(targets).flatMap((t) => (t.palette ? [t.palette] : [])), "Storage", "Start hygiene review"]; const DESKTOP = [480, 900, 1279, 1280, 1440] as const; /** The cells' themes alternate by start, so each width is drawn in both. */ diff --git a/packages/ui-react/tests/browser/storage-guidance-navigation.layout.tsx b/packages/ui-react/tests/browser/storage-guidance-navigation.layout.tsx new file mode 100644 index 0000000000..0d1ea882f2 --- /dev/null +++ b/packages/ui-react/tests/browser/storage-guidance-navigation.layout.tsx @@ -0,0 +1,40 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { commands, page, userEvent } from "vitest/browser"; +import { flushSync } from "react-dom"; +import { createRoot, type Root } from "react-dom/client"; +import { createBrainUiRoot, type BrainUiRoot } from "../../src/root.js"; +import { BrainUiProvider } from "../../src/root-context.js"; +import { AppShell } from "../../src/components/layout/app-shell.js"; +import { ActivityPage } from "../../src/components/activity/activity-page.js"; +import { GraphPage } from "../../src/components/graph/graph-page.js"; +import { useUIStore } from "../../src/stores/ui-store.js"; +import { useHashRoutes } from "../../src/hooks/use-hash-routes.js"; +import { storageGuidanceFixture, guidanceTime } from "../fixtures/storage-guidance.js"; +let ui: BrainUiRoot, renderer: Root, host: HTMLElement, style: HTMLElement; +const original = { width: innerWidth, height: innerHeight, theme: document.documentElement.dataset.theme, hash: location.hash }; +afterEach(async () => { + if (renderer) flushSync(() => renderer.unmount()); ui?.dispose(); host?.remove(); style?.remove(); vi.useRealTimers(); + if (original.theme === undefined) delete document.documentElement.dataset.theme; else document.documentElement.dataset.theme = original.theme; + history.replaceState(null, "", `${location.pathname}${location.search}${original.hash}`); await page.viewport(original.width, original.height); +}); +function Shell() { useHashRoutes(); const view = useUIStore(state => state.activeView); return {view === "activity" ? : }; } +for (const view of ["graph", "activity"] as const) for (const entry of ["indicator", "notification", "palette"] as const) for (const width of entry === "palette" ? [1280] : [320, 1280]) + test(`Storage ${entry} draws its pane from ${view} at ${width}`, async () => { + await page.viewport(width, 900); vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(guidanceTime); + document.documentElement.dataset.theme = view === "graph" ? "dark" : "light"; + style = document.createElement("style"); style.textContent = await commands.formConsumerStyles(); document.head.append(style); + host = document.createElement("div"); host.style.cssText = "position:fixed;inset:0"; document.body.append(host); + history.replaceState(null, "", `#/${view}`); const fixture = storageGuidanceFixture(); ui = createBrainUiRoot({ storage: null, request: fixture.request }); + ui.stores.ui.getState().setActiveView(view); renderer = createRoot(host); + flushSync(() => renderer.render()); + const indicator = page.getByRole("button", { name: "Open Storage: Very low space", exact: true }); await expect.poll(() => indicator.elements().length).toBe(1); + if (entry === "indicator") { (indicator.element() as HTMLElement).focus(); await userEvent.keyboard("{Enter}"); } + else if (entry === "notification") { history.replaceState(null, "", "#/settings/storage"); window.dispatchEvent(new HashChangeEvent("hashchange")); } + else { await page.getByRole("button", { name: /^All commands/ }).click(); await page.getByRole("option", { name: /^Storage/ }).click(); } + await expect.poll(() => host.querySelector("[data-storage-pane]"), { timeout: 5000, message: "STORAGE_FULL_SHELL_PANE_DRAWN" }).not.toBeNull(); + await expect.poll(() => host.querySelector("[data-storage-pane]")?.textContent).toContain("Pressure and growth"); + expect(host.querySelector("[data-storage-pane]")!.getBoundingClientRect().height).toBeGreaterThan(0); + expect(ui.stores.ui.getState().activeView, "STORAGE_PRESERVES_HOSTING_VIEW").toBe(view); + expect(ui.stores.ui.getState().settingsTab).toBe("storage"); + expect(fixture.inventory.requests.filter(row => row.method === "POST"), "STORAGE_NAVIGATION_HAS_NO_EFFECT").toHaveLength(0); + }); diff --git a/packages/ui-react/tests/browser/storage-guidance.layout.tsx b/packages/ui-react/tests/browser/storage-guidance.layout.tsx index 3c94d62df0..fb92aada6a 100644 --- a/packages/ui-react/tests/browser/storage-guidance.layout.tsx +++ b/packages/ui-react/tests/browser/storage-guidance.layout.tsx @@ -147,7 +147,7 @@ for (const width of [320, 1280]) "STORAGE_HISTORY_NO_OVERFLOW" ).toBeLessThanOrEqual(host.clientWidth + 1); for (const target of host.querySelectorAll( - "summary,select,[data-storage-indicator] button" + 'summary,select,[data-storage-indicator] [role="button"]' )) expect( target.getBoundingClientRect().height / zoom, diff --git a/packages/ui-react/tests/render/render-smoke.test.tsx b/packages/ui-react/tests/render/render-smoke.test.tsx index 29e1cf7def..4efa596bd7 100644 --- a/packages/ui-react/tests/render/render-smoke.test.tsx +++ b/packages/ui-react/tests/render/render-smoke.test.tsx @@ -4291,10 +4291,10 @@ describe("DesktopPalette on the kit CommandPalette", () => { const dialog = view.getByRole("dialog", { name: "Command palette" }); expect(dialog).toBeTruthy(); const names = view.getAllByRole("option").map((o) => o.textContent); - // Jump to: the five destinations with their remapped keys, then Graph - // with none, then New chat. Sessions is one row, not two (D52 §1). - expect(names.slice(0, 7).map((n) => n?.replace(/⏎/g, ""))).toEqual([ - "Chat⌘1", "Sessions⌘2", "Actions⌘3", "Files⌘4", "Settings⌘5", "Graph", "New chat", + // Jump to: the five destinations with their remapped keys, then keyless + // Storage and Graph, then New chat. Sessions is one row, not two (D52 §1). + expect(names.slice(0, 8).map((n) => n?.replace(/⏎/g, ""))).toEqual([ + "Chat⌘1", "Sessions⌘2", "Actions⌘3", "Files⌘4", "Settings⌘5", "Storage", "Graph", "New chat", ]); expect(names.filter((n) => n?.includes("Sessions"))).toHaveLength(1); // Every other route is still here, the rail's acts included. From 1ae453a6ab239ba5d301b27464404293058f89b3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 18:44:37 +0200 Subject: [PATCH 19/44] fix(storage): preserve unknown evidence after owner report overflow --- docs/storage-history.md | 6 ++- .../ui-server/src/storage/history-store.ts | 38 ++++++++++++++----- .../tests/storage-history-store.test.ts | 26 +++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/storage-history.md b/docs/storage-history.md index d389f8e9df..bd44c89313 100644 --- a/docs/storage-history.md +++ b/docs/storage-history.md @@ -24,7 +24,11 @@ prevent qualification even when direct available-byte warnings remain usable. One newest aggregate can wait in memory between history transactions, at most one per minute. Intervening failures still break comparison/recovery. Pending owner refusals are bounded separately in memory and are applied without being -erased by a newer aggregate. A crash can lose pending observations. A failed +erased by a newer aggregate. Repeated successes without a matching refusal do +not occupy that buffer. If distinct owner evidence exceeds its bound, the next +successful transaction durably marks it unknown; later scans and restores cannot +reconstruct the missing refusals or automatically clear that warning. Ordinary +inventory remains available. A crash can lose pending observations. A failed SQLite write preserves the last valid durable payload and reports paused history; it never deletes recovery data to gain headroom. diff --git a/packages/ui-server/src/storage/history-store.ts b/packages/ui-server/src/storage/history-store.ts index 2dc21253a3..a7e6ae439f 100644 --- a/packages/ui-server/src/storage/history-store.ts +++ b/packages/ui-server/src/storage/history-store.ts @@ -8,7 +8,7 @@ import { HISTORY_LIMITS, bucket, hashScope, jsonBytes, retainAggregate, validate export interface HistoryState { version: 1; salt: string; lastPersistedAt: number; lastObservedAt: number; lastAttemptBucket: number | null; - paused: "collection-unavailable" | "clock-reversal" | null; + paused: "collection-unavailable" | "clock-reversal" | "history-owner-evidence-budget" | null; pressures: Pressure[]; /** Restoring an old backup cannot revive authority to send its old episodes. */ suppressedEpisodes: string[]; @@ -34,7 +34,7 @@ const pressureSchema = z.object({ && new Set(value.refusals.map(row => row.target + ":" + row.resource)).size === value.refusals.length); const stateSchema = z.object({ version: z.literal(1), salt: z.string().regex(/^[a-f0-9]{64}$/), lastPersistedAt: instant, - lastObservedAt: instant, lastAttemptBucket: instant.nullable(), paused: z.enum(["collection-unavailable", "clock-reversal"]).nullable(), + lastObservedAt: instant, lastAttemptBucket: instant.nullable(), paused: z.enum(["collection-unavailable", "clock-reversal", "history-owner-evidence-budget"]).nullable(), pressures: z.array(pressureSchema).max(4), suppressedEpisodes: z.array(z.string().uuid()).max(8), scopeLimit: z.boolean(), }).strict().refine(value => new Set(value.pressures.map(row => row.identity)).size === value.pressures.length); export function parseStoredHistory(value: string): StoredHistory { @@ -51,7 +51,7 @@ export function createStorageHistory(db: Database, brainPath: string, options: { const brain = storageBrainKey(brainPath), now = options.now ?? Date.now; const initial = emptyHistory(); let pending: Pending | null = null, unavailable: string | null = null, closed = false; - let pendingWrites: StorageWriteReport[] = [], interrupted = false; + let pendingWrites: StorageWriteReport[] = [], interrupted = false, ownerOverflow = false; // Constructing the store never collects filesystem inventory. Failure here is // a capability result; it cannot prevent existing recovery operations booting. const initialize = () => db.query("INSERT OR IGNORE INTO storage_history (brain, payload_json) VALUES (?, ?)").run(brain, JSON.stringify(initial)); @@ -85,10 +85,11 @@ export function createStorageHistory(db: Database, brainPath: string, options: { const previous = readStoredHistory(db, brain); if (!previous) throw Error("history-unavailable"); const state = previous.metadata.state; + const ownerEvidenceLost = ownerOverflow || state.paused === "history-owner-evidence-budget"; if (time < state.lastPersistedAt || at < state.lastObservedAt || at > time) throw Error("history-clock-reversal"); if (time - state.lastPersistedAt < HISTORY_LIMITS.writeInterval) return false; let next: StoredHistory; - if (interrupted) state.pressures = state.pressures.map(value => ({ ...value, recoveryAt: null, supportAt: null })); + if (interrupted || ownerEvidenceLost) state.pressures = state.pressures.map(value => ({ ...value, recoveryAt: null, supportAt: null })); if (candidate?.kind === "aggregate") { next = retainAggregate(previous, candidate.value); if (interrupted || state.paused) for (const sample of next.samples) if (sample[3] === candidate.value.at) sample[2] = 1; @@ -100,7 +101,13 @@ export function createStorageHistory(db: Database, brainPath: string, options: { const pressures = state.pressures.filter(value => value.severity !== "observed" || incoming.some(([identity]) => identity === value.identity)) .map(value => value.severity === "observed" ? newPressure(value.identity, incoming.find(([identity]) => identity === value.identity)![1]) : value); for (const [identity, rootScope] of incoming) if (!pressures.some(value => value.identity === identity) && pressures.length < 4) pressures.push(newPressure(identity, rootScope)); - next.metadata.state.pressures = pressures.map(value => observePressure(value, capacityEvidence(evaluation, value.identity, time), time)); + next.metadata.state.pressures = pressures.map(value => { + const observed = observePressure(value, capacityEvidence(evaluation, value.identity, time), time); + // Fresh direct observations can escalate pressure, but cannot prove + // recovery of an owner refusal whose identity was not retained. + if (ownerEvidenceLost) observed.recoveryAt = null; + return observed; + }); next.metadata.state.scopeLimit = candidate.value.reason === 3 || incoming.some(([identity]) => !pressures.some(value => value.identity === identity)); next.metadata.state.paused = null; } else if (candidate) { @@ -124,6 +131,7 @@ export function createStorageHistory(db: Database, brainPath: string, options: { changed.at = Math.max(changed.at, pressure.at); next.metadata.state.pressures[index] = changed; } + if (ownerEvidenceLost) next.metadata.state.paused = "history-owner-evidence-budget"; next.metadata.state.lastPersistedAt = time; if (candidate) { const observedAt = candidate.kind === "aggregate" ? candidate.value.at : candidate.at; @@ -136,7 +144,7 @@ export function createStorageHistory(db: Database, brainPath: string, options: { db.query("UPDATE storage_history SET payload_json = ?, revision = revision + 1 WHERE brain = ?").run(JSON.stringify(next), brain); return true; }).immediate(); - if (changed) { if (pending === candidate) pending = null; if (pendingWrites === writes) pendingWrites = []; interrupted = false; unavailable = null; } + if (changed) { if (pending === candidate) pending = null; if (pendingWrites === writes) pendingWrites = []; interrupted = false; ownerOverflow = false; unavailable = null; } return changed; } catch (error) { interrupted = true; @@ -151,9 +159,19 @@ export function createStorageHistory(db: Database, brainPath: string, options: { ownerWrite(report: StorageWriteReport) { if (closed) return; if (![report.identity, report.rootScope, report.target].every(value => /^[a-f0-9]{32}$/.test(value)) || !Number.isSafeInteger(report.at) || report.at < 0 || report.at > now()) throw Error("history-write-report-invalid"); - if (report.at < (read()?.metadata.state.lastPersistedAt ?? 0)) { interrupted = true; unavailable = "history-clock-reversal"; return; } - const next = [...pendingWrites, report]; - if (jsonBytes(next) > HISTORY_LIMITS.metadataBytes) { unavailable = "history-owner-evidence-budget"; return; } + const stored = read(); + if (report.at < (stored?.metadata.state.lastPersistedAt ?? 0)) { interrupted = true; unavailable = "history-clock-reversal"; return; } + const same = (value: StorageWriteReport) => value.identity === report.identity && value.rootScope === report.rootScope && value.target === report.target && value.resource === report.resource; + // Success is evidence only for an existing matching refusal. Repeated + // successes must not consume the bounded space reserved for failures. + if (report.outcome === "success" && !pendingWrites.some(value => same(value) && value.outcome === "no-space") + && !stored?.metadata.state.pressures.some(value => value.identity === report.identity && value.rootScope === report.rootScope + && value.refusals.some(refusal => refusal.target === report.target && refusal.resource === report.resource))) return; + const next = [...pendingWrites.filter(value => !same(value) || value.at > report.at + || report.outcome === "success" && value.outcome === "no-space"), report]; + if (jsonBytes(next) > HISTORY_LIMITS.metadataBytes) { + ownerOverflow = true; interrupted = true; unavailable = "history-owner-evidence-budget"; return; + } pendingWrites = next; flush(); }, status() { @@ -175,7 +193,7 @@ export function invalidateRestoredStorage(db: Database, at: number): void { state.suppressedEpisodes = state.pressures.flatMap(value => [value.episode, value.resolvedEpisode].filter((id): id is string => id !== null)); state.pressures = state.pressures.map(value => ({ ...value, recoveryAt: null, supportAt: null })); state.lastPersistedAt = Math.max(state.lastPersistedAt, at); - state.paused = "collection-unavailable"; + if (state.paused !== "history-owner-evidence-budget") state.paused = "collection-unavailable"; validateHistory(payload); db.query("UPDATE storage_history SET payload_json = ?, revision = revision + 1 WHERE brain = ?").run(JSON.stringify(payload), row.brain); } diff --git a/packages/ui-server/tests/storage-history-store.test.ts b/packages/ui-server/tests/storage-history-store.test.ts index 891ff9b52b..a552dc9362 100644 --- a/packages/ui-server/tests/storage-history-store.test.ts +++ b/packages/ui-server/tests/storage-history-store.test.ts @@ -139,3 +139,29 @@ test("a failed bucket stays a comparison gap after history reopens before its fi recovered.offer(aggregate(start + RECENT + 60_000)); expect(recovered.read()!.samples.find(row => row[1] === 0 && row[3] === start + RECENT + 60_000)?.[2], "STORAGE_DURABLE_FAILURE_BREAKS_COMPARISON").toBe(1); }); + +test("repeated successful owner reports cannot hide the next refusal", () => { + const f = fixture(); f.history.offer(aggregate(start)); f.advance(start + 1000); + const report = { identity, rootScope: scope, target: hashScope("Odysseus retained attachment"), resource: "unknown" as const, outcome: "success" as const, at: start + 1000 }; + for (let index = 0; index < 500; index++) f.history.ownerWrite(report); + f.history.ownerWrite({ ...report, outcome: "no-space" }); + f.advance(start + 60_000); f.history.flush(); + expect(f.history.read()!.metadata.state.pressures[0], "STORAGE_OWNER_BURST_PRESERVES_REFUSAL") + .toMatchObject({ severity: "critical", refusals: [{ target: report.target, successAt: null }] }); + expect(f.history.status().paused).toBeNull(); +}); + +test("unrecordable owner evidence remains unknown after flush, restart and fresh capacity", () => { + const f = fixture(); f.history.offer(aggregate(start)); f.advance(start + 1000); + for (let index = 0; index < 500; index++) f.history.ownerWrite({ identity, rootScope: scope, + target: hashScope(`Odysseus attachment ${index}`), resource: "unknown", outcome: "no-space", at: start + 1000 }); + expect(f.history.status().paused).toBe("history-owner-evidence-budget"); + f.advance(start + 60_000); f.history.flush(); f.history.close(); + invalidateRestoredStorage(f.db, start + 90_000); + let time = start + 120_000; + const reopened = createStorageHistory(f.db, f.root, { now: () => time }); + for (let index = 0; index < 3; index++) { reopened.offer(aggregate(time)); time += 60_000; } + expect(reopened.status().paused, "STORAGE_OWNER_OVERFLOW_STAYS_UNKNOWN").toBe("history-owner-evidence-budget"); + expect(reopened.read()!.metadata.state.pressures[0]?.severity).toBe("critical"); + reopened.close(); +}); From 503a6342d69c8a20d5b5a1432d6156dae434c371 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 19:49:45 +0200 Subject: [PATCH 20/44] test(ui-server): retain provider model identity in registry expectations --- packages/ui-server/tests/backend-registry.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ui-server/tests/backend-registry.test.ts b/packages/ui-server/tests/backend-registry.test.ts index 259469dd26..2de57132b8 100644 --- a/packages/ui-server/tests/backend-registry.test.ts +++ b/packages/ui-server/tests/backend-registry.test.ts @@ -79,6 +79,7 @@ describe("backend registry", () => { id: "claude", label: "Claude", vendor: "anthropic", + model: "claude-opus-5-5", source: "builtin", thinkingLevel: "medium", supportedThinkingLevels: ["low", "medium", "high", "xhigh", "max"], @@ -333,7 +334,7 @@ describe("configured GPT-6 profiles", () => { const providers = createProviderRoutes({ registry }); const models = createModelRoutes({ registry, db }); const expected = profiles.map(({ id, label, vendor, model, thinkingLevel }) => ({ - id, label, vendor, thinkingLevel, backendId: "pi", + id, label, vendor, model, thinkingLevel, backendId: "pi", supportedThinkingLevels: [ ...(["gpt-5.6-sol", "gpt-6-sol", "gpt-6-luna"].includes(model) ? ["off"] : []), ...(vendor === "openai-codex" ? ["minimal"] : []), From c6c23c079a4c10ff56b4035e23d1cda7fddbb997 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 19:57:46 +0200 Subject: [PATCH 21/44] test(ui-sdk): account for the Btrfs filesystem filter prefix --- packages/ui-sdk/tests/worker-limits.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ui-sdk/tests/worker-limits.test.ts b/packages/ui-sdk/tests/worker-limits.test.ts index fdae8b6fdb..93dd7751ab 100644 --- a/packages/ui-sdk/tests/worker-limits.test.ts +++ b/packages/ui-sdk/tests/worker-limits.test.ts @@ -4,8 +4,9 @@ import { launchFixtureAgentWorker as launchAgentWorker } from "./helpers/worker- * prlimit. Each fixture runs inside a worker and is observed from outside. */ import { describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, statfsSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, statfsSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; +import { WORKER_FILESYSTEM_FILTER_PROGRAM } from "../src/server/worker-filesystem-filter"; import { workerCommand, WorkerHostError } from "../src/server/worker-launcher"; import { cgroupMemoryCeiling, defaultWorkerLimits, effectiveRlimits, parseWorkerLimits, PRLIMIT_REQUIREMENT, prlimitArgs, ROOT_NPROC_REASON, @@ -118,7 +119,11 @@ describe("configuration", () => { const argv = workerCommand({ brainPath: f.brain, command: ["/bin/true", "odysseus"], env: {}, limits }); expect(argv.slice(argv.lastIndexOf("--") - 5)).toEqual(["/usr/bin/prlimit", "--nproc=64:64", `--data=${GIB}:${GIB}`, `--fsize=${MIB}:${MIB}`, "--nofile=256:256", "--", "/bin/true", "odysseus"]); - expect(argv[0]).toEndWith("bwrap"); + const bwrap = Bun.which("bwrap")!; + const prefix = statfsSync(f.brain).type === 0x9123683e + ? [realpathSync(Bun.which("python3")!), "-I", "-S", "-c", WORKER_FILESYSTEM_FILTER_PROGRAM, bwrap] + : [bwrap]; + expect(argv.slice(0, prefix.length), "LIMITS_TRUSTED_FILESYSTEM_PREFIX").toEqual(prefix); } finally { f.cleanup(); } }); }); From 67df2779ebd68cc9645b27053d9f4519d0d260a2 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 19:57:46 +0200 Subject: [PATCH 22/44] test(ui-server): preserve completed-response usage through cancellation --- packages/ui-server/tests/claude-cancellation.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ui-server/tests/claude-cancellation.test.ts b/packages/ui-server/tests/claude-cancellation.test.ts index 7b26a05ade..b702f5b6be 100644 --- a/packages/ui-server/tests/claude-cancellation.test.ts +++ b/packages/ui-server/tests/claude-cancellation.test.ts @@ -39,6 +39,10 @@ for await (const line of createInterface({ input: process.stdin })) { send({ type: "assistant", session_id: session, uuid: "odysseus-command", parent_tool_use_id: null, message: { role: "assistant", content: [{ type: "tool_use", id: "loom", name: "Bash", input: { command: "sleep 90", run_in_background: false } }] } }); } else { + const stream = event => send({ type: "stream_event", session_id: session, uuid: "odysseus-usage", parent_tool_use_id: null, event }); + stream({ type: "message_start", message: { id: "odysseus-response", model: "claude-sonnet-5-5", role: "assistant", content: [], usage: { input_tokens: 17, output_tokens: 0, cache_creation_input_tokens: 2, cache_read_input_tokens: 3 } } }); + stream({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 9 } }); + stream({ type: "message_stop" }); send({ type: "result", subtype: "success", session_id: session, uuid: "odysseus-result", is_error: mode === "error-before-cancel", duration_ms: 123, duration_api_ms: 1, num_turns: 1, result: mode === "error-before-cancel" ? "The model call failed." : "", errors: [], total_cost_usd: 0.25, usage: { input_tokens: 17, output_tokens: 9, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, modelUsage: { "claude-sonnet-5-5": { inputTokens: 17, outputTokens: 9, cacheReadInputTokens: 3, cacheCreationInputTokens: 2, costUSD: 0.25 } }, permission_denials: [] }); @@ -107,7 +111,9 @@ for (const mode of ["success-before-cancel", "error-before-cancel", "no-result", expect(result.outcome, "HOST_CANCEL_OUTRANKS_DELAYED_SDK_TERMINAL").toBe(expected); expect(result.isError).toBe(false); expect(result.failure).toBeUndefined(); expect(result.outcomeDetail).toBeUndefined(); - if (mode !== "no-result") expect(result, "CANCELLATION_RETAINS_REPORTED_USAGE").toMatchObject({ costUsd: 0.25, durationMs: 123, numTurns: 1, usage: { inputTokens: 17, outputTokens: 9, cacheReadTokens: 3, cacheCreationTokens: 2, perModel: { "claude-sonnet-5-5": { costUsd: 0.25 } } } }); + if (mode !== "no-result") expect(result, "CANCELLATION_RETAINS_REPORTED_USAGE").toMatchObject({ durationMs: 123, numTurns: 1, usage: { inputTokens: 17, outputTokens: 9, cacheReadTokens: 3, cacheCreationTokens: 2, perModel: { "claude-sonnet-5-5": { inputTokens: 17, outputTokens: 9, cacheReadTokens: 3, cacheCreationTokens: 2 } } } }); + expect(result.costUsd, "CUMULATIVE_COST_IS_NOT_TURN_USAGE").toBeUndefined(); + expect(result.usage?.perModel?.["claude-sonnet-5-5"]?.costUsd, "CUMULATIVE_MODEL_COST_IS_NOT_TURN_USAGE").toBeUndefined(); expect(host.activity!.store.snapshotRun(result.turnId!)!.spans.find(span => span.kind === "turn")?.outcome, "ACTIVITY_MATCHES_WIRE").toBe(expected); const journal = host.journal!.details(SESSION, result.turnId!); expect(journal?.summary.state, "FINAL_JOURNAL_RETAINED").toBe("complete"); From 7ee5f620f1d1be12dda2cfeec11425f62920662d Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:16:23 +0200 Subject: [PATCH 23/44] test(ui): exercise real renderer bootstrap in render fixtures (#2201) --- .../tests/render/dictation-provider-end.test.tsx | 13 +++++++------ .../ui-react/tests/render/render-smoke.test.tsx | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/ui-react/tests/render/dictation-provider-end.test.tsx b/packages/ui-react/tests/render/dictation-provider-end.test.tsx index 2fc0c8f832..bf0a107afa 100644 --- a/packages/ui-react/tests/render/dictation-provider-end.test.tsx +++ b/packages/ui-react/tests/render/dictation-provider-end.test.tsx @@ -18,7 +18,6 @@ import type { VoiceSessionResponse } from "@schlessera/brain-ui-sdk/client"; import { BrainUiProvider } from "../../src/root-context.js"; import { createBrainUiRoot, type BrainUiRoot } from "../../src/root.js"; -import type { BrainApi } from "../../src/lib/api-client.js"; import { useVpnStatus } from "../../src/hooks/use-vpn-status.js"; import type { LocalWork } from "../../src/lib/local-work.js"; import { useDictation } from "../../src/voice/use-dictation.js"; @@ -189,11 +188,13 @@ function voiceRoot(provider: Provider) { expiresAt: Date.now() + 60_000, capabilities, }; - const api = { - voiceSession: async () => session, - voiceOverrides: async () => ({ overrides: [] }), - } as unknown as BrainApi; - const root = createBrainUiRoot({ storage: null, api }); + const root = createBrainUiRoot({ storage: null, request: async input => { + const path = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, "http://localhost").pathname; + if (path === "/api/voice/session") return Response.json(session); + if (path === "/api/voice/overrides") return Response.json({ overrides: [] }); + if (path === "/api/status") return Response.json({ contentRenderers: [] }); + throw new Error(`Unexpected dictation fixture request: ${path}`); + } }); roots.push(root); return root; } diff --git a/packages/ui-react/tests/render/render-smoke.test.tsx b/packages/ui-react/tests/render/render-smoke.test.tsx index 4efa596bd7..39d76078bf 100644 --- a/packages/ui-react/tests/render/render-smoke.test.tsx +++ b/packages/ui-react/tests/render/render-smoke.test.tsx @@ -6776,10 +6776,11 @@ test("code without an info string has no invented language or sample or copy gly expect(view.container.textContent).toBe(""); expect(view.container.querySelectorAll("svg")).toHaveLength(0); }); -test("Mermaid keeps its renderer instead of a code card", () => { +test("Mermaid keeps its renderer instead of a code card", async () => { const root = createBrainUiRoot({ storage: null }); try { const view = render({"graph LR; Troy-->Ithaca\n"}); + await waitFor(() => expect(view.container.querySelector('[data-content-renderer="mermaid"]'), "MERMAID_LAZY_RENDERER_SETTLED").toBeTruthy()); expect(view.container.querySelectorAll('[data-kit-code-block]')).toHaveLength(0); expect(view.container.textContent).toContain("graph LR; Troy-->Ithaca"); } finally { root.dispose(); } From 0bc577042aa402a99a490d933c2409c61b88996c Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:23:38 +0200 Subject: [PATCH 24/44] test(storage): account for guidance requests and the Settings tab --- packages/ui-react/tests/render/render-smoke.test.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ui-react/tests/render/render-smoke.test.tsx b/packages/ui-react/tests/render/render-smoke.test.tsx index 39d76078bf..1bc98244cf 100644 --- a/packages/ui-react/tests/render/render-smoke.test.tsx +++ b/packages/ui-react/tests/render/render-smoke.test.tsx @@ -3115,14 +3115,14 @@ describe("activity and device root ownership", () => { const panel = (root: BrainUiRoot) => ; const view = render(panel(a.root)); try { - await act(async () => { a.requests[0].response.resolve(principalResponse([])); await flushPromises(); }); + await act(async () => { a.matching("/auth/principals")[0].response.resolve(principalResponse([])); await flushPromises(); }); changeControlledInput(view.getByLabelText("Label") as HTMLInputElement, "Alpha helper"); fireEvent.click(view.getByRole("button", { name: "Create agent credential" })); - expect(a.requests[1].init?.method).toBe("POST"); + expect(a.matching("/auth/principals")[1].init?.method).toBe("POST"); view.rerender(panel(b.root)); await act(async () => { - b.requests[0].response.resolve(principalResponse([])); - a.requests[1].response.resolve(Response.json({ id: "issued", label: "Alpha helper", expiresAt: 1, cookie: "alpha-once" })); + b.matching("/auth/principals")[0].response.resolve(principalResponse([])); + a.matching("/auth/principals")[1].response.resolve(Response.json({ id: "issued", label: "Alpha helper", expiresAt: 1, cookie: "alpha-once" })); await flushPromises(); }); expect(a.root.stores.principal.getState().oneTimeCredential?.cookie).toBe("alpha-once"); @@ -4752,7 +4752,7 @@ describe("desktop panes (D37)", () => { const onClose = mock(() => {}); const view = render(); const tabs = view.getAllByRole("tab").map((t) => t.getAttribute("aria-label") ?? t.textContent); - expect(tabs).toEqual(["Appearance & input", "Models", "Accounts", "Skills", "Modules", "Security", "Devices & agents"]); + expect(tabs).toEqual(["Appearance & input", "Models", "Accounts", "Skills", "Modules", "Storage", "Security", "Devices & agents"]); fireEvent.click(view.getByRole("tab", { name: "Appearance & input" })); expect(useUIStore.getState().settingsTab).toBe("appearance"); expect(view.getByRole("radiogroup", { name: "Theme" })).toBeTruthy(); From 92874e59c0086eb7ad77669dfbf11c4bb2d366e9 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:23:17 +0200 Subject: [PATCH 25/44] test: retain admission and approved remedy contract receipts (#2204, #2205) --- packages/ui-server/tests/run-session.test.ts | 5 ++++- tests/bridge-tools.test.ts | 2 +- tests/fixtures/bridge-tools.after.json | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/ui-server/tests/run-session.test.ts b/packages/ui-server/tests/run-session.test.ts index 3c68160577..2525c67bbc 100644 --- a/packages/ui-server/tests/run-session.test.ts +++ b/packages/ui-server/tests/run-session.test.ts @@ -625,6 +625,8 @@ describe("handleChatMessage concurrency", () => { sessionId: "session-2", attachments: [], }); + expect(routingCalls, "ADMISSION_ONE_ROUTING_READ_WHILE_RESERVED").toBe(1); + expect(host.coordinator.startingSessions, "ADMISSION_RESERVATION_HELD").toBe(1); releaseRouting.resolve(); if (routingCalls > 0) await backendStarted.promise; await until( @@ -632,7 +634,8 @@ describe("handleChatMessage concurrency", () => { host.coordinator.startingSessions === 0 && host.coordinator.running.size === 0 ); - expect(routingCalls).toBe(1); + // Target selection reads profiles first; billing resolves the admitted provider later. + expect(routingCalls, "routing plus admitted billing profile snapshot").toBe(2); expect(reservationWhileRouting).toBe(1); expect(prompts).toEqual(["admitted"]); expect(sent.filter((message) => message.type === "error")).toEqual([ diff --git a/tests/bridge-tools.test.ts b/tests/bridge-tools.test.ts index b776999815..91a8756287 100644 --- a/tests/bridge-tools.test.ts +++ b/tests/bridge-tools.test.ts @@ -329,7 +329,7 @@ describe("bridge tool adapter identity", () => { } }); - test("Claude preserves migration receipts except the additive roadblocks definition", async () => { + test("Claude preserves migration receipts except the additive roadblocks/remedy definition", async () => { const before = readSnapshot("before"); const after = readSnapshot("after"); for (const name of SNAPSHOT_NAMES) { diff --git a/tests/fixtures/bridge-tools.after.json b/tests/fixtures/bridge-tools.after.json index 9615559f65..ac618eaad0 100644 --- a/tests/fixtures/bridge-tools.after.json +++ b/tests/fixtures/bridge-tools.after.json @@ -15,7 +15,7 @@ "rootSymlinkResultSha256": "de608613847303e2a2b5fd48ede3be912ed65e7b263767bba1577930bc308513" }, "query_activity": { - "definitionSha256": "b9adf30de97f533f17b2cd9cb159d93cc9567e4e3c519d8616c75868f92749af", + "definitionSha256": "4ee65805488e221af8bacc939c078713892636293bc2d70e18093f9e1ac7e5da", "resultSha256": "1b00ed051a07f010014a7e1ae4b0aa218dc009066c708a0c29641908f74cf6b9" } }, @@ -35,7 +35,7 @@ "rootSymlinkResultSha256": "e07c772f128b25862c280a212f62e16c87d48b0ed94244b24e480a1e2a01165c" }, "query_activity": { - "definitionSha256": "6d7230eae7045690515836a5c5bfd4496d984157fa0eced1a5494e7b632d7b62", + "definitionSha256": "cdb3b79a5f3865558c1ac0919c6a9507b2699d4051c0cf438acb2c506312ad06", "resultSha256": "78d8e4c778d4261ed9bb09197a728f44566bd3ca99858834211ee2384f5f52bb" } } From ef3f420301863b50679f89f32b137330fd01e9e3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:22:26 +0200 Subject: [PATCH 26/44] CONTRACT: fix Activity association for late session identities (#2203) --- .changeset/late-activity-session.md | 5 +++ api-report/ui-server.txt | 1 + docs/decisions/async-collaboration.md | 2 +- docs/decisions/claude-code-runtime.md | 2 +- docs/decisions/cost-tracking.md | 2 +- docs/decisions/governed-writable-workers.md | 2 +- docs/decisions/voice-permission.md | 2 +- docs/integration-contract/package-api.md | 9 +++++ .../plans/async-collaboration-requirements.md | 4 +-- docs/plans/async-collaboration.md | 6 ++-- packages/ui-server/src/activity/recorder.ts | 3 +- packages/ui-server/src/activity/sql.ts | 4 +-- packages/ui-server/src/activity/store.ts | 9 +++-- .../ui-server/tests/activity-store.test.ts | 35 +++++++++++++++++++ 14 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 .changeset/late-activity-session.md diff --git a/.changeset/late-activity-session.md b/.changeset/late-activity-session.md new file mode 100644 index 0000000000..da7e294b2b --- /dev/null +++ b/.changeset/late-activity-session.md @@ -0,0 +1,5 @@ +--- +"@schlessera/brain-ui-server": minor +--- + +Persist a late backend session identity on its active Activity span so session queries and run summaries include turns whose first frame preceded session identification. Refuse replacement of an established identity. diff --git a/api-report/ui-server.txt b/api-report/ui-server.txt index 5668e8caed..7fde4befbb 100644 --- a/api-report/ui-server.txt +++ b/api-report/ui-server.txt @@ -117,6 +117,7 @@ signatures (every public export and the types it is made of) startSpan(input: StartSpanInput): SpanRow; endSpan(spanId: string, input: EndSpanInput): boolean; patchSpan(spanId: string, patch: { + sessionId?: string; principalId?: string; attrs?: Record; usage?: SpanUsage; diff --git a/docs/decisions/async-collaboration.md b/docs/decisions/async-collaboration.md index 7fa7b339cb..cb2252f15d 100644 --- a/docs/decisions/async-collaboration.md +++ b/docs/decisions/async-collaboration.md @@ -241,7 +241,7 @@ suppressed-per-model Action; unknown usage cannot become a guessed zero. Subscription billing moves the turn counter even when spend is zero. The activity sum exposes unpriced runs (`sumEffectiveCost`, `packages/ui-server/src/activity/store.ts:179-194`) and settles only after execution -(`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:650`). +(`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:651`). Retrospective totals alone cannot enforce admission. The maintainer's [2026-10-01 ruling on #678](https://github.com/schlessera/brain-kit/issues/678#issuecomment-5926691626) diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index cce732ad01..0e338b0440 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -468,7 +468,7 @@ into them. billing now stays unknown for every origin; server credentials and an observed runtime version cannot establish a child's billing. The recorded root-span attribute is the source of truth - (`const attrBilling`, `packages/ui-server/src/activity/store.ts:535-536`); + (`const attrBilling`, `packages/ui-server/src/activity/store.ts:537-538`); [cost-tracking.md](cost-tracking.md#recorded-billing-is-the-rollup-source-of-truth-293) records the 2026-09-30 ruling and frozen-history semantics. - **Model discovery prefers the API key** and describes that as "mirroring the diff --git a/docs/decisions/cost-tracking.md b/docs/decisions/cost-tracking.md index 5f5fe9a834..33d79bece7 100644 --- a/docs/decisions/cost-tracking.md +++ b/docs/decisions/cost-tracking.md @@ -128,7 +128,7 @@ math and authoritative backend cost remain available independently; aggregates exclude unknown effective costs from their sum and count the runs as unpriced. Explicit valid subscription/API classifications keep their existing behavior. The rollup reads the root attribute and prices child usage for cron locally -(`const attrBilling`, `packages/ui-server/src/activity/store.ts:535-536`). +(`const attrBilling`, `packages/ui-server/src/activity/store.ts:537-538`). This is a change to the evidence used for new computations, not a historical backfill. First non-null billing, list cost, effective cost and estimate values diff --git a/docs/decisions/governed-writable-workers.md b/docs/decisions/governed-writable-workers.md index e2f01ea712..6ead78d7cd 100644 --- a/docs/decisions/governed-writable-workers.md +++ b/docs/decisions/governed-writable-workers.md @@ -1231,7 +1231,7 @@ limits from the same policy inputs and states independent hard limits. **What exists.** The audit trail is the Activity span record: tool name, the full input up to its cap, output and approval decisions -(`"tool_input"`, `packages/ui-server/src/activity/recorder.ts:286`). +(`"tool_input"`, `packages/ui-server/src/activity/recorder.ts:287`). **What changes:** diff --git a/docs/decisions/voice-permission.md b/docs/decisions/voice-permission.md index 92c1976240..aa71a53ce6 100644 --- a/docs/decisions/voice-permission.md +++ b/docs/decisions/voice-permission.md @@ -460,7 +460,7 @@ An approval given by voice must be as reviewable afterwards as one given by tapping a card. Most of that already exists: every decision is written as an append-only `approval_decision` event carrying the principal, the decision and the request kind, and it patches the span -(`onApprovalDecision`, `packages/ui-server/src/activity/recorder.ts:580-595`), +(`onApprovalDecision`, `packages/ui-server/src/activity/recorder.ts:581-596`), fed from the bridge's `recorded()` wrapper (`const recorded`, `packages/ui-server/src/ws/bridge.ts:333-348`). diff --git a/docs/integration-contract/package-api.md b/docs/integration-contract/package-api.md index a93965d2b9..f0cec98ec5 100644 --- a/docs/integration-contract/package-api.md +++ b/docs/integration-contract/package-api.md @@ -3500,3 +3500,12 @@ private pending receipt before invoking the closure, then publishes a steering change only after the save succeeds. Failed or interrupted finalization remains partial/unconfirmed. Other backends need not implement Claude native-memory persistence; this adds no agent history authority or new public request route. + +### Late Activity session association + +`ActivityStore.patchSpan` accepts an optional `sessionId` to bind an unnamed +active span when the backend reports its session after the first frame. Repeating +the same identity is allowed; changing an established identity or patching a +terminal span returns false without modifying the row. The recorder persists +this identity in both the queryable session column and the existing attributes. +This additive API ships in a minor release. diff --git a/docs/plans/async-collaboration-requirements.md b/docs/plans/async-collaboration-requirements.md index 5ff3116e5b..ab2fc16b91 100644 --- a/docs/plans/async-collaboration-requirements.md +++ b/docs/plans/async-collaboration-requirements.md @@ -150,7 +150,7 @@ Stated before the requirements because five of them derive from it. thread_id, item_id, seq)` **carries `thread_id` explicitly** — activity stores its scope in the change row for exactly this reason ((`CREATE TABLE IF NOT EXISTS activity_changes (`, `packages/ui-server/migrations/007_activity.sql:58-66`), - (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:872-901`)), and deriving scope through a join on a + (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:877-906`)), and deriving scope through a join on a mutable item is weaker and breaks tombstones. `change_id` is the global cursor, `seq` is per-thread order. Append-only checkpoints, unique resolution records, scheduler heartbeats and budget reservations also need explicit durable storage. The four @@ -343,7 +343,7 @@ Stated before the requirements because five of them derive from it. policy proposals, and discuss-session digest construction when those v2 features ship. Revision 1's "T2 is the entire bill" was false. - R41. **Budgets are enforced by reservation at claim time, not by summing history.** - `rollupRun` runs in `finish()` (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:650`), so cost exists only after a + `rollupRun` runs in `finish()` (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:651`), so cost exists only after a run ends: two runs can both start under the cap and finish over it, and unknown effective costs are excluded from the sum (`export function sumEffectiveCost(`, `packages/ui-server/src/activity/store.ts:179-194`) so the query **fails open**. Required: transactional reservations on claim, in-flight reservations counted, settlement at diff --git a/docs/plans/async-collaboration.md b/docs/plans/async-collaboration.md index a5a81df74e..ec4cbd1557 100644 --- a/docs/plans/async-collaboration.md +++ b/docs/plans/async-collaboration.md @@ -125,7 +125,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. ### Relevant Code and Patterns - **Cursors and transactional snapshots:** (`CREATE TABLE IF NOT EXISTS activity_changes (`, `packages/ui-server/migrations/007_activity.sql:58-66`) - and (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:872-901`). Reuse the algorithm; the + and (`snapshotRun(runId) {`, `packages/ui-server/src/activity/store.ts:877-906`). Reuse the algorithm; the activity stream is bound to activity store methods and frame types. - **Runtime lifecycle:** (`export function createActivityRuntime(`, `packages/ui-server/src/activity/runtime.ts:55-194`). Cron's scheduler owns manual triggers/history; it does not tick the inbox. @@ -146,7 +146,7 @@ system gate. It is not a prerequisite to rebuild or a new independent package. - **Pi resources and extension gate:** (`export function createSessionResources(`, `packages/ui-backend-pi/src/session-resources.ts:29-144`) and (`export function createPermissionGate(`, `packages/ui-backend-pi/src/permission-gate.ts:95-124`). Built-ins are disabled, but ambient resources/extensions and in-process execution still need containment. -- **Cost timing:** (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:650`) settles after execution; +- **Cost timing:** (`store.rollupRun(runId);`, `packages/ui-server/src/activity/recorder.ts:651`) settles after execution; (`export function sumEffectiveCost(`, `packages/ui-server/src/activity/store.ts:179-194`) explicitly counts unpriced runs. Reservations must cover in-flight work, not only this retrospective sum. - **Notifications:** (`CREATE TABLE IF NOT EXISTS notification_intents (`, `packages/ui-server/migrations/007_activity.sql:99-114`), @@ -717,7 +717,7 @@ recovered tick after killing the interval model-bearing operation; folded compaction is included in its existing call. **Patterns to follow:** the frozen-at-first-computation discipline of effective cost -(`function rollupRunInTx(runId: string) {`, `packages/ui-server/src/activity/store.ts:516-618`); env descriptor array for the new configuration values +(`function rollupRunInTx(runId: string) {`, `packages/ui-server/src/activity/store.ts:518-620`); env descriptor array for the new configuration values **Test scenarios:** - Covers AE11: two claims that would each fit but jointly exceed the cap — the second is diff --git a/packages/ui-server/src/activity/recorder.ts b/packages/ui-server/src/activity/recorder.ts index adc1231436..f00f6ae55a 100644 --- a/packages/ui-server/src/activity/recorder.ts +++ b/packages/ui-server/src/activity/recorder.ts @@ -241,11 +241,12 @@ export function createTurnRecorder( turn.lifecycle?.firstFrame(rootStarted ? runId : null); switch (msg.type) { case "session_info": { + if (sessionId && sessionId !== msg.sessionId) return; sessionId = msg.sessionId; turn.lifecycle?.bindSession(msg.sessionId); ensureRoot(); // The root may have been opened before the session had a name. - store.patchSpan(rootSpanId, { attrs: { "session.id": msg.sessionId } }); + store.patchSpan(rootSpanId, { sessionId: msg.sessionId, attrs: { "session.id": msg.sessionId } }); onWrite?.(); break; } diff --git a/packages/ui-server/src/activity/sql.ts b/packages/ui-server/src/activity/sql.ts index 8b5b54778b..b2c9a72be3 100644 --- a/packages/ui-server/src/activity/sql.ts +++ b/packages/ui-server/src/activity/sql.ts @@ -4,7 +4,7 @@ export const ACTIVITY_SQL = { insertChange: "INSERT INTO activity_changes (run_id, seq, span_id, event_index) VALUES (?, ?, ?, ?)", spanById: "SELECT * FROM activity_spans WHERE span_id = ?", - spanStateById: "SELECT run_id, outcome, attrs FROM activity_spans WHERE span_id = ?", + spanStateById: "SELECT run_id, outcome, attrs, session_id FROM activity_spans WHERE span_id = ?", endSpan: `UPDATE activity_spans SET outcome = ?, outcome_reason = ?, ended_at = ?, attrs = ?, input_tokens = COALESCE(?, input_tokens), output_tokens = COALESCE(?, output_tokens), @@ -52,7 +52,7 @@ export const ACTIVITY_SQL = { (span_id, run_id, parent_span_id, name, kind, origin, session_id, job_name, principal_id, attrs, started_at, writer, last_heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - patchSpan: `UPDATE activity_spans SET principal_id = COALESCE(?, principal_id), + patchSpan: `UPDATE activity_spans SET session_id = COALESCE(session_id, ?), principal_id = COALESCE(?, principal_id), attrs = ?, wait_until = COALESCE(?, wait_until), input_tokens = COALESCE(?, input_tokens), output_tokens = COALESCE(?, output_tokens), diff --git a/packages/ui-server/src/activity/store.ts b/packages/ui-server/src/activity/store.ts index 38bc4a148f..82e14a9cd7 100644 --- a/packages/ui-server/src/activity/store.ts +++ b/packages/ui-server/src/activity/store.ts @@ -307,6 +307,8 @@ export interface ActivityStore { patchSpan( spanId: string, patch: { + /** Bind an unnamed active span once; a different established identity is refused. */ + sessionId?: string; principalId?: string; attrs?: Record; usage?: SpanUsage; @@ -475,7 +477,7 @@ export function createActivityStore( function endSpanInTx(spanId: string, input: EndSpanInput): boolean { const existing = db .query(ACTIVITY_SQL.spanStateById) - .get(spanId) as { run_id: string; outcome: string | null; attrs: string | null } | null; + .get(spanId) as { run_id: string; outcome: string | null; attrs: string | null; session_id: string | null } | null; if (!existing || existing.outcome !== null) return false; const endedAt = input.endedAt ?? Date.now(); @@ -692,13 +694,16 @@ export function createActivityStore( return inWrite(() => { const existing = db .query(ACTIVITY_SQL.spanStateById) - .get(spanId) as { run_id: string; outcome: string | null; attrs: string | null } | null; + .get(spanId) as { run_id: string; outcome: string | null; attrs: string | null; session_id: string | null } | null; if (!existing || existing.outcome !== null) return false; + if (patch.sessionId !== undefined && (typeof patch.sessionId !== "string" || !patch.sessionId + || (existing.session_id !== null && existing.session_id !== patch.sessionId))) return false; const attrs = patch.attrs ? JSON.stringify({ ...(safeParse(existing.attrs) ?? {}), ...patch.attrs }) : existing.attrs; const u = patch.usage ?? {}; db.query(ACTIVITY_SQL.patchSpan).run( + patch.sessionId ?? null, patch.principalId ?? null, attrs, patch.waitUntil ?? null, diff --git a/packages/ui-server/tests/activity-store.test.ts b/packages/ui-server/tests/activity-store.test.ts index 6ac7d12d1b..0e4eef3777 100644 --- a/packages/ui-server/tests/activity-store.test.ts +++ b/packages/ui-server/tests/activity-store.test.ts @@ -3,6 +3,7 @@ import { Database } from "bun:sqlite"; import { readdirSync, readFileSync } from "fs"; import { join } from "path"; +import { createTurnRecorder } from "../src/activity/recorder.js"; import { runActivityQuery } from "../src/activity/query.js"; import { createUiDb } from "../src/db/client.js"; import { @@ -1240,3 +1241,37 @@ describe("activity store: effective cost (migration 010)", () => { expect(r.pricing_estimate).toBeNull(); }); }); + + +test("late session binding persists once and refuses reassignment or terminal writes", () => { + const {store, db} = freshStore(); + try { + store.startSpan({spanId:"early-root", runId:"early", name:"invoke_agent", kind:"turn", origin:"session"}); + startTurn(store,"other","athena-session"); + expect(store.patchSpan("early-root",{sessionId:"ithaca-session"})).toBe(true); + expect(store.getSpan("early-root")?.sessionId,"LATE_SESSION_PERSISTED").toBe("ithaca-session"); + expect(store.patchSpan("early-root",{sessionId:"ithaca-session"})).toBe(true); + expect(store.patchSpan("early-root",{sessionId:"other-session",attrs:{unexpected:true}}),"SESSION_IDENTITY_WRITE_ONCE").toBe(false); + expect(store.getSpan("early-root")?.attrs.unexpected).toBeUndefined(); + store.endSpan("early-root",{outcome:"success"}); + store.rollupRun("early"); + expect(store.patchSpan("early-root",{sessionId:"other-session"})).toBe(false); + expect(db.query("SELECT session_id FROM activity_run_rollups WHERE run_id='early'").get()).toEqual({session_id:"ithaca-session"}); + expect(db.query("SELECT span_id FROM activity_spans WHERE session_id='ithaca-session'").all()).toEqual([{span_id:"early-root"}]); + expect(store.getSpan("other-root")?.sessionId).toBe("athena-session"); + } finally { db.close(); } +}); + +test("session_info binds a root opened by an earlier frame", () => { + const {store,db}=freshStore(); + try { + const recorder=createTurnRecorder({store},{turnId:"early-frame",sessionId:null}); + recorder.observeFrame({type:"tool_use_start",toolUseId:"wax",toolName:"Bash"}); + expect(store.getSpan("early-frame:turn")?.sessionId).toBeNull(); + recorder.observeFrame({type:"session_info",sessionId:"ithaca-session",isNew:true}); + recorder.observeFrame({type:"session_info",sessionId:"other-session",isNew:false}); + recorder.finish("success"); + expect(store.getSpan("early-frame:turn")?.sessionId,"RECORDER_LATE_SESSION_BOUND").toBe("ithaca-session"); + expect(db.query("SELECT session_id FROM activity_run_rollups WHERE run_id='early-frame'").get()).toEqual({session_id:"ithaca-session"}); + } finally { db.close(); } +}); From cbd7e75e2d0788ca54a24fc74d28b2325dd8e8b5 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:25:06 +0200 Subject: [PATCH 27/44] test(ui): use the declared root request URL type --- packages/ui-react/tests/render/dictation-provider-end.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui-react/tests/render/dictation-provider-end.test.tsx b/packages/ui-react/tests/render/dictation-provider-end.test.tsx index bf0a107afa..76a5bbc8d9 100644 --- a/packages/ui-react/tests/render/dictation-provider-end.test.tsx +++ b/packages/ui-react/tests/render/dictation-provider-end.test.tsx @@ -189,7 +189,7 @@ function voiceRoot(provider: Provider) { capabilities, }; const root = createBrainUiRoot({ storage: null, request: async input => { - const path = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, "http://localhost").pathname; + const path = new URL(input, "http://localhost").pathname; if (path === "/api/voice/session") return Response.json(session); if (path === "/api/voice/overrides") return Response.json({ overrides: [] }); if (path === "/api/status") return Response.json({ contentRenderers: [] }); From aa51b77039027c04d5cd602e102600efaffbda9b Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:25:30 +0200 Subject: [PATCH 28/44] test(ui): wait for composer readiness before approval keyboard send (#2202) --- packages/ui-react/tests/approval-recovery-runtime.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ui-react/tests/approval-recovery-runtime.test.ts b/packages/ui-react/tests/approval-recovery-runtime.test.ts index 37bbc4d06d..c071775cfc 100644 --- a/packages/ui-react/tests/approval-recovery-runtime.test.ts +++ b/packages/ui-react/tests/approval-recovery-runtime.test.ts @@ -107,6 +107,7 @@ describe.skipIf(!executablePath)("mounted approval recovery", () => { await connected(page); const composer = page.locator("textarea.bk-composer"); await composer.fill("Row past the Sirens"); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); await page.locator("[data-approval-card]").waitFor(); // Raised while this page watched the turn: not restored. @@ -148,6 +149,7 @@ describe.skipIf(!executablePath)("mounted approval recovery", () => { await connected(first); const composer = first.locator("textarea.bk-composer"); await composer.fill("Steer clear of Scylla"); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); await first.locator("[data-approval-card]").waitFor(); await first.reload(); From 5f0d3469e006c6ed5eb0033bbb7cb850cf7ff2a3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 20:37:02 +0200 Subject: [PATCH 29/44] fix(ui-sdk): preserve already restored executable aliases (#2211) --- .changeset/restricted-runtime-alias.md | 5 +++ docs/decisions/container-fresh-proc.md | 2 +- docs/decisions/governed-writable-workers.md | 18 +++++++---- docs/decisions/policy-write-boundary.md | 4 +-- docs/decisions/worker-writable-areas.md | 6 ++-- packages/ui-sdk/src/server/worker-launcher.ts | 32 +++++++++++++++++-- .../tests/worker-runtime-placement.test.ts | 17 +++++++++- 7 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 .changeset/restricted-runtime-alias.md diff --git a/.changeset/restricted-runtime-alias.md b/.changeset/restricted-runtime-alias.md new file mode 100644 index 0000000000..24178dc0a8 --- /dev/null +++ b/.changeset/restricted-runtime-alias.md @@ -0,0 +1,5 @@ +--- +"@schlessera/brain-ui-sdk": patch +--- + +Preserve executable aliases already supplied by restricted workers’ read-only runtime view, avoiding startup failures when a system shell or owned runtime entry is a symlink. Keep temporary runtimes and host state isolated. diff --git a/docs/decisions/container-fresh-proc.md b/docs/decisions/container-fresh-proc.md index 93dd885ad1..4c94b18428 100644 --- a/docs/decisions/container-fresh-proc.md +++ b/docs/decisions/container-fresh-proc.md @@ -22,7 +22,7 @@ does not change. /proc because the parent /proc has masked or covered paths; in Docker, run the container with --security-opt systempaths=unconfined", followed by bubblewrap's own line. Every other launch failure keeps the generic list - (`FRESH_PROC_REFUSED`, `packages/ui-sdk/src/server/worker-launcher.ts:1342`). + (`FRESH_PROC_REFUSED`, `packages/ui-sdk/src/server/worker-launcher.ts:1368`). The match is on bubblewrap's literal message, not a guess from the environment: a host that refuses for another reason is not told to change Docker options. diff --git a/docs/decisions/governed-writable-workers.md b/docs/decisions/governed-writable-workers.md index 34e2509004..81fafb723d 100644 --- a/docs/decisions/governed-writable-workers.md +++ b/docs/decisions/governed-writable-workers.md @@ -700,7 +700,7 @@ worker's command starts, the server opens that tmpfs from outside through the sandbox's `/proc//root`. It waits until the path is a tmpfs of exactly the allowance on another device than the server's own `/tmp`, and keeps the descriptor -(`function openPrivateTurnTemp`, `packages/ui-sdk/src/server/worker-launcher.ts:959-987`). +(`function openPrivateTurnTemp`, `packages/ui-sdk/src/server/worker-launcher.ts:985-1013`). Every later read goes through that descriptor, after the worker's namespace is gone as well, and never enters another mount, so runtime state below `/tmp` is never read as turn temp. The allowance (256 MiB by default, @@ -1315,7 +1315,7 @@ directories with empty read-only tmpfs mounts: After the masks, the launcher re-binds the runtime the worker needs, read-only and file by file, with the same resolution the restricted envelope uses -(`export function restrictedReadPaths`, `packages/ui-sdk/src/server/worker-launcher.ts:1565`): +(`export function restrictedReadPaths`, `packages/ui-sdk/src/server/worker-launcher.ts:1591`): - the Bun executable; - the backend entry and its `node_modules` chains; @@ -1346,8 +1346,12 @@ host-temp siblings or make runtime bytes writable. The concrete checks and mounts are in (`validateRuntimePlacement`, `packages/ui-sdk/src/server/worker-launcher.ts:650-656`), -(`temporaryRuntimeMounts`, `packages/ui-sdk/src/server/worker-launcher.ts:657-670`) -and (`runtimeMaskOverlays`, `packages/ui-sdk/src/server/worker-launcher.ts:671-689`). +(`temporaryRuntimeMounts`, `packages/ui-sdk/src/server/worker-launcher.ts:657-696`) +and (`runtimeMaskOverlays`, `packages/ui-sdk/src/server/worker-launcher.ts:697-715`). +Alias restoration recognizes names already supplied by a read-only bind or an +explicit system-view symlink. It does not mount a canonical executable over an +existing alias such as `/bin/sh`; missing owned temporary paths still receive +only their narrow read-only binds. Broad host-temp binding would reveal other workers and operator state; blanket relocation would reject the selected supported placement. Neither is the @@ -1596,11 +1600,11 @@ An ordinary group-shared directory was accessible, but the worker's private mount and namespace links through `/proc/` were denied. Changing the private directory to `0770` did not remove that process-access barrier. The current launcher opens mounts directly (`function openPrivateState`, -`packages/ui-sdk/src/server/worker-launcher.ts:926-954`; +`packages/ui-sdk/src/server/worker-launcher.ts:952-980`; `function openPrivateTurnTemp`, -`packages/ui-sdk/src/server/worker-launcher.ts:959-985`) before population and +`packages/ui-sdk/src/server/worker-launcher.ts:985-1011`) before population and start-gate release (`export function launchAgentWorker`, -`packages/ui-sdk/src/server/worker-launcher.ts:1118-1177`). Group modes and a uid +`packages/ui-sdk/src/server/worker-launcher.ts:1144-1203`). Group modes and a uid switch alone therefore cannot implement the selected cross-uid lifecycle. **Authority and launch.** One trusted external supervisor owns each worker diff --git a/docs/decisions/policy-write-boundary.md b/docs/decisions/policy-write-boundary.md index b1a9a96ea9..54715826fa 100644 --- a/docs/decisions/policy-write-boundary.md +++ b/docs/decisions/policy-write-boundary.md @@ -448,13 +448,13 @@ sessions. before the runtime starts and read it after the worker is gone, while the tmpfs is reachable only inside the worker. The launcher does both from the host (`export function launchAgentWorker`, -`packages/ui-sdk/src/server/worker-launcher.ts:1118-1240`): +`packages/ui-sdk/src/server/worker-launcher.ts:1144-1266`): 1. It finds the sandbox's first process from the kernel's record of bubblewrap's only child, then opens the tmpfs root through `/proc//root` and keeps that directory descriptor (`function openPrivateState`, - `packages/ui-sdk/src/server/worker-launcher.ts:926-950`). bubblewrap's + `packages/ui-sdk/src/server/worker-launcher.ts:952-976`). bubblewrap's process exists before its mounts do, so the open is accepted only once the path is its own mount (a device other than its parent's), a tmpfs of exactly the allowance, and empty; until then it would name the host's diff --git a/docs/decisions/worker-writable-areas.md b/docs/decisions/worker-writable-areas.md index adc9af09a9..228c5d0ce1 100644 --- a/docs/decisions/worker-writable-areas.md +++ b/docs/decisions/worker-writable-areas.md @@ -72,7 +72,7 @@ on the transcript persist path and is not part of this record. **One resolution, two consumers.** `prepareWorkerSandbox` resolves the areas and the home mode once per worker -(`export function prepareWorkerSandbox`, `packages/ui-sdk/src/server/worker-launcher.ts:755-795`). +(`export function prepareWorkerSandbox`, `packages/ui-sdk/src/server/worker-launcher.ts:781-821`). The backend passes the result both to the launcher, which turns it into bubblewrap arguments, and to `workerSandboxNote`, which renders the agent's note (`export function workerSandboxNote`, `packages/ui-sdk/src/server/worker-sandbox.ts:475-514`). @@ -80,7 +80,7 @@ The launcher re-checks the resolved layout against the actual launch: the same brain and envelope, each writable area still a real directory at its own path in the brain, and `$HOME` still the home the overlay was probed for. A mismatch refuses the launch instead of binding something the note does not -describe (`function sandboxMounts`, `packages/ui-sdk/src/server/worker-launcher.ts:704-734`). +describe (`function sandboxMounts`, `packages/ui-sdk/src/server/worker-launcher.ts:730-760`). A launch without a resolved layout keeps the 0.41 view: only the per-turn scratch is writable. @@ -142,7 +142,7 @@ its private, size-limited turn-temp tmpfs. Real HOME stays read-only: a tool that writes it directly receives `EROFS`, recorded through the roadblock path. `prepareWorkerSandbox` resolves this same XDG mode on every eligible host (`export function prepareWorkerSandbox`, -`packages/ui-sdk/src/server/worker-launcher.ts:755-795`). Host-backed overlay +`packages/ui-sdk/src/server/worker-launcher.ts:781-821`). Host-backed overlay uppers cannot live on the worker's private tmpfs; keeping them would leave an unbounded write area. Tools reading XDG config see the per-turn directory, while direct reads of real HOME retain the read-only host view. diff --git a/packages/ui-sdk/src/server/worker-launcher.ts b/packages/ui-sdk/src/server/worker-launcher.ts index 25f8a889a4..1c0e259ccc 100644 --- a/packages/ui-sdk/src/server/worker-launcher.ts +++ b/packages/ui-sdk/src/server/worker-launcher.ts @@ -656,13 +656,39 @@ function validateRuntimePlacement(requested: string, reserved: readonly string[] } function temporaryRuntimeMounts(paths: readonly string[], existing: readonly string[], explicitAliases: boolean): string[] { const bindings = new Map(); - const bound = new Set(); - for (let index = 0; index < existing.length - 2; index++) if (["--bind", "--ro-bind"].includes(existing[index]!)) { bound.add(existing[index + 2]!); index += 2; } + const restored: Array<{ source: string; target: string }> = []; + const aliases: Array<{ source: string; target: string }> = []; + for (let index = 0; index < existing.length - 2; index++) { + const option = existing[index]!; + if (!["--bind", "--ro-bind", "--symlink"].includes(option)) continue; + const source = existing[index + 1]!, target = existing[index + 2]!; + if (option === "--symlink") aliases.push({ source: isAbsolute(source) ? source : join(dirname(target), source), target }); + else restored.push({ source, target }); + index += 2; + } + const supplied = (requested: string, real: string): boolean => { + let path = requested; + // Follow only aliases installed in this view, not arbitrary host aliases. + for (let depth = 0; depth <= aliases.length; depth++) { + const alias = aliases.find(({ target }) => within(target, path)); + if (!alias) break; + path = alias.source + path.slice(alias.target.length); + if (depth === aliases.length) return false; + } + return restored.some(({ source, target }) => { + if (!within(target, path)) return false; + try { return realpathSync(source + path.slice(target.length)) === real; } + catch { return false; } + }); + }; for (const requested of paths) { let real: string; try { real = realpathSync(requested); } catch { continue; } for (const path of new Set([real, normalize(requested)])) { - if (!bound.has(path) && (within(TURN_TEMP_ROOT, path) || explicitAliases && path !== real)) bindings.set(path, real); + if ((within(TURN_TEMP_ROOT, path) || explicitAliases && path !== real) && !supplied(path, real)) { + bindings.set(path, real); + restored.push({ source: real, target: path }); + } } } return [...bindings].flatMap(([path, real]) => ["--ro-bind", real, path]); diff --git a/packages/ui-sdk/tests/worker-runtime-placement.test.ts b/packages/ui-sdk/tests/worker-runtime-placement.test.ts index 141ce24d19..de10920ae8 100644 --- a/packages/ui-sdk/tests/worker-runtime-placement.test.ts +++ b/packages/ui-sdk/tests/worker-runtime-placement.test.ts @@ -3,7 +3,7 @@ import { afterEach, expect, test } from "bun:test"; import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { workerCommand, type WorkerLaunch } from "../src/server/worker-launcher.js"; +import { restrictedReadPaths, workerCommand, type WorkerLaunch } from "../src/server/worker-launcher.js"; const ROOT = resolve(import.meta.dir, "../../.."); const ALLOWANCE = 8 * 1024 * 1024; @@ -96,3 +96,18 @@ test("owned runtime restoration cannot resurrect explicitly masked files or subd expect(result.stdout.toString(), "OWNED_RUNTIME_MASKS_SURVIVE_ALIASES").toBe("Odysseus public runtime"); } }); + + +test("restricted workers preserve executable aliases already supplied by their read-only view", () => { + const f = fixture(), runtime = join(f.root, "runtime"); mkdirSync(runtime); + writeFileSync(join(runtime, "package.json"), '{"name":"odysseus-runtime-alias"}\n'); + copyFileSync(realpathSync("/bin/sh"), join(runtime, "native-cli")); + const alias = join(runtime, "shell-alias"); symlinkSync("native-cli", alias); + for (const executable of ["/bin/sh", alias]) { + const launch = input(f, "restricted", runtime, [executable, "-c", "printf Odysseus"]); + launch.restricted!.readPaths = [...restrictedReadPaths([executable]), runtime]; + const result = Bun.spawnSync(workerCommand(launch), { stdin: Buffer.from("go\n"), stdout: "pipe", stderr: "pipe", timeout: 10_000 }); + expect(result.exitCode, `RESTORED_EXECUTABLE_ALIAS_LAUNCH: ${result.stderr}`).toBe(0); + expect(result.stdout.toString()).toBe("Odysseus"); + } +}); From 92fdc8bd32acd3bd3d17e7a62726ae5934d26601 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 21:04:19 +0200 Subject: [PATCH 30/44] test(ui): complete auth bootstrap and successful-send fixture readiness --- packages/ui-react/tests/auth-lock.test.ts | 31 +++++++++++++------ .../ui-react/tests/chat-focus-runtime.test.ts | 1 + .../tests/external-speech-runtime.test.ts | 1 + .../ui-react/tests/tracker-runtime.test.ts | 2 ++ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/ui-react/tests/auth-lock.test.ts b/packages/ui-react/tests/auth-lock.test.ts index 6e5fd8c36a..bd11655b10 100644 --- a/packages/ui-react/tests/auth-lock.test.ts +++ b/packages/ui-react/tests/auth-lock.test.ts @@ -2,10 +2,15 @@ import { expect, test } from "bun:test"; import { createBrainUiRoot, type BrainUiRoot } from "../src/root.js"; import type { LocalWork } from "../src/lib/local-work.js"; import { createBrainApi } from "../src/lib/api-client.js"; -import type { BrainApi } from "../src/lib/api-client.js"; import { updateHeld } from "../src/lib/update-holds.js"; import { TRACKER_STORAGE_KEY } from "../src/lib/trackers.js"; +// Renderer discovery is independent of the request whose auth lifetime is tested. +function withBootstrap(request: (url: string) => Promise) { + return (url: string) => new URL(url, "http://localhost").pathname === "/api/status" + ? Promise.resolve(Response.json({ contentRenderers: [] })) : request(url); +} + function deferred() { let resolve!: (value: T) => void; const promise = new Promise((yes) => { resolve = yes; }); @@ -46,7 +51,7 @@ test("overlapping same-account sign-ins restore once and install one manager", a }); test("a delayed 401 from an old authentication lifetime cannot expire the new one", async()=>{ - const pending=deferred(); const root=createBrainUiRoot({storage:null,request:()=>pending.promise}); + const pending=deferred(); const root=createBrainUiRoot({storage:null,request:withBootstrap(()=>pending.promise)}); try { work(root,async()=>true); const stale=root.request("/api/sessions").catch(error=>error); await lock(root); expect(await root.authLock.signedIn("odysseus-key")).toBe(true); @@ -58,8 +63,8 @@ test("a delayed 401 from an old authentication lifetime cannot expire the new on test("a body decoding and an injected API resolving after expiry cannot repopulate stores",async()=>{ const body=deferred(); const apiResult=deferred<{sessions:unknown[]}>(); - const api={sessions:()=>apiResult.promise} as unknown as BrainApi; - const root=createBrainUiRoot({storage:null,api,request:async()=>Object.assign(new Response(),{json:async()=>JSON.parse(await body.promise)})}); + const api=createBrainApi(()=>"/api",withBootstrap(async()=>Response.json(await apiResult.promise))); + const root=createBrainUiRoot({storage:null,api,request:withBootstrap(async()=>Object.assign(new Response(),{json:async()=>JSON.parse(await body.promise)}))}); try { work(root,async()=>true); const response=await root.request("/api/sessions"); const reading=response.json().catch(e=>e); @@ -74,9 +79,15 @@ test("lock clears activity read caches; restoration refetches finished history a let histories=0,payloads=0; const runId="odysseus-run",spanId="odysseus-tool"; const span={spanId,runId,origin:"session",sessionId:"odysseus-ithaca",operation:"agent.turn",startedAt:1,endedAt:2,outcome:"success",attrs:{}}; - const api={activityRuns:async()=>{histories++;return{history:[{runId,detailPruned:false}]};},activityRun:async(_id:string,options?:unknown)=>{ - if(options)payloads++; return{runId,spans:[span],events:[],highWaterSeq:1}; - }} as unknown as BrainApi; + const api=createBrainApi(()=>"/api",withBootstrap(async url=>{ + const parsed=new URL(url,"http://localhost"); + if(parsed.pathname==="/api/activity/runs"){histories++;return Response.json({live:[],history:[{runId,detailPruned:false}]});} + if(parsed.pathname===`/api/activity/runs/${runId}`){ + if(parsed.searchParams.get("include")==="payloads")payloads++; + return Response.json({runId,spans:[span],events:[],highWaterSeq:1}); + } + throw new Error(`Unexpected auth fixture request: ${parsed.pathname}`); + })); const root=createBrainUiRoot({storage:null,api}); try { work(root,async()=>true); await root.stores.activity.loadSessionActivityHistory("odysseus-ithaca"); await root.stores.activity.loadSpanPayloads(spanId); @@ -92,7 +103,7 @@ test("lock clears activity read caches; restoration refetches finished history a test("classified recovery 401 from the injected real client locks auth",async()=>{ - const api=createBrainApi(()=>"/api",async()=>new Response(null,{status:401})); + const api=createBrainApi(()=>"/api",withBootstrap(async()=>new Response(null,{status:401}))); const root=createBrainUiRoot({storage:null,api}); try { work(root,async()=>true); expect(await root.api.sessionRecovery("odysseus-ithaca")).toEqual({ok:false,reason:"unauthorized"}); @@ -102,7 +113,7 @@ test("classified recovery 401 from the injected real client locks auth",async()= test("only the exact account-confirmation route remains readable while locked",async()=>{ const requests:string[]=[]; - const root=createBrainUiRoot({storage:null,request:async(url)=>{requests.push(url);return Response.json({accountKey:"odysseus-key"});}}); + const root=createBrainUiRoot({storage:null,request:withBootstrap(async(url)=>{requests.push(url);return Response.json({accountKey:"odysseus-key"});})}); try { work(root,async()=>true); await lock(root); await expect(root.request("/api/files?path=/api/vpn-check")).rejects.toMatchObject({name:"AbortError"}); @@ -143,7 +154,7 @@ test("lock cannot reintroduce account metadata captured in a persisted boot stat for(const invalidate of ["lock","dispose"] as const) test(`classified recovery remains a result after ${invalidate}`,async()=>{ const pending=deferred(); let calls=0; - const api=createBrainApi(()=>"/api",async()=>{calls++;return pending.promise;}); + const api=createBrainApi(()=>"/api",withBootstrap(async()=>{calls++;return pending.promise;})); const root=createBrainUiRoot({storage:null,api}); try { work(root,async()=>true); diff --git a/packages/ui-react/tests/chat-focus-runtime.test.ts b/packages/ui-react/tests/chat-focus-runtime.test.ts index 4bfccc3ea0..57aa733d87 100644 --- a/packages/ui-react/tests/chat-focus-runtime.test.ts +++ b/packages/ui-react/tests/chat-focus-runtime.test.ts @@ -101,6 +101,7 @@ async function open(prompt: string): Promise { const composer = page.locator("textarea.bk-composer"); expect(await composer.count()).toBe(1); await composer.fill(prompt); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); return page; } diff --git a/packages/ui-react/tests/external-speech-runtime.test.ts b/packages/ui-react/tests/external-speech-runtime.test.ts index 5b92f9fa05..c32f574749 100644 --- a/packages/ui-react/tests/external-speech-runtime.test.ts +++ b/packages/ui-react/tests/external-speech-runtime.test.ts @@ -92,6 +92,7 @@ async function open(proveBackend = false): Promise { const before = turns; const composer = page.locator("textarea.bk-composer"); await composer.fill("Typed fixture control"); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); await page.waitForFunction("window.__speechFixture.turnIdle()"); expect(turns).toBe(before + 1); diff --git a/packages/ui-react/tests/tracker-runtime.test.ts b/packages/ui-react/tests/tracker-runtime.test.ts index 7ecc5ee9ac..a69ff7aecf 100644 --- a/packages/ui-react/tests/tracker-runtime.test.ts +++ b/packages/ui-react/tests/tracker-runtime.test.ts @@ -147,6 +147,7 @@ describe.skipIf(!executablePath)("mounted trackers", () => { await until(page, "f?.connected()"); const composer = page.locator("textarea.bk-composer"); await composer.fill(prompt); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); await until(page, "f.activeSessionId() !== null && f.lastTurnId() !== null"); const sessionId = (await fixture(page, (f) => f.activeSessionId()))!; @@ -241,6 +242,7 @@ describe.skipIf(!executablePath)("mounted trackers", () => { await until(page, "f?.connected()"); const composer = page.locator("textarea.bk-composer"); await composer.fill(prompt); + await composer.locator("..").locator('[aria-label="Send"]').waitFor(); await composer.press("Enter"); await until(page, "f.activeSessionId() !== null && f.lastTurnId() !== null"); const sessionId = (await fixture(page, (f) => f.activeSessionId()))!; From dc0dbf1852ecd46ea000cfee216a7d2a49b1f0d3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 21:07:41 +0200 Subject: [PATCH 31/44] test(ui): await admitted composer state in parallel session sends --- packages/ui-react/tests/parallel-sessions-runtime.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/ui-react/tests/parallel-sessions-runtime.test.ts b/packages/ui-react/tests/parallel-sessions-runtime.test.ts index 1adff325d9..3c18bf0375 100644 --- a/packages/ui-react/tests/parallel-sessions-runtime.test.ts +++ b/packages/ui-react/tests/parallel-sessions-runtime.test.ts @@ -469,6 +469,12 @@ async function newChat(run: Run, page: Page) { /** Sends what the composer holds, with the keyboard's Enter (or the send button on a phone). */ async function send(run: Run, page: Page, text: string) { await composer(page).fill(text); + await page.waitForFunction(() => { + const send = document.querySelector('[role="button"][aria-label="Send"]'); + // An admitted streaming session still accepts Enter while showing Stop. + return (send !== null && send.getAttribute("aria-disabled") !== "true") + || document.querySelector('[role="button"][aria-label="Stop generating"]') !== null; + }); if (phone(run)) await press(run, button(page, "Send")); else await composer(page).press("Enter"); } From e6d5fef84618ab04ac6b88e73968def9a8c52baa Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 23:30:23 +0200 Subject: [PATCH 32/44] test(ui-server): distinguish shutdown refusal from unhealthy database --- packages/ui-server/tests/app-wiring.test.ts | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/packages/ui-server/tests/app-wiring.test.ts b/packages/ui-server/tests/app-wiring.test.ts index a3bd3c011f..5aa0903082 100644 --- a/packages/ui-server/tests/app-wiring.test.ts +++ b/packages/ui-server/tests/app-wiring.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { describe, test, expect, beforeAll, afterAll, spyOn } from "bun:test"; import { mkdirSync } from "node:fs"; import { join } from "path"; import { tmpdir } from "os"; @@ -14,6 +14,7 @@ import { bundledClaudeBinary } from "./helpers/claude-binary"; import { createStaticBackendRegistry } from "../src/agent/backend"; import { makeFakeBackend } from "./helpers/fake-backend"; import { createUiDb } from "../src/db/client"; +import { createHealthRoutes } from "../src/routes/health"; import { resolveAmbientPrincipal, resolvePrincipal, @@ -683,21 +684,33 @@ describe("app wiring — request logging", () => { }); }); -describe("app wiring — health probes the database", () => { - test("a dead SQLite handle turns /api/health into 503 unhealthy", async () => { +describe("app wiring — health and shutdown", () => { + test("shutdown refuses health before touching the closed database", async () => { const wired = await createApp(); const send = () => wired.fetch(new Request("http://localhost/api/health")); + try { + expect((await send()).status).toBe(200); + await wired.close(); + const query = spyOn(wired.db, "query"); + try { + const res = await send(); + expect(res.status).toBe(503); + expect(await res.json(), "SHUTDOWN_REFUSES_BEFORE_HEALTH_DISPATCH").toEqual({ error: "Server shutting down" }); + expect(query, "SHUTDOWN_NEVER_QUERIES_CLOSED_STATE").not.toHaveBeenCalled(); + } finally { query.mockRestore(); } + } finally { await wired.close(); } + }); - expect((await send()).status).toBe(200); - - // close() releases the handle — the same state a wedged database presents. - // Awaited: the handle goes only once the scratch prune pass has ended. - await wired.close(); - const res = await send(); - expect(res.status).toBe(503); - const body = await res.json(); - // Fails minimal: the route is public, so the body says unhealthy and - // nothing else. - expect(body).toEqual({ status: "unhealthy" }); + test("the public health route reports only unhealthy for a dead SQLite handle", async () => { + const db = createUiDb(":memory:"); + const route = createHealthRoutes({ db }); + let closed = false; + try { + expect((await route.request("/health")).status).toBe(200); + db.close(); closed = true; + const res = await route.request("/health"); + expect(res.status, "HEALTH_PROBES_REAL_DATABASE").toBe(503); + expect(await res.json(), "PUBLIC_UNHEALTHY_RESPONSE_IS_MINIMAL").toEqual({ status: "unhealthy" }); + } finally { if (!closed) db.close(); } }); }); From 26665d03a0eac2886fc9ddb02da2033d1ab52848 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 16 Oct 2026 23:44:38 +0200 Subject: [PATCH 33/44] test(ui-react): retain bounded composer admission failure evidence --- .../tests/parallel-sessions-runtime.test.ts | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/ui-react/tests/parallel-sessions-runtime.test.ts b/packages/ui-react/tests/parallel-sessions-runtime.test.ts index 3c18bf0375..5d4a8d6d47 100644 --- a/packages/ui-react/tests/parallel-sessions-runtime.test.ts +++ b/packages/ui-react/tests/parallel-sessions-runtime.test.ts @@ -351,7 +351,8 @@ function waitingWorker() { type ApprovalInput = { type: string; target: string | null; at: number; node: number | null; connected: boolean; y: number | null }; type ApprovalFrame = { event: "framesent" | "framereceived"; socket: number; at: number; frame: ClientMessage | ServerMessage }; -type Device = { context: BrowserContext; page: Page; dialogs: string[]; sockets: string[]; approvals: ApprovalFrame[] }; +type ProfileRead = { path: string; status?: number; body?: string; failure?: string }; +type Device = { profileReads: ProfileRead[]; context: BrowserContext; page: Page; dialogs: string[]; sockets: string[]; approvals: ApprovalFrame[] }; async function device(run: Run, opts: { clock?: boolean; update?: boolean } = {}): Promise { const context = await browser!.newContext(run.options); await context.route("**/*", (route) => new URL(route.request().url()).origin === origin ? route.continue() : route.abort()); @@ -368,6 +369,18 @@ async function device(run: Run, opts: { clock?: boolean; update?: boolean } = {} // Every socket the page opens, so a reconnect is observed rather than inferred. const sockets: string[] = []; const approvals: ApprovalFrame[] = []; + // Keep bounded evidence for a composer that never reaches admitted Send. + // Successful later runs cannot explain a missing response in an earlier run. + const profileReads: ProfileRead[] = []; + const rememberProfile = (read: ProfileRead) => { profileReads.push(read); if (profileReads.length > 8) profileReads.shift(); }; + page.on("response", async response => { + const path = new URL(response.url()).pathname; + if (path.endsWith("/profile")) rememberProfile({ path, status: response.status(), body: (await response.text().catch(error => String(error))).slice(0, 4096) }); + }); + page.on("requestfailed", request => { + const path = new URL(request.url()).pathname; + if (path.endsWith("/profile")) rememberProfile({ path, failure: request.failure()?.errorText }); + }); page.on("websocket", (ws) => { const socket = sockets.push(ws.url()) - 1; const record = (event: ApprovalFrame["event"], payload: string | Buffer) => { @@ -401,7 +414,7 @@ async function device(run: Run, opts: { clock?: boolean; update?: boolean } = {} await page.goto(opts.update ? `${origin}/?update` : origin); await until(page, "p?.connected() && p.draftsSupported() === true"); expect(await page.evaluate(() => document.documentElement.dataset.theme), "the run's theme is the one drawn").toBe(run.theme); - return { context, page, dialogs, sockets, approvals }; + return { context, page, dialogs, sockets, approvals, profileReads }; } /** A real tap under a coarse pointer, a real click otherwise; Playwright waits until the control is at rest (#992). */ @@ -1018,7 +1031,12 @@ describe.skipIf(!executablePath)("mounted parallel sessions", () => { expect(replies(), "completion sends no duplicate decision").toHaveLength(1); if (decision === "Always allow") expect(await readPolicy(), "only the explicit online decision wrote policy").toMatchObject({ tools: expect.arrayContaining(["Read"]) }); } catch (error) { - console.error("Refused approval diagnostic", JSON.stringify({ run: run.name, surface, decision, runtime: Bun.version, browser: browser!.version(), frames: a.approvals, tools: await a.page.evaluate((id) => (window as unknown as { __parallel: { approvalTools: (id: string) => unknown } }).__parallel.approvalTools(id), sessionId) })); + console.error("Refused approval diagnostic", JSON.stringify({ run: run.name, surface, decision, runtime: Bun.version, browser: browser!.version(), frames: a.approvals, profileReads: a.profileReads, + composer: await a.page.evaluate(() => ({ value: (document.querySelector("textarea[data-composer]") as HTMLTextAreaElement | null)?.value, + placeholder: document.querySelector("textarea[data-composer]")?.getAttribute("placeholder"), + visibleText: document.body.innerText.slice(-4000), + send: [...document.querySelectorAll('[role="button"][aria-label="Send"]')].map(node => ({ disabled: node.getAttribute("aria-disabled") })) })).catch(() => null), + tools: await a.page.evaluate((id) => (window as unknown as { __parallel: { approvalTools: (id: string) => unknown } }).__parallel.approvalTools(id), sessionId) })); throw error; } finally { await a.context.close(); From e47372f53dda7b576bac7d8396bdffe6bdce4fed Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 12:43:08 +0200 Subject: [PATCH 34/44] test(ui-sdk): isolate ordinary native identity fixture storage Refs #2313. Preserve the preceding manager fixture and normal Git ownership checks. --- packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts | 9 +++++++-- packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts b/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts index 4484d85087..d6a35ebb2f 100644 --- a/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts +++ b/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts @@ -3,7 +3,7 @@ import { chmodSync, chownSync, mkdirSync, writeFileSync, existsSync, readdirSync import { join } from "node:path"; import { serveNativeBrainManager } from "../../../ui-sdk/src/server/native-brain-manager.js"; import { hostOutputStagingLimits } from "../../../ui-sdk/src/server/worker-state-limits.js"; -const brain = "/brain-fixture/brain"; +const store = "/brain-fixture/ordinary-identity", brain = join(store, "brain"); function command(argv: string[]) { const child = Bun.spawnSync(argv, { stdout: "pipe", stderr: "pipe" }); expect(child.exitCode, child.stderr.toString()).toBe(0); return child.stdout.toString(); @@ -57,6 +57,11 @@ if (process.argv[2] === "--child") { await import("./ordinary-identity-ws.js"); } else { expect(process.getuid!()).toBe(0); + // The preceding manager proof owns a different brain and leaves its private + // worker files intact. Bootstrap this case in a fresh store with normal Git + // ownership checks, then transfer only this case's root to the app identity. + mkdirSync(store, { mode: 0o755 }); + command(["btrfs", "subvolume", "create", brain]); mkdirSync("/identity-fixture", { recursive: true }); chownSync("/identity-fixture", 21001, 21001); chmodSync("/identity-fixture", 0o2775); for (const path of ["notes", ".brain/scratch", "node_modules/.bin"]) mkdirSync(join(brain, path), { recursive: true }); writeFileSync(join(brain, "brain.config.ts"), 'export default {profile:{name:"Odysseus",cliTitle:"Odysseus notebook"}};\n'); @@ -74,7 +79,7 @@ if (process.argv[2] === "--child") { const fsid = /^fsid\s+([0-9a-f-]+)/m.exec(command(["btrfs", "inspect-internal", "dump-super", "/dev/brain-native-proof-loop"]))![1]!; mkdirSync("/native-runtime", { mode: 0o755, recursive: true }); const staging = hostOutputStagingLimits({}); - const service = await serveNativeBrainManager({ storePath: "/brain-fixture", devicePath: "/dev/brain-native-proof-loop", fsid, brainUuid, socketPath: "/native-runtime/server-ws-manager.sock", + const service = await serveNativeBrainManager({ storePath: store, devicePath: "/dev/brain-native-proof-loop", fsid, brainUuid, socketPath: "/native-runtime/server-ws-manager.sock", appUid: 21001, appGid: 21001, workerUid: 21002, stageAllowanceBytes: staging.allowanceBytes, stageBudgetBytes: staging.budgetBytes, capacityBytes: 4 * 1024 ** 3, recoveryReserveBytes: 16 * 1024 ** 2 }); let foreignRequests = 0; diff --git a/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts b/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts index 574f78e78c..bc9dbe8e8e 100644 --- a/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts +++ b/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts @@ -11,7 +11,7 @@ import { createClaudeBackend } from "../../../ui-backend-claude/src/backend.js"; import { nativeModelReply } from "../../../ui-backend-claude/tests/helpers/native-model-reply.js"; import { serializeSigned } from "hono/utils/cookie"; -const brain = "/brain-fixture/brain", root = "/identity-server-private/server-ws"; +const brain = "/brain-fixture/ordinary-identity/brain", root = "/identity-server-private/server-ws"; const home = join(root, "home"), config = join(home, ".claude"), data = "/identity-fixture/server-app"; mkdirSync(data, { mode: 0o700 }); mkdirSync(config, { recursive: true, mode: 0o700 }); const requests: Array<{ model: string; history: number; apiCredential: boolean }> = []; From 904e2a0128884079791629c3c0b18b013c7764bc Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 12:59:52 +0200 Subject: [PATCH 35/44] test(claude): compose deadlines for four-turn native relay profiles Refs #2315. Preserve per-turn and cancellation limits and retain phase observations. Defer the composed Git fixture repair to independently delivered #2317. --- docs/decisions/claude-code-runtime.md | 4 ++-- .../tests/fixtures/sdk-host-relay-limits.ts | 6 ++++++ .../tests/fixtures/sdk-host-relay-probe.ts | 21 +++++++++++++++---- .../tests/sdk-host-relay-identity.native.ts | 8 ++++--- .../tests/sdk-host-relay.test.ts | 11 +++++----- .../tests/fixtures/ordinary-identity-root.ts | 9 ++------ .../tests/fixtures/ordinary-identity-ws.ts | 2 +- 7 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 packages/ui-backend-claude/tests/fixtures/sdk-host-relay-limits.ts diff --git a/docs/decisions/claude-code-runtime.md b/docs/decisions/claude-code-runtime.md index f36391c680..118801aa31 100644 --- a/docs/decisions/claude-code-runtime.md +++ b/docs/decisions/claude-code-runtime.md @@ -1371,7 +1371,7 @@ outward approvals. The native qualification now constructs the full SDK turn, including its MCP server, system prompt, per-turn context and permission hooks, rather than constructing query options independently (`const sdkTurn = createClaudeSdkTurn`, -`packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts:205-226`). +`packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts:209-231`). Strict and governed controls retain history, model changes, host login rotation/refresh and refresh cancellation. An actual auto-allowed native Bash call still reaches the existing per-use refusal: removing the assembled @@ -1379,7 +1379,7 @@ PreToolUse hook creates the refused file. Native managed Unix-socket overrides and project API credentials withhold the prompt through the existing subscription settings check, before provider contact or host credential resolution (`settingsRefusal(settings`, -`packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts:238-243`). +`packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts:246-251`). The actual process state ledger is observed admitted while the worker is live and empty after teardown. diff --git a/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-limits.ts b/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-limits.ts new file mode 100644 index 0000000000..b3765cdb8f --- /dev/null +++ b/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-limits.ts @@ -0,0 +1,6 @@ +/** Whole-profile allowance composes the existing per-turn fixture limits. */ +export const SDK_RELAY_TURN_COUNT = 4; +export const SDK_RELAY_TURN_TIMEOUT_MS = 20_000; +export const SDK_RELAY_CANCELLATION_TIMEOUT_MS = 5_000; +export const SDK_RELAY_SINGLE_TURN_TIMEOUT_MS = SDK_RELAY_TURN_TIMEOUT_MS + SDK_RELAY_CANCELLATION_TIMEOUT_MS; +export const SDK_RELAY_PROFILE_TIMEOUT_MS = SDK_RELAY_TURN_COUNT * SDK_RELAY_SINGLE_TURN_TIMEOUT_MS; diff --git a/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts b/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts index 12e5001c9f..0447f550c3 100644 --- a/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts +++ b/packages/ui-backend-claude/tests/fixtures/sdk-host-relay-probe.ts @@ -1,3 +1,4 @@ +import { SDK_RELAY_TURN_COUNT, SDK_RELAY_TURN_TIMEOUT_MS, SDK_RELAY_CANCELLATION_TIMEOUT_MS } from "./sdk-host-relay-limits.js"; /** Actual ordinary Claude spawn/state path; copied storage fixture, not Btrfs qualification. */ import { query, type AccountInfo } from "@anthropic-ai/claude-agent-sdk"; import { startInferenceRelay, ANTHROPIC_INFERENCE_ROUTES, INFERENCE_PLACEHOLDER, acquireHostCredentialExclusion, startEgressProxy, type EgressRecord } from "@schlessera/brain-ui-sdk/internal"; @@ -159,13 +160,15 @@ function files(path: string): string { async function bounded(promise: Promise): Promise { let timer: ReturnType | undefined; try { return await Promise.race([promise, new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("Native cancellation did not reach its owned boundary.")), 5000); + timer = setTimeout(() => reject(new Error("Native cancellation did not reach its owned boundary.")), SDK_RELAY_CANCELLATION_TIMEOUT_MS); })]); } finally { clearTimeout(timer); } } let session: string | undefined, hostAfterRefresh = false; let cancellationRetainedHostBytes = false, cancellationReleasedLock = false; const copyOutRefusals: Array<{ name: string; signalAborted: boolean }> = []; const originalConsoleError = console.error; +const observedStart = performance.now(); +const observePhase = (phase: string, turn: number) => { if (process.argv.includes("--observe-phases")) originalConsoleError("SDK_RELAY_PHASE " + JSON.stringify({ phase, turn, elapsedMs: Math.round(performance.now() - observedStart) })); }; let currentAbort: AbortController | undefined; if (separateIdentity) console.error = (...args: unknown[]) => { if (args[0] === "Claude worker runtime-state copy-out refused:") { @@ -173,14 +176,15 @@ if (separateIdentity) console.error = (...args: unknown[]) => { } else originalConsoleError(...args); }; try { - for (let turn = 0; turn < (inspectProof || permissionProof || settingsProof ? 1 : 4); turn++) { + for (let turn = 0; turn < (inspectProof || permissionProof || settingsProof ? 1 : SDK_RELAY_TURN_COUNT); turn++) { + observePhase("turn-start", turn); if (turn === 1) save(access[1]!); if (turn === 2) save(access[1]!, 1); if (turn === 3) { save(access[1]!, 1); holdRefresh = true; } const before = readFileSync(join(host, ".credentials.json")); const abort = new AbortController(); currentAbort = abort; - const timer = setTimeout(() => abort.abort(), 20_000); + const timer = setTimeout(() => abort.abort(), SDK_RELAY_TURN_TIMEOUT_MS); let release!: () => void; const held = new Promise(resolve => { release = resolve; }); let live: LaunchedWorker | undefined; @@ -212,6 +216,7 @@ try { return { behavior: "deny" as const, message: "The fictional user refused this command." }; } }, workerStarted: started => { + observePhase("worker-started", turn); live = started as LaunchedWorker; workers.push(live); const owned = live; void owned.exited.then(() => exitedWorkers.add(owned), () => exitedWorkers.add(owned)); observedLedgerHeld = Math.max(observedLedgerHeld, ledger.held); @@ -232,9 +237,12 @@ try { else yield* sdkTurn.prompt; })(), options: { ...sdkTurn.options, maxTurns: permissionProof ? 2 : 1, stderr: text => { stderr.push(text); } } }); try { + observePhase("initialization-wait", turn); const account = (await runtime.initializationResult()).account; + observePhase("initialized", turn); accounts.push(account); const settings = await (runtime as unknown as { getSettings(): Promise }).getSettings(); + observePhase("settings-read", turn); if (!subscriptionVerdict(account).ok || settingsRefusal(settings, sdkTurn.pinnedSettingsEnv)) { if (!settingsProof) throw new Error("Prepared SDK subscription handshake refused."); withheldForSettings = true; @@ -253,9 +261,11 @@ try { nativeEnvironments.push(readFileSync(`/proc/${pid}/environ`, "utf8")); } if (inspectProof) continue; - release(); + release(); observePhase("prompt-released", turn); if (turn === 3) { + observePhase("refresh-wait", turn); await bounded(refreshEntered); + observePhase("refresh-entered", turn); abort.abort(); runtime.close(); } for await (const message of runtime) { @@ -265,9 +275,12 @@ try { } catch (error) { if (turn !== 3 || !abort.signal.aborted) throw error; } finally { + observePhase("finish-start", turn); abort.abort(); release(); runtime.close(); clearTimeout(timer); try { await spawner.finish(); } finally { turnLock.close(); await startupView?.cleanup(); } + observePhase("finish-complete", turn); } + observePhase("turn-complete", turn); if (turn === 2) hostAfterRefresh = JSON.parse(readFileSync(join(host, ".credentials.json"), "utf8")).claudeAiOauth.accessToken === access[2]; if (turn === 3) { try { await bounded(refreshEnded); } diff --git a/packages/ui-backend-claude/tests/sdk-host-relay-identity.native.ts b/packages/ui-backend-claude/tests/sdk-host-relay-identity.native.ts index 470b0041e2..d89d4926e1 100644 --- a/packages/ui-backend-claude/tests/sdk-host-relay-identity.native.ts +++ b/packages/ui-backend-claude/tests/sdk-host-relay-identity.native.ts @@ -1,3 +1,4 @@ +import { SDK_RELAY_PROFILE_TIMEOUT_MS } from "./fixtures/sdk-host-relay-limits.js"; /** Full SDK with the real identity helper; copied startup fixture, not Btrfs admission. */ import { expect } from "bun:test"; import { existsSync } from "node:fs"; @@ -7,14 +8,15 @@ if (process.getuid?.() !== 21001 || !existsSync("/usr/local/bin/identity-helper" } for (const profile of ["governed", "strict"]) { const fixture = new URL("./fixtures/sdk-host-relay-probe.ts", import.meta.url).pathname; - const child = Bun.spawn([process.execPath, "--preload", "/repo/scripts/test-network-child-preload.ts", fixture, profile, "ordinary", "separate-uid"], { + const child = Bun.spawn([process.execPath, "--preload", "/repo/scripts/test-network-child-preload.ts", fixture, profile, "ordinary", "separate-uid", "--observe-phases"], { env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: "/identity-server-private", CLAUDE_CONFIG_DIR: "/identity-server-private/claude", - BRAIN_UI_WORKER_IDENTITY_HELPER: "/usr/local/bin/identity-helper" }, stdout: "pipe", stderr: "pipe", timeout: 25_000, + BRAIN_UI_WORKER_IDENTITY_HELPER: "/usr/local/bin/identity-helper" }, stdout: "pipe", stderr: "pipe", timeout: SDK_RELAY_PROFILE_TIMEOUT_MS, }); const [stdout, stderr, exit] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + console.log(JSON.stringify({ diagnosticProfile: profile, exit, signal: child.signalCode, phases: stderr.split("\n").filter(line => line.startsWith("SDK_RELAY_PHASE ")) })); expect(exit, stderr).toBe(0); expect(stderr.split("\n").filter(Boolean).every(line => line.includes("[CLAUDE_SDK_CAN_USE_TOOL_SHADOWED]") - || line.startsWith("(Use `bun --trace-warnings")), stderr).toBe(true); + || line.startsWith("SDK_RELAY_PHASE ") || line.startsWith("(Use `bun --trace-warnings")), stderr).toBe(true); const value = JSON.parse(stdout); expect(value.nativeIdentities, "FULL_SDK_ACTUAL_SEPARATE_IDENTITY").toHaveLength(4); for (const identity of value.nativeIdentities) { diff --git a/packages/ui-backend-claude/tests/sdk-host-relay.test.ts b/packages/ui-backend-claude/tests/sdk-host-relay.test.ts index 994f388f5a..08a6293a6a 100644 --- a/packages/ui-backend-claude/tests/sdk-host-relay.test.ts +++ b/packages/ui-backend-claude/tests/sdk-host-relay.test.ts @@ -1,3 +1,4 @@ +import { SDK_RELAY_PROFILE_TIMEOUT_MS, SDK_RELAY_SINGLE_TURN_TIMEOUT_MS, SDK_RELAY_CANCELLATION_TIMEOUT_MS } from "./fixtures/sdk-host-relay-limits.js"; /** Full SDK construction, native startup and prepared ordinary transport; fixture storage is not Btrfs qualification. */ import { expect, test } from "bun:test"; import { createHash } from "node:crypto"; @@ -17,7 +18,7 @@ test.each(["strict", "governed"] as const)("prepared %s SDK turn preserves nativ const fixture = new URL("./fixtures/sdk-host-relay-probe.ts", import.meta.url).pathname; const child = Bun.spawn(["bwrap", "--clearenv", "--setenv", "PATH", "/usr/bin:/bin", "--setenv", "HOME", "/tmp", "--unshare-net", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", "--tmpfs", "/tmp", "--tmpfs", "/etc", "--proc", "/proc", "--dev", "/dev", process.execPath, fixture, profile], { - env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: 25_000, + env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: SDK_RELAY_PROFILE_TIMEOUT_MS, }); const [stdout, stderr, exit] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); expect(exit, stderr).toBe(0); @@ -69,14 +70,14 @@ test.each(["strict", "governed"] as const)("prepared %s SDK turn preserves nativ expect(result.refreshCancelled, "NATIVE_ABORT_CLOSES_ACTUAL_HOST_REFRESH").toBe(true); expect(result.cancellationRetainedHostBytes).toBe(true); expect(result.cancellationReleasedLock).toBe(true); -}); +}, SDK_RELAY_PROFILE_TIMEOUT_MS + SDK_RELAY_CANCELLATION_TIMEOUT_MS); test("prepared SDK native auto-allowed Bash still honors the existing per-use refusal", async () => { const fixture = new URL("./fixtures/sdk-host-relay-probe.ts", import.meta.url).pathname; const child = Bun.spawn(["bwrap", "--clearenv", "--setenv", "PATH", "/usr/bin:/bin", "--setenv", "HOME", "/tmp", "--unshare-net", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", "--tmpfs", "/tmp", "--tmpfs", "/etc", "--proc", "/proc", "--dev", "/dev", process.execPath, fixture, "governed", "permission"], { - env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: 25_000, + env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: SDK_RELAY_SINGLE_TURN_TIMEOUT_MS, }); const [stdout, stderr, exit] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); expect(exit, stderr).toBe(0); @@ -95,7 +96,7 @@ test.each(["managed", "project"] as const)("prepared SDK %s credential or transp const fixture = new URL("./fixtures/sdk-host-relay-probe.ts", import.meta.url).pathname; const child = Bun.spawn(["bwrap", "--clearenv", "--setenv", "PATH", "/usr/bin:/bin", "--setenv", "HOME", "/tmp", "--unshare-net", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", "--tmpfs", "/tmp", "--tmpfs", "/etc", "--proc", "/proc", "--dev", "/dev", process.execPath, fixture, "governed", source], { - env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: 25_000, + env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: SDK_RELAY_SINGLE_TURN_TIMEOUT_MS, }); const [stdout, stderr, exit] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); expect(exit, stderr).toBe(0); @@ -112,7 +113,7 @@ test("prepared SDK initializes with no projected host credential store before an const fixture = new URL("./fixtures/sdk-host-relay-probe.ts", import.meta.url).pathname; const child = Bun.spawn(["bwrap", "--clearenv", "--setenv", "PATH", "/usr/bin:/bin", "--setenv", "HOME", "/tmp", "--unshare-net", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", "--tmpfs", "/tmp", "--tmpfs", "/etc", "--proc", "/proc", "--dev", "/dev", process.execPath, fixture, "governed", "inspect"], { - env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: 25_000, + env: { PATH: "/usr/bin:/bin" }, stdout: "pipe", stderr: "pipe", timeout: SDK_RELAY_SINGLE_TURN_TIMEOUT_MS, }); const [stdout, stderr, exit] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); expect(exit, stderr).toBe(0); diff --git a/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts b/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts index d6a35ebb2f..4484d85087 100644 --- a/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts +++ b/packages/ui-sdk/tests/fixtures/ordinary-identity-root.ts @@ -3,7 +3,7 @@ import { chmodSync, chownSync, mkdirSync, writeFileSync, existsSync, readdirSync import { join } from "node:path"; import { serveNativeBrainManager } from "../../../ui-sdk/src/server/native-brain-manager.js"; import { hostOutputStagingLimits } from "../../../ui-sdk/src/server/worker-state-limits.js"; -const store = "/brain-fixture/ordinary-identity", brain = join(store, "brain"); +const brain = "/brain-fixture/brain"; function command(argv: string[]) { const child = Bun.spawnSync(argv, { stdout: "pipe", stderr: "pipe" }); expect(child.exitCode, child.stderr.toString()).toBe(0); return child.stdout.toString(); @@ -57,11 +57,6 @@ if (process.argv[2] === "--child") { await import("./ordinary-identity-ws.js"); } else { expect(process.getuid!()).toBe(0); - // The preceding manager proof owns a different brain and leaves its private - // worker files intact. Bootstrap this case in a fresh store with normal Git - // ownership checks, then transfer only this case's root to the app identity. - mkdirSync(store, { mode: 0o755 }); - command(["btrfs", "subvolume", "create", brain]); mkdirSync("/identity-fixture", { recursive: true }); chownSync("/identity-fixture", 21001, 21001); chmodSync("/identity-fixture", 0o2775); for (const path of ["notes", ".brain/scratch", "node_modules/.bin"]) mkdirSync(join(brain, path), { recursive: true }); writeFileSync(join(brain, "brain.config.ts"), 'export default {profile:{name:"Odysseus",cliTitle:"Odysseus notebook"}};\n'); @@ -79,7 +74,7 @@ if (process.argv[2] === "--child") { const fsid = /^fsid\s+([0-9a-f-]+)/m.exec(command(["btrfs", "inspect-internal", "dump-super", "/dev/brain-native-proof-loop"]))![1]!; mkdirSync("/native-runtime", { mode: 0o755, recursive: true }); const staging = hostOutputStagingLimits({}); - const service = await serveNativeBrainManager({ storePath: store, devicePath: "/dev/brain-native-proof-loop", fsid, brainUuid, socketPath: "/native-runtime/server-ws-manager.sock", + const service = await serveNativeBrainManager({ storePath: "/brain-fixture", devicePath: "/dev/brain-native-proof-loop", fsid, brainUuid, socketPath: "/native-runtime/server-ws-manager.sock", appUid: 21001, appGid: 21001, workerUid: 21002, stageAllowanceBytes: staging.allowanceBytes, stageBudgetBytes: staging.budgetBytes, capacityBytes: 4 * 1024 ** 3, recoveryReserveBytes: 16 * 1024 ** 2 }); let foreignRequests = 0; diff --git a/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts b/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts index bc9dbe8e8e..574f78e78c 100644 --- a/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts +++ b/packages/ui-sdk/tests/fixtures/ordinary-identity-ws.ts @@ -11,7 +11,7 @@ import { createClaudeBackend } from "../../../ui-backend-claude/src/backend.js"; import { nativeModelReply } from "../../../ui-backend-claude/tests/helpers/native-model-reply.js"; import { serializeSigned } from "hono/utils/cookie"; -const brain = "/brain-fixture/ordinary-identity/brain", root = "/identity-server-private/server-ws"; +const brain = "/brain-fixture/brain", root = "/identity-server-private/server-ws"; const home = join(root, "home"), config = join(home, ".claude"), data = "/identity-fixture/server-app"; mkdirSync(data, { mode: 0o700 }); mkdirSync(config, { recursive: true, mode: 0o700 }); const requests: Array<{ model: string; history: number; apiCredential: boolean }> = []; From 4376cf9cd2f17e841115551090152bd4842780ca Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 15:54:27 +0200 Subject: [PATCH 36/44] test(ui-react): attribute silent speech bundle termination --- .../tests/external-speech-runtime.test.ts | 4 ++-- .../tests/fixtures/runtime-hook-phases.ts | 3 ++- .../ui-react/tests/runtime-hook-phases.test.ts | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/ui-react/tests/external-speech-runtime.test.ts b/packages/ui-react/tests/external-speech-runtime.test.ts index 234e503534..a7a8a830b5 100644 --- a/packages/ui-react/tests/external-speech-runtime.test.ts +++ b/packages/ui-react/tests/external-speech-runtime.test.ts @@ -68,8 +68,8 @@ if (worker) { app = await httpContractApp({ staticRoot: true, env: { VOICE_PROVIDER: provider.id }, speechProvider: provider, registry: createStaticBackendRegistry([backend]) }); process.stderr.write("[external-speech] bundling public browser client\n"); const bundle = Bun.spawn([process.execPath, "-e", 'const result = await Bun.build({entrypoints:[process.argv[1]],target:"browser",outdir:process.argv[2],naming:"client.js"}); if(!result.success){console.error(result.logs);process.exit(1);}', resolve(import.meta.dir, "fixtures/external-speech-browser.ts"), app.staticRoot], { cwd: repo, stdout: "pipe", stderr: "pipe" }); - const [bundleOut, bundleErr, bundleCode] = await Promise.all([relay(bundle.stdout), relay(bundle.stderr), bundle.exited]); - if (bundleCode !== 0) throw new Error(`Speech fixture client build failed: ${bundleOut}\n${bundleErr}`); + const [bundleOut, bundleErr, bundleCode] = await runtimeHookPhases("external-speech-runtime").child("client-bundle", bundle, relay); + if (bundleCode !== 0) throw new Error(`Speech fixture client build failed (${bundleCode}, ${bundle.signalCode ?? "no signal"}): ${bundleOut}\n${bundleErr}`); for (const [file, pkg] of [["kit.css", "ui-kit"], ["app.css", "ui-react"]]) { await writeFile(resolve(app.staticRoot, file!), await readFile(resolve(repo, `packages/${pkg}/dist/styles.css`))); } diff --git a/packages/ui-react/tests/fixtures/runtime-hook-phases.ts b/packages/ui-react/tests/fixtures/runtime-hook-phases.ts index 0c33f2b2e4..b4e1290eda 100644 --- a/packages/ui-react/tests/fixtures/runtime-hook-phases.ts +++ b/packages/ui-react/tests/fixtures/runtime-hook-phases.ts @@ -18,6 +18,7 @@ export function runtimeHookPhases(file: string) { } async function child(stage: string, child: { pid: number; stdout: ReadableStream; stderr: ReadableStream; exited: Promise; + readonly signalCode?: string | null; }, consume: (stream: ReadableStream) => Promise = (stream) => new Response(stream).text()): Promise<[string, string, number]> { const details = { childPid: child.pid }; @@ -52,7 +53,7 @@ export function runtimeHookPhases(file: string) { } } const result = await Promise.all([output(child.stdout, "stdout"), output(child.stderr, "stderr"), - child.exited.then((code) => { report(stage, "exit", { ...details, code }); return code; })]); + child.exited.then((code) => { report(stage, "exit", { ...details, code, signal: child.signalCode ?? null }); return code; })]); report(stage, "complete", details); return result; } diff --git a/packages/ui-react/tests/runtime-hook-phases.test.ts b/packages/ui-react/tests/runtime-hook-phases.test.ts index 24f874b141..5ba6fb41d1 100644 --- a/packages/ui-react/tests/runtime-hook-phases.test.ts +++ b/packages/ui-react/tests/runtime-hook-phases.test.ts @@ -26,6 +26,7 @@ test("real child reports exit and each drained channel without exposing output", expect(events).toContain("stderr-complete"); expect(events.at(-1)).toBe("complete"); expect(records.find((r) => r.event === "exit")?.code).toBe(7); + expect(records.find((r) => r.event === "exit")?.signal).toBeNull(); expect(records.find((r) => r.event === "build-progress")?.package).toBe("core"); expect(JSON.stringify(records)).not.toContain("sentinel"); expect(records.every((r) => r.childPid === child.pid)).toBe(true); @@ -48,6 +49,20 @@ test("pending output remains distinguishable from a completed channel", async () expect(records.at(-1)?.event).toBe("complete"); }); +test("a silent signalled child retains its PID, exit code and signal", async () => { + const hook = capture(); + const child = Bun.spawn([process.execPath, "-e", 'process.kill(process.pid, "SIGTERM")'], { stdout: "pipe", stderr: "pipe" }); + const [out, err, code] = await hook.child("client-bundle", child); + expect(out).toBe(""); + expect(err).toBe(""); + expect(code).toBe(143); + expect(records.find((record) => record.event === "exit"), "BUNDLE_CHILD_TERMINATION_IS_ATTRIBUTED").toMatchObject({ + stage: "client-bundle", childPid: child.pid, code: 143, signal: "SIGTERM", + }); + expect(records.some((record) => record.event === "stdout-complete" && record.bytes === 0)).toBe(true); + expect(records.some((record) => record.event === "stderr-complete" && record.bytes === 0)).toBe(true); +}); + test("named phase preserves values and the original rejection", async () => { const hook = capture(); const marker = new Error("private-rejection-sentinel"); From 4440e0f9b5b3f23960c9d6814e73578fbcfb737f Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 16:10:05 +0200 Subject: [PATCH 37/44] fix(storage): honor the approved thirty-day forecast window --- .changeset/storage-history-guidance.md | 2 +- docs/decisions/storage-guidance.md | 5 ++-- docs/plans/storage-growth-experiment.ts | 6 +++- docs/plans/storage-growth.md | 2 +- docs/storage-history.md | 5 ++-- .../ui-server/src/storage/history-evidence.ts | 2 +- .../tests/storage-history-model.test.ts | 29 ++++++++++++++++++- 7 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.changeset/storage-history-guidance.md b/.changeset/storage-history-guidance.md index acd3609dbf..e27b673bee 100644 --- a/.changeset/storage-history-guidance.md +++ b/.changeset/storage-history-guidance.md @@ -3,4 +3,4 @@ "@schlessera/brain-ui-sdk": minor --- -Keep bounded Recent, Daily and Weekly Storage observations, qualified capacity scenarios and independent pressure causes. Retain warning and notification evidence across restart, use existing owner-authorized push subscriptions, and suppress ambiguous or restored delivery attempts. +Keep bounded Recent, Daily and Weekly Storage observations, qualified seven-day and 30-day capacity scenarios and independent pressure causes. Retain warning and notification evidence across restart, use existing owner-authorized push subscriptions, and suppress ambiguous or restored delivery attempts. diff --git a/docs/decisions/storage-guidance.md b/docs/decisions/storage-guidance.md index e435886f37..9dc3799359 100644 --- a/docs/decisions/storage-guidance.md +++ b/docs/decisions/storage-guidance.md @@ -15,8 +15,9 @@ record overhead for longer daily retention; physical SQLite/WAL overhead remains separately measured operational storage. Exhausted metadata capacity pauses history rather than destroying previous valid evidence. -Observed rates describe measured change. Short- and long-window capacity -exhaustion estimates are conditional scenarios, shown only with sufficient +Observed rates describe measured change. The approved [seven-day and 30-day +windows](https://github.com/schlessera/brain-kit/issues/2014#issuecomment-6212083563) +yield conditional capacity-exhaustion scenarios, shown only with sufficient comparable evidence and identified by their windows. A range is not a confidence interval. Missing, partial or incompatible data cannot produce a number or clear a warning. Qualified flat/falling depletion is labelled without infinite headroom. diff --git a/docs/plans/storage-growth-experiment.ts b/docs/plans/storage-growth-experiment.ts index fee98152c2..99ffc913c9 100644 --- a/docs/plans/storage-growth-experiment.ts +++ b/docs/plans/storage-growth-experiment.ts @@ -119,7 +119,7 @@ function forecast(samples: Sample[], now: number, long = false): Forecast { for (let i = samples.length - 1; i >= 0; i--) { const point = samples[i], next = points[0]; if (!point.complete || point.gap || point.available === null || point.scope !== latest.scope || point.basis !== latest.basis || point.total !== latest.total - || latest.at - point.at > (long ? 28 : 7) * day || (next && (next.at <= point.at || next.at - point.at > (long ? 48 : 12) * hour))) break; + || latest.at - point.at > (long ? 30 : 7) * day || (next && (next.at <= point.at || next.at - point.at > (long ? 48 : 12) * hour))) break; points.unshift(point); } if (points.length < (long ? 15 : 5) || latest.at - points[0].at < (long ? 14 : 2) * day) return { kind: "unavailable" }; @@ -138,6 +138,10 @@ const long = capacityTrace(22, day, i => 3_750_000_000 - i * 50_000_000); const shortResult = forecast(short, short.at(-1)!.at), longResult = forecast(long, long.at(-1)!.at, true); assert.deepEqual(shortResult, { kind: "depleting", days: 27 }); assert.deepEqual(longResult, { kind: "depleting", days: 54 }); +// The approved 30-day window includes the early change 29 days ago, while +// excluding the preceding 31-day-old endpoint. A 28-day window loses it. +const month = capacityTrace(32, day, i => 4_000_000_000 - Math.min(i, 2) * 100_000_000 - Math.max(0, i - 2) * 50_000_000); +assert.deepEqual(forecast(month, month.at(-1)!.at, true), { kind: "depleting", days: 2_350_000_000 / (1_550_000_000 / 30) }, "STORAGE_DESIGN_USES_30_DAY_POLICY"); assert.equal(forecast(short.map((p, i) => ({ ...p, total: i === 8 ? p.total + 1 : p.total })), short.at(-1)!.at).kind, "unavailable"); assert.equal(forecast(short, short.at(-1)!.at + 7 * hour).kind, "unavailable"); assert.equal(forecast(short, short.at(-1)!.at - 1).kind, "unavailable"); diff --git a/docs/plans/storage-growth.md b/docs/plans/storage-growth.md index c3707d293d..96ad2a86b0 100644 --- a/docs/plans/storage-growth.md +++ b/docs/plans/storage-growth.md @@ -200,7 +200,7 @@ throughout a window. Never use logical/allocated App growth to forecast capacity | Window | Evidence | | --- | --- | | Short | Recent points within seven days; at least five distinct endpoints spanning 48 hours; no interval over 12 hours. | -| Long | Completed Daily endpoints within 28 days plus the latest Recent endpoint; at least 15 distinct endpoints spanning 14 days; no interval over 48 hours. | +| Long | Completed Daily endpoints within 30 days plus the latest Recent endpoint; at least 15 distinct endpoints spanning 14 days; no interval over 48 hours. | Deduplicate identical timestamps. The latest point must be the current complete capacity observation and at most six hours old, with no future timestamp. Walk diff --git a/docs/storage-history.md b/docs/storage-history.md index bd44c89313..dd00a166d7 100644 --- a/docs/storage-history.md +++ b/docs/storage-history.md @@ -55,8 +55,9 @@ bounds do not promise recovery from arbitrary kernel/filesystem I/O stalls. ## Pressure and recovery Available bytes trigger Low at 10% and Critical at 5%. Qualified conditional -scenarios use separate recent and daily windows; either at 30 days triggers Low, -both at seven days trigger Critical. Each carries its actual evidence window. +scenarios use recent endpoints within seven days and daily endpoints within 30 +days. Either scenario at or below 30 days until exhaustion triggers Low; both +at or below seven days trigger Critical. Each carries its actual evidence window. Neither a projected range nor an observed rate is an accuracy guarantee. Each cause must recover independently. Capacity recovery is strictly above diff --git a/packages/ui-server/src/storage/history-evidence.ts b/packages/ui-server/src/storage/history-evidence.ts index cb37a2d97c..9ad25c18b1 100644 --- a/packages/ui-server/src/storage/history-evidence.ts +++ b/packages/ui-server/src/storage/history-evidence.ts @@ -85,6 +85,6 @@ export function capacityEvidence(payload: HistoryPayload, identity: string return { identity, rootScope: latest.rootScope, at: latest.sample[3], total: latest.capacity[1], available: latest.capacity[4], fresh: latest.capacity[5] !== 2 && latest.capacity[1] !== null && latest.capacity[1] > 0 && latest.capacity[4] !== null && latest.sample[3] <= now && now - latest.sample[3] <= RECENT, short: scenario(recent, 5, 48 * HOUR, 7 * DAY, 12 * HOUR, now), - long: scenario(long, 15, 14 * DAY, 28 * DAY, 48 * HOUR, now), + long: scenario(long, 15, 14 * DAY, 30 * DAY, 48 * HOUR, now), appLogical: rate(recent, 8, now), appAllocated: rate(recent, 9, now), availableChange: rate(recent, 4, now) }; } diff --git a/packages/ui-server/tests/storage-history-model.test.ts b/packages/ui-server/tests/storage-history-model.test.ts index 3afcbe51c2..6ae51761bb 100644 --- a/packages/ui-server/tests/storage-history-model.test.ts +++ b/packages/ui-server/tests/storage-history-model.test.ts @@ -94,7 +94,34 @@ test("short/long scenarios and observed App growth use distinct endpoint quantit expect(result.short.days).toBe(195); expect(result.long.days).toBe(195); expect(result.appAllocated?.bytesPerDay).toBe(24); expect(result.availableChange?.bytesPerDay).toBe(-4_000_000); - expect(result.long.samples).toBeLessThanOrEqual(29); + expect(result.long.samples).toBeLessThanOrEqual(31); +}); +test("long scenarios include the approved 30-day boundary and exclude older endpoints", () => { + const payload = steady(35), now = at + 35 * DAY; + const result = capacityEvidence(payload, identity, now)!; + const start = at + 5 * DAY + 18 * HOUR; + expect(result.long.start, "STORAGE_LONG_SCENARIO_USES_30_DAY_POLICY").toBe(start); + expect(result.long.end).toBe(now); + expect(result.long.samples).toBe(31); + expect(result.long.depletionPerDay).toBe(4_000_000); + expect(result.short.start).toBe(now - 7 * DAY); + // A 30.25-day-old change cannot enter the selected window. + payload.samples.find(row => row[1] === 1 && row[3] === start - DAY)![9][0]![4] = 0; + expect(capacityEvidence(payload, identity, now)?.long).toEqual(result.long); + // The 29.25-day-old endpoint must affect the estimate, even though it was + // excluded by the superseded 28-day implementation. + payload.samples.find(row => row[1] === 1 && row[3] === start)![9][0]![4]! += 3_000_000; + const changed = capacityEvidence(payload, identity, now)!; + expect(changed.long.state).toBe("finite"); + expect(changed.long.depletionPerDay).toBeCloseTo(120_000_000 / 29.25, 4); + expect(changed.short).toEqual(result.short); + const boundary = payload.samples.find(row => row[1] === 1 && row[3] === start)!; + boundary[3] = now - 30 * DAY; boundary[4] = boundary[3] - 100; + boundary[9][0]![2] = 120_000_000; + boundary[9][0]![3] = boundary[9][0]![4] = 880_000_000; + const exact = capacityEvidence(payload, identity, now)!; + expect(exact.long.start).toBe(now - 30 * DAY); + expect(exact.long.depletionPerDay).toBe(4_000_000); }); test("minimum spans, stale/future evidence and gaps refuse scenarios without infinity", () => { const payload = steady(1), now = at + DAY; From dbeabf662339676c92e3e812d9d601090d86bc40 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 17:28:02 +0200 Subject: [PATCH 38/44] test(ci): review the Storage browser command configuration --- scripts/bound-browser-documents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bound-browser-documents.ts b/scripts/bound-browser-documents.ts index 225a40c8e7..972f6dba5f 100644 --- a/scripts/bound-browser-documents.ts +++ b/scripts/bound-browser-documents.ts @@ -194,7 +194,7 @@ export function browserDocumentRefusals(root:string,files:readonly string[],impo function evaluateBrowserDocuments(root:string,files:readonly string[],importers:BrowserImporters){ const result=new Map>(),refusals=new Map(); const matches=(file:string,hash:string)=>{const path=resolve(root,file);return existsSync(path)&&realpathSync(path)===path&&createHash("sha256").update(readFileSync(path)).digest("hex")===hash;}; - if(!matches(config,"e724696a55955e9f32d396b2031214600086533c3f9b8baabcc1c6bb7311f1db")){for(const declaration of declarations)refusals.set(declaration.owner,`unreviewed configuration ${config}`);return {bound:result,refusals};} + if(!matches(config,"89ea7a414a191a5624fce2dff8e0434aac5474c4c5b74d13499848cd17e2b257")){for(const declaration of declarations)refusals.set(declaration.owner,`unreviewed configuration ${config}`);return {bound:result,refusals};} for(const declaration of declarations){ const callers=new Map([[declaration.caller,{hash:declaration.callerHash,calls:declaration.calls}],...Object.entries(declaration.additionalCallers??{})]); const absent=[config,declaration.owner,...callers.keys()].find(file=>!files.includes(file)); From 95dc5e8bfd33b2cb251c891c603b46ce6fee4df3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 17 Oct 2026 18:54:45 +0200 Subject: [PATCH 39/44] test(ci): align remaining audits with Storage registration --- scripts/browser-command-activation.ts | 2 +- scripts/browser-worker-audit.ts | 2 +- scripts/generated-theme-inputs.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/browser-command-activation.ts b/scripts/browser-command-activation.ts index 6fc9cd1536..2f7f7c7fd1 100644 --- a/scripts/browser-command-activation.ts +++ b/scripts/browser-command-activation.ts @@ -211,7 +211,7 @@ export function auditCommandActivation(root: string, config: string, selected: r const auditedSources: Record = { "packages/ui-react/tests/browser/mermaid-proof-command.ts": "df7cc849b743c55c897236cd021793ad489548d780db2d053248116563832d3c", "packages/ui-react/tests/browser/offline/scene-commands.ts": "ad76a69e553d1d0208a5a92c8270876403930788964bd75f3887d06a426c17d2", - "packages/ui-kit/vitest.config.ts": "e724696a55955e9f32d396b2031214600086533c3f9b8baabcc1c6bb7311f1db", + "packages/ui-kit/vitest.config.ts": "89ea7a414a191a5624fce2dff8e0434aac5474c4c5b74d13499848cd17e2b257", "packages/ui-kit/vite.config.ts": "ffdb82cbcf2fd0106799690580ca7af612a0bcceeacf198ef0e8a77313b4cb91", "scripts/visual.mjs": "2576ce91c63843b6d9b2018c477e64734ea8343124170d4fca5cfd8257c3d1c8", "packages/ui-react/tests/browser/hygiene-loop-command.ts": "fd9c5e18f1e374be3657fefd6c7aac76f16794f1ca59f0f038761c75130da35a", diff --git a/scripts/browser-worker-audit.ts b/scripts/browser-worker-audit.ts index 5d0216917e..9d02f79d4a 100644 --- a/scripts/browser-worker-audit.ts +++ b/scripts/browser-worker-audit.ts @@ -8,7 +8,7 @@ export const workerAuditFile="scripts/browser-worker-audit.json"; export const pdfWorkerInput='Undeclared constructed input packages/ui-react/src/lib/pdf.ts: new Worker(url, { type: "module" })'; const guards=["packages/ui-kit/tests/visual/worker-boundary-setup.ts","scripts/browser-worker-boundary.ts"]; const enforcement:Record={ - "packages/ui-kit/vitest.config.ts":"e724696a55955e9f32d396b2031214600086533c3f9b8baabcc1c6bb7311f1db", + "packages/ui-kit/vitest.config.ts":"89ea7a414a191a5624fce2dff8e0434aac5474c4c5b74d13499848cd17e2b257", "scripts/browser-worker-boundary.ts":"fe7b63742370e3a6f2d8f99a77d23ce538d905744ad34d34e99ce342ff62d31b", "packages/ui-kit/tests/visual/worker-boundary-setup.ts":"80ba88b24fc1f0269d0f8a6b58520d715375b8af21ca636112f3294ffd7ad60f" }; diff --git a/scripts/generated-theme-inputs.ts b/scripts/generated-theme-inputs.ts index 701f18b4ab..48fa42da01 100644 --- a/scripts/generated-theme-inputs.ts +++ b/scripts/generated-theme-inputs.ts @@ -7,7 +7,7 @@ import {generatedCssInputs} from "./generated-build-inputs"; import {lockedPackageAsset} from "./locked-browser-assets"; const sources={ "packages/ui-kit/tests/visual/dictation-motion.ts": "426a9ae556c11ea60ab8563c71ee220a3bb7140638b0ef1062aacf1bb3b65e70", - "packages/ui-kit/vitest.config.ts": "e724696a55955e9f32d396b2031214600086533c3f9b8baabcc1c6bb7311f1db", + "packages/ui-kit/vitest.config.ts": "89ea7a414a191a5624fce2dff8e0434aac5474c4c5b74d13499848cd17e2b257", "packages/ui-kit/tests/visual/dictation-theme.css": "dfe3bb18ef30797d9d2ded77a675520035bf0eb8b3d82de180fe640bea84c7c6" }; export function generatedThemeInputs(root:string,files:readonly string[],importers:BrowserImporters){ From 9c4419b2df85c9ef69fd91c768369d713954e743 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sun, 18 Oct 2026 09:37:17 +0200 Subject: [PATCH 40/44] test(ci): own the reviewed Storage theme caller fixture --- .../generated-theme-vitest.config.txt | 322 ++++++++++++++++++ tests/generated-theme-inputs.test.ts | 6 +- 2 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/generated-theme-vitest.config.txt diff --git a/tests/fixtures/generated-theme-vitest.config.txt b/tests/fixtures/generated-theme-vitest.config.txt new file mode 100644 index 0000000000..691e5fbbaf --- /dev/null +++ b/tests/fixtures/generated-theme-vitest.config.txt @@ -0,0 +1,322 @@ +import { diagramArtifact, diagramFontStyles, diagramRasterComparison } from "../ui-react/tests/browser/diagram-export-command.ts"; +import { diagramCapture, diagramFixture } from "../ui-react/tests/browser/diagram-proof-command.ts"; +import { mermaidMedia, mermaidOutput, mermaidScreenComparison } from "../ui-react/tests/browser/mermaid-proof-command.ts"; +import { consumerParityViewport } from "./tests/visual/consumer-parity-viewport.ts"; +import { storageServer } from "../ui-react/tests/browser/storage-server-command.ts"; +import { policyServer } from "../ui-react/tests/browser/policy-server-command.ts"; +import { hygieneLoop } from "../ui-react/tests/browser/hygiene-loop-command.ts"; +import { hygieneServer } from "../ui-react/tests/browser/hygiene-server-command.ts"; +import { dispatchActions } from "../ui-react/tests/browser/dispatch-actions-command.ts"; +import { transcriptionHttp } from "../ui-react/tests/browser/offline/transcription-http.ts"; +import path from "node:path"; +import { MeasuredBrowserSequencer } from "../../scripts/browser-sequencer.ts"; +import { fileURLToPath } from "node:url"; + +import { storybookTest } from "@storybook/addon-vitest/vitest-plugin"; +import { playwright } from "@vitest/browser-playwright"; +import { defineConfig, mergeConfig } from "vitest/config"; + +import { requestLog, startRequestLog } from "./tests/visual/request-log.ts"; +import { grantMicrophone } from "./tests/visual/microphone-permission.ts"; +import { formViewport, formConsumerStyles, htmlPreviewFixture, overlayImageFixture, overlayImageRelease, codeHighlightFailure, codeHighlightFailureCount } from "./tests/visual/form-browser.ts"; +import { dictationThemeStyles, dictationMotion } from "./tests/visual/dictation-motion.ts"; +import { moduleSettingsScreenshot, moduleSettingsImportGate } from "./tests/visual/module-settings-browser.ts"; +import { dictationPointer } from "./tests/visual/dictation-pointer.ts"; +import { focusCapture } from "./tests/visual/focus-browser.ts"; +import { buttonPointer, buttonCapture } from "./tests/visual/button-browser.ts"; +import { ghostMedia, ghostPixels, ghostMaskPixels, ghostTrace } from "./tests/visual/ghost-browser.ts"; +import { rankTap, rankTouch } from "./tests/visual/rank-pointer.ts"; +import { sheetInput } from "./tests/visual/sheet-pointer.ts"; +import { overlayMouse, destinationPaint, destinationMotion, destinationSafeArea } from "./tests/visual/overlay-pointer.ts"; +import { rankFooterFonts, rankFooterDrag, rankFooterCapture } from "./tests/visual/rank-footer-browser.ts"; +import { designFontUsage } from "./tests/visual/design-font-browser.ts"; +import { canvasEncodingLoad } from "./tests/visual/canvas-encoding-load.ts"; +import { coldCapture } from "../ui-react/tests/browser/offline/cold-capture/commands.ts"; +import { shareReview } from "../ui-react/tests/browser/share-review/commands.ts"; +import { offlineScene } from "../ui-react/tests/browser/offline/scene-commands.ts"; +import { fakeMicrophoneFile } from "../ui-react/tests/browser/offline/fake-microphone-file.ts"; +import viteConfig from "./vite.config.ts"; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); + +// Every story becomes a Vitest test — a render smoke test plus its play +// function — running in a real Chromium through Playwright, not happy-dom. +// This is a SEPARATE runner from the repo's `bun test`; the two coexist and +// are run as two commands. +// +// There is deliberately no `.storybook/vitest.setup.ts`: since Storybook 10.3 +// the addon applies `setProjectAnnotations` itself, and adding the file back +// makes it skip that and warn. The published docs snippets are stale on this. +// +// Two story projects, one per theme — D9's matrix. `initialGlobals` pins the +// addon-themes `theme` global for every story the project runs, so the light +// project renders all 540-odd stories on paper with the same play functions +// and the same a11y gate at `'error'`. That is the light theme's contrast +// proof: axe on every rendered story, not a table of numbers. +const storyProject = (name: string, theme: string) => ({ + extends: true, + plugins: [ + storybookTest({ + configDir: path.join(dirname, ".storybook"), + storybookScript: "bun run storybook --no-open", + storybookUrl: process.env.SB_URL, + initialGlobals: { theme }, + }), + ], + test: { + name, + provide: { storyTheme: theme }, + setupFiles: ["./tests/visual/story-theme-setup.ts"], + browser: { + enabled: true, + commands: { consumerParityViewport }, + provider: playwright({}), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, +}); + +// Chromium's pointer settings are bit fields: coarse=2, fine=4. Keep each +// input scene in its own browser; the tests assert the actual media queries +// before measuring CSS or dispatching native touch input (no matchMedia mock). +// The rail, the overlay discs and the working-session strip (#949) share these +// scenes. The strip's pixel baselines are taken in `rail-mixed` only, so one +// image covers each case. +const railProject = (mode: "fine" | "coarse" | "mixed") => ({ + extends: true, + test: { + name: `rail-${mode}`, + // The app's own rail and palette (#946), phone navigation (#947) and + // pressing the current destination (#1078) and the working sessions + // (#950) and the #929 reach proof (#953) run here too, for the real + // pointer; they need the consumer stylesheet. Button's hover text + // contrast (#974) and the palette's reason rows (#1106) are measured + // under all three pointers as well. + include: ["tests/visual/native-buttons.visual.tsx", "tests/visual/side-rail-targets.visual.tsx", "tests/visual/button-hover-contrast.visual.tsx", "tests/visual/overlay-targets.visual.tsx", "tests/visual/session-strip.visual.tsx", "tests/visual/pending-follow-ups.visual.tsx", "tests/visual/palette-reasons.visual.tsx", "../ui-react/tests/browser/desktop-navigation.pointer.tsx", "../ui-react/tests/browser/keyboard-hints.pointer.tsx", "../ui-react/tests/browser/phone-navigation.pointer.tsx", "../ui-react/tests/browser/destination-press.pointer.tsx", "../ui-react/tests/browser/working-sessions.pointer.tsx", "../ui-react/tests/browser/session-drafts.pointer.tsx", "../ui-react/tests/browser/navigation-reach.pointer.tsx", "../ui-react/tests/browser/recordings-tray.offline.tsx", "../ui-react/tests/browser/dictation-notice.pointer.tsx", "../ui-react/tests/browser/overlay-stacking.pointer.tsx", "../ui-react/tests/browser/session-profile.pointer.tsx", "../ui-react/tests/browser/turn-budget.pointer.tsx"], + provide: { railPointer: mode, keyboardHintsLive: false }, + browser: { + enabled: true, + commands: { sheetInput, rankTouch, rankTap, formViewport, formConsumerStyles, htmlPreviewFixture, overlayImageFixture, overlayImageRelease, buttonPointer, focusCapture, overlayMouse, destinationPaint, destinationMotion, destinationSafeArea, dictationMotion }, + provider: playwright({ + launchOptions: { args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", `--use-file-for-fake-audio-capture=${fakeMicrophoneFile()}`, "--autoplay-policy=no-user-gesture-required", `--blink-settings=availablePointerTypes=${mode === "mixed" ? 6 : mode === "coarse" ? 2 : 4},primaryPointerType=${mode === "coarse" ? 2 : 4}`] }, + contextOptions: { reducedMotion: "reduce" }, + }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, +}); + +export default mergeConfig( + viteConfig, + defineConfig({ + // The story addon inserts preview-head.html into its runner HTML. Its + // network font links would compete with the shared locked offline faces. + plugins: [{ + name: "locked-preview-fonts", + transformIndexHtml: { + order: "post", + handler: (html) => html.replace(/]*href="https:\/\/fonts\.(?:googleapis|gstatic)\.com(?:\/[^\"]*)?"[^>]*>/g, ""), + }, + }], + test: { + sequence: { sequencer: MeasuredBrowserSequencer }, + provide: { proofWorkerMode: "selected-zero-v1" }, + setupFiles: ["./tests/visual/worker-boundary-setup.ts", "./tests/visual/design-font-setup.ts"], + browser: { commands: { designFonts: rankFooterFonts, designFontUsage } }, + projects: [ + { + extends: true, + test: { + name: "diagrams-chromium", + include: ["../ui-react/tests/browser/diagrams-*.layout.tsx"], + browser: { + enabled: true, screenshotFailures: false, + commands: { diagramRasterComparison, diagramFontStyles, diagramArtifact, mermaidScreenComparison, diagramCapture, diagramFixture, overlayMouse, mermaidMedia, mermaidOutput, formConsumerStyles, formViewport }, + provider: playwright({ contextOptions: { reducedMotion: "reduce" } }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + name: "diagrams-webkit", + include: ["../ui-react/tests/browser/diagrams-*.layout.tsx"], + browser: { + enabled: true, screenshotFailures: false, + commands: { diagramRasterComparison, diagramFontStyles, diagramArtifact, mermaidScreenComparison, diagramCapture, diagramFixture, overlayMouse, mermaidMedia, mermaidOutput, formConsumerStyles, formViewport }, + provider: playwright({ contextOptions: { reducedMotion: "reduce" } }), + headless: true, + instances: [{ browser: "webkit" }], + }, + }, + }, + { + extends: true, + test: { + name: "ui-react-layout", + provide: { proofWorkerMode: "fresh-worker-runtime-v1" }, + // Measurements of the real consumer, kept out of Bun's test glob, + // and the offline fault harness's self-tests (#1016). + include: ["../ui-react/tests/browser/**/*.layout.tsx", "../ui-react/tests/browser/**/*.offline.tsx"], + exclude: ["../ui-react/tests/browser/diagrams-*.layout.tsx"], + browser: { + enabled: true, + screenshotFailures: false, + commands: { mermaidMedia, mermaidOutput, mermaidScreenComparison, buttonCapture, overlayMouse, dictationMotion, policyServer, storageServer, hygieneLoop, hygieneServer, dispatchActions, sheetInput, focusCapture, formViewport, formConsumerStyles, rankFooterFonts, codeHighlightFailure, codeHighlightFailureCount, offlineScene, coldCapture, shareReview, transcriptionHttp, startRequestLog, requestLog, grantMicrophone }, + // The fake microphone plays the generated 10-second fixture + // (#1016). Chromium reads the file at launch, so it is written + // when this config loads; the fake UI grants the permission. + // Audio contexts start without a gesture, as the injected + // microphone and the tone check need. + provider: playwright({ + launchOptions: { args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", `--use-file-for-fake-audio-capture=${fakeMicrophoneFile()}`, "--autoplay-policy=no-user-gesture-required"] }, + contextOptions: { reducedMotion: "reduce" }, + }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + name: "ui-react-endurance", + provide: { proofWorkerMode: "fresh-worker-runtime-v1" }, + // Real-time native media proof has its own scheduling and discovery; + // keep the same offline launch, commands and lifecycle as layout. + include: ["../ui-react/tests/browser/**/*.endurance.tsx"], + browser: { + enabled: true, + screenshotFailures: false, + commands: { buttonCapture, overlayMouse, dictationMotion, policyServer, hygieneLoop, hygieneServer, dispatchActions, sheetInput, focusCapture, formViewport, formConsumerStyles, rankFooterFonts, codeHighlightFailure, codeHighlightFailureCount, offlineScene, coldCapture, shareReview, transcriptionHttp, startRequestLog, requestLog, grantMicrophone }, + // The fake microphone plays the generated 10-second fixture + // (#1016). Chromium reads the file at launch, so it is written + // when this config loads; the fake UI grants the permission. + // Audio contexts start without a gesture, as the injected + // microphone and the tone check need. + provider: playwright({ + launchOptions: { args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", `--use-file-for-fake-audio-capture=${fakeMicrophoneFile()}`, "--autoplay-policy=no-user-gesture-required"] }, + contextOptions: { reducedMotion: "reduce" }, + }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + storyProject("storybook", "dark"), + storyProject("storybook-light", "light"), + railProject("fine"), + railProject("coarse"), + railProject("mixed"), + { + extends: true, + test: { + name: "visual", + // `.visual.tsx`, deliberately NOT `.test.tsx`: the repo's `bun run test` + // globs `packages/**` for `*.test.ts(x)` and would claim these, then + // fail on the first story import because bun's runner has no Vite and + // cannot resolve `#.storybook/preview`. Two runners, two extensions. + include: ["tests/visual/**/*.visual.tsx"], + exclude: ["tests/visual/native-buttons.visual.tsx", "tests/visual/rank-footer-touch.visual.tsx", "tests/visual/module-settings.visual.tsx", "tests/visual/subjects.visual.tsx", "tests/visual/dictation-panel.visual.tsx", "tests/visual/side-rail-targets.visual.tsx", "tests/visual/button-hover-contrast.visual.tsx", "tests/visual/overlay-targets.visual.tsx", "tests/visual/session-strip.visual.tsx", "tests/visual/pending-follow-ups.visual.tsx", "tests/visual/palette-reasons.visual.tsx"], + browser: { + enabled: true, + // The link card's no-request proof reads the network from + // Playwright (`tests/visual/request-log.ts`). + commands: { sheetInput, startRequestLog, requestLog, rankTouch, rankFooterFonts, rankFooterDrag, rankFooterCapture, formViewport, formConsumerStyles, buttonPointer, buttonCapture, focusCapture, ghostMedia, ghostPixels, ghostMaskPixels, ghostTrace, canvasEncodingLoad, dictationMotion }, + // Chromium's own web-test mode retains native encoding without + // waiting for idle periods that can exceed a functional poll (#940). + provider: playwright({ launchOptions: { args: ["--enable-blink-features=NoIdleEncodingForWebTests"] } }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + // Chromium's touch-emulation disable does not restore a fine + // pointer. Keep this complete consumer fixture in its own provider + // lifetime, even when files in another project reuse their page. + name: "dictation", + include: ["tests/visual/dictation-panel.visual.tsx"], + browser: { + enabled: true, + commands: { sheetInput, formViewport, formConsumerStyles, dictationPointer, dictationThemeStyles, dictationMotion, rankTouch }, + provider: playwright({}), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + // A failure screenshot in another file changes Chromium's used + // monospace fallback even after that file restores its DOM/viewport. + // Keep incumbent subject pixels in a separate browser process (#879). + name: "subjects", + include: ["tests/visual/subjects.visual.tsx"], + browser: { + enabled: true, + commands: { formViewport }, + provider: playwright({}), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + // Consumer styles, viewport changes and font fallback must not + // share the kit baseline project's browser process. + name: "module-settings", + include: ["tests/visual/module-settings.visual.tsx"], + browser: { + enabled: true, + commands: { formViewport, formConsumerStyles, moduleSettingsScreenshot, moduleSettingsImportGate }, + provider: playwright({}), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + name: "rank-footer-touch", + include: ["tests/visual/rank-footer-touch.visual.tsx"], + browser: { + enabled: true, + commands: { rankTouch, rankFooterFonts, rankFooterDrag, rankFooterCapture }, + provider: playwright({ contextOptions: { hasTouch: true } }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + { + extends: true, + test: { + // Touch emulation does not restore fine-pointer media on disable. + // Own its browser lifetime so live hint proof cannot affect other files. + name: "keyboard-hints-live", + include: ["../ui-react/tests/browser/keyboard-hints.pointer.tsx"], + provide: { railPointer: "fine", keyboardHintsLive: true }, + browser: { + enabled: true, + commands: { dictationPointer, formViewport, formConsumerStyles, rankTouch, rankTap }, + provider: playwright({ contextOptions: { reducedMotion: "reduce" } }), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, + }, + ], + }, + }) +); diff --git a/tests/generated-theme-inputs.test.ts b/tests/generated-theme-inputs.test.ts index e9949047e5..5d534a792a 100644 --- a/tests/generated-theme-inputs.test.ts +++ b/tests/generated-theme-inputs.test.ts @@ -7,8 +7,12 @@ import {browserImporters} from "../scripts/bound-browser-runtime-inputs"; import {generatedThemeInputs} from "../scripts/generated-theme-inputs"; const repo=resolve(import.meta.dir,".."),roots:string[]=[],owner="packages/ui-kit/tests/visual/dictation-motion.ts",caller="packages/ui-kit/vitest.config.ts",entry="packages/ui-kit/tests/visual/dictation-theme.css"; const producers=["scripts/build.ts","scripts/build-receipt.ts","scripts/build-repeat-observation.ts","scripts/bin-modes.ts","scripts/publishable-packages.ts","scripts/workspace-lease.mjs","scripts/ci-proof.ts","scripts/check-pr.ts","scripts/ci-runner.ts",".github/workflows/ci.yml","package.json","bun.lock","packages/ui-react/src/theme.css","packages/ui-react/package.json"]; +// The reviewed Storage caller has its own committed positive fixture. Shared +// producer inputs retain their independent main owner; current config edits +// cannot redefine this positive, and feature-branch Git history is not required. +const callerFixture=new URL("./fixtures/generated-theme-vitest.config.txt",import.meta.url); afterEach(()=>{for(const root of roots.splice(0))rmSync(root,{recursive:true,force:true});}); -function fixture(){const root=mkdtempSync(join(tmpdir(),"generated-theme-"));roots.push(root);const write=(file:string,text:string)=>{mkdirSync(dirname(join(root,file)),{recursive:true});writeFileSync(join(root,file),text);};for(const file of [...producers,owner,caller,entry])write(file,frozenGeneratedSource(file));symlinkSync(join(repo,"node_modules"),join(root,"node_modules"),"dir");return{root,write};} +function fixture(){const root=mkdtempSync(join(tmpdir(),"generated-theme-"));roots.push(root);const write=(file:string,text:string)=>{mkdirSync(dirname(join(root,file)),{recursive:true});writeFileSync(join(root,file),text);};for(const file of [...producers,owner,caller,entry])write(file,file===caller?readFileSync(callerFixture,"utf8"):frozenGeneratedSource(file));symlinkSync(join(repo,"node_modules"),join(root,"node_modules"),"dir");return{root,write};} test("temporary theme output binds the audited command, copied theme and installed compiler",()=>{ const {root}=fixture(),files=[owner,caller],bound=generatedThemeInputs(root,files,browserImporters(root,files)); expect(bound?.css).toEqual([entry,"packages/ui-react/src/theme.css"]); From 832c20879a3e55412da8d713268f2285988ea7ba Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 19 Oct 2026 12:05:10 +0200 Subject: [PATCH 41/44] test(ci): observe HTTP keepalive timers by owner --- .../tests/helpers/http-mount-timers.ts | 26 +++++++--- .../ui-server/tests/http-mount-timers.test.ts | 48 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 packages/ui-server/tests/http-mount-timers.test.ts diff --git a/packages/ui-server/tests/helpers/http-mount-timers.ts b/packages/ui-server/tests/helpers/http-mount-timers.ts index d4863c7818..06702a48e6 100644 --- a/packages/ui-server/tests/helpers/http-mount-timers.ts +++ b/packages/ui-server/tests/helpers/http-mount-timers.ts @@ -1,18 +1,30 @@ import { spyOn } from "bun:test"; +import { globalAgent } from "node:http"; /** Observe real timers, including unref'ed ones Bun's active-resource API omits. */ export function observeMountTimers() { const pending = new Set>(); const timeout = globalThis.setTimeout, interval = globalThis.setInterval; const clearTimeoutOriginal = globalThis.clearTimeout, clearIntervalOriginal = globalThis.clearInterval; + const keepaliveDescriptor = Object.getOwnPropertyDescriptor(globalAgent, "keepSocketAlive"); + const keepalive = globalAgent.keepSocketAlive; + let inKeepalive = false; + // This owner's native timer may bypass the global clear observer. Fixture + // socket close is joined and asserted separately; identify its actual call + // instead of depending on optional/formatted Error.stack frames. + Object.defineProperty(globalAgent, "keepSocketAlive", { + configurable: true, writable: true, + value: function (this: typeof globalAgent, ...args: Parameters) { + const previous = inKeepalive; + inKeepalive = true; + try { return Reflect.apply(keepalive, this, args); } + finally { inKeepalive = previous; } + }, + }); const timeoutSpy = spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => { const handle = timeout((...values: unknown[]) => { pending.delete(handle); callback(...values); }, delay, ...args); - // Bun's Node HTTP agent cancels this one timer through a captured native - // clear reference. Its fixture-owned socket close is joined and asserted - // separately. Unknown origins (including SDK timers) remain in the ledger. - const stack = new Error().stack ?? ""; - const nodeKeepalive = stack.includes("at keepSocketAlive (node:_http_agent:") && stack.includes("at setTimeout (node:net:"); - if (!nodeKeepalive) pending.add(handle); + // Unknown origins (including SDK and other socket timers) remain observed. + if (!inKeepalive) pending.add(handle); return handle; }) as typeof setTimeout); const intervalSpy = spyOn(globalThis, "setInterval").mockImplementation(((...args: Parameters) => { @@ -31,5 +43,7 @@ export function observeMountTimers() { return { pending, restore() { timeoutSpy.mockRestore(); intervalSpy.mockRestore(); clearTimeoutSpy.mockRestore(); clearIntervalSpy.mockRestore(); + if (keepaliveDescriptor) Object.defineProperty(globalAgent, "keepSocketAlive", keepaliveDescriptor); + else Reflect.deleteProperty(globalAgent, "keepSocketAlive"); } }; } diff --git a/packages/ui-server/tests/http-mount-timers.test.ts b/packages/ui-server/tests/http-mount-timers.test.ts new file mode 100644 index 0000000000..2aadca971f --- /dev/null +++ b/packages/ui-server/tests/http-mount-timers.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; +import { globalAgent } from "node:http"; +import { Socket } from "node:net"; +import { observeMountTimers } from "./helpers/http-mount-timers"; + +test("HTTP keepalive ownership survives shallow stacks without hiding application timers", () => { + const stackDescriptor = Object.getOwnPropertyDescriptor(Error, "stackTraceLimit")!; + const agentDescriptor = Object.getOwnPropertyDescriptor(globalAgent, "keepSocketAlive"); + const timers = observeMountTimers(); + const socket = new Socket(); + let timeout: ReturnType | undefined; + let interval: ReturnType | undefined; + try { + Object.defineProperty(Error, "stackTraceLimit", { ...stackDescriptor, value: 1 }); + expect(globalAgent.keepSocketAlive(socket)).toBe(true); + expect(socket.timeout).toBeGreaterThan(0); + expect(timers.pending.size, "HTTP_KEEPALIVE_OWNER_DOES_NOT_REQUIRE_STACK_FRAMES").toBe(0); + + timeout = setTimeout(() => {}, 60_000).unref(); + interval = setInterval(() => {}, 60_000).unref(); + expect(timers.pending.has(timeout), "ACTIVE_UNREF_TIMEOUT_REMAINS_OBSERVED").toBe(true); + expect(timers.pending.has(interval), "ACTIVE_UNREF_INTERVAL_REMAINS_OBSERVED").toBe(true); + clearTimeout(timeout); + clearInterval(interval); + expect(timers.pending.size).toBe(0); + } finally { + socket.setTimeout(0); + socket.destroy(); + clearTimeout(timeout); + clearInterval(interval); + timers.restore(); + Object.defineProperty(Error, "stackTraceLimit", stackDescriptor); + } + expect(Object.getOwnPropertyDescriptor(globalAgent, "keepSocketAlive")).toEqual(agentDescriptor); +}); + +test("a socket timer outside HTTP keepalive remains observed", () => { + const timers = observeMountTimers(); + const socket = new Socket(); + try { + socket.setTimeout(60_000); + expect(timers.pending.size, "UNKNOWN_SOCKET_TIMER_REMAINS_OBSERVED").toBe(1); + } finally { + socket.setTimeout(0); + socket.destroy(); + timers.restore(); + } +}); From 47e5032ed682d7d866930eb85566703d0a86a364 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 19 Oct 2026 12:08:54 +0200 Subject: [PATCH 42/44] test(ci): assert socket effects without an undocumented return type --- packages/ui-server/tests/http-mount-timers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui-server/tests/http-mount-timers.test.ts b/packages/ui-server/tests/http-mount-timers.test.ts index 2aadca971f..18dedf8e7d 100644 --- a/packages/ui-server/tests/http-mount-timers.test.ts +++ b/packages/ui-server/tests/http-mount-timers.test.ts @@ -12,7 +12,7 @@ test("HTTP keepalive ownership survives shallow stacks without hiding applicatio let interval: ReturnType | undefined; try { Object.defineProperty(Error, "stackTraceLimit", { ...stackDescriptor, value: 1 }); - expect(globalAgent.keepSocketAlive(socket)).toBe(true); + globalAgent.keepSocketAlive(socket); expect(socket.timeout).toBeGreaterThan(0); expect(timers.pending.size, "HTTP_KEEPALIVE_OWNER_DOES_NOT_REQUIRE_STACK_FRAMES").toBe(0); From 4823227be4d733c116cbd227169cae11179048d2 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 19 Oct 2026 12:24:33 +0200 Subject: [PATCH 43/44] test(core): isolate busy-checkpoint coverage from bulk indexing --- packages/core/tests/indexer.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/core/tests/indexer.test.ts b/packages/core/tests/indexer.test.ts index 45eaa73fc9..357a7cca97 100644 --- a/packages/core/tests/indexer.test.ts +++ b/packages/core/tests/indexer.test.ts @@ -1418,7 +1418,12 @@ describe("the WAL after an index run (#423)", () => { }); test("a reader holding a transaction open makes the checkpoint busy, not the run", async () => { - const root = bulkyCorpus(); + // A read snapshot makes the checkpoint busy regardless of corpus size. + // The preceding controls retain the multi-megabyte truncation proof. + const root = makeCorpus({ + "notes/bulk-0.md": md("Bulk 0", "Odysseus prepares the return voyage."), + "notes/unchanged.md": md("Unchanged", "Penelope keeps the arrival record."), + }); await runIndex(root); writeFileSync(join(root, "notes/bulk-0.md"), md("Bulk 0", "rewritten so the run writes")); From feef19cbb0dda994cf4cd217e0e2ba005095b598 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 19 Oct 2026 15:46:39 +0200 Subject: [PATCH 44/44] fix(storage): restore package loading and maintenance proof --- .../ui-sdk/tests/owned-prune-variant.test.ts | 24 +++++++++---- packages/ui-server/src/app.ts | 2 +- packages/ui-server/src/storage/process.ts | 2 +- .../ui-server/src/storage/removal-facts.ts | 3 +- packages/ui-server/src/storage/removal.ts | 27 +++++++++++--- scripts/check-ui-server-query-package.ts | 35 ++++++++++++++++++- scripts/native-proof/owned-prune-variant.ts | 15 ++++---- 7 files changed, 85 insertions(+), 23 deletions(-) diff --git a/packages/ui-sdk/tests/owned-prune-variant.test.ts b/packages/ui-sdk/tests/owned-prune-variant.test.ts index 1259b7b7ea..bd5cb2a69d 100644 --- a/packages/ui-sdk/tests/owned-prune-variant.test.ts +++ b/packages/ui-sdk/tests/owned-prune-variant.test.ts @@ -78,14 +78,19 @@ test("real original and variant boundary methods preserve owned operation order, const descriptor = Object.getOwnPropertyDescriptor(console, "log")!, diagnostics: unknown[] = []; Object.defineProperty(console, "log", { ...descriptor, value: (line: string) => diagnostics.push(JSON.parse(line)) }); const failure = Error("Odysseus owned operation refusal"); - async function execute(source: string, fail: boolean) { + async function execute(source: string, fail: boolean, publish = true) { const signal = new AbortController().signal, logger = { emit() {} }, launch = { command: ["/bin/true"] }; + const areas = ["scratch"]; const calls: string[] = [], view = { async cleanup() { calls.push("view-cleanup"); } }; const output = { view, output: {}, async publish() { calls.push("publish"); }, async cleanup() { calls.push("output-cleanup"); } }; const dependencies = { gated: async (_: unknown, work: (hold: null) => unknown) => { calls.push("gate"); return work(null); }, isolated: () => true, credentialed: async (_: unknown, work: () => unknown) => { calls.push("credential"); return work(); }, - prepareInput: async () => { calls.push("view"); return view; }, prepareMaintenanceOutput: async () => { calls.push("output"); return output; }, + prepareInput: async () => { calls.push("view"); return view; }, prepareMaintenanceOutput: async (_: unknown, options: { areas: string[]; signal: AbortSignal; startupView: unknown }) => { + expect(options.areas, "OWNED_PRUNE_OUTPUT_AREAS_PRESERVED").toBe(areas); + expect(options.signal).toBe(signal); expect(options.startupView).toBe(view); + calls.push("output"); return output; + }, options: { log: logger }, command: () => launch, prepareMaintenanceGit: async (...args: unknown[]) => { expect(args.length, "OWNED_PRUNE_GIT_ARGUMENT_COUNT_PRESERVED").toBe(4); expect(args[0], "OWNED_PRUNE_GIT_COMMAND_IDENTITY_PRESERVED").toBe(launch); @@ -94,22 +99,29 @@ test("real original and variant boundary methods preserve owned operation order, expect(args[3], "OWNED_PRUNE_GIT_LOGGER_IDENTITY_PRESERVED").toBe(logger); calls.push("git"); return undefined; }, - runMaintenanceWorker: async () => { calls.push("worker"); if (fail) throw failure; return { stdout: "Odysseus", stderr: "", exitCode: 0, killed: false }; }, + runMaintenanceWorker: async (options: { signal: AbortSignal; timeoutMs: number; outputBytes: number }, beforeStart: () => void) => { + expect(options, "OWNED_PRUNE_WORKER_BOUNDS_PRESERVED").toMatchObject({ signal, timeoutMs: 1000, outputBytes: 4096 }); + beforeStart(); calls.push("worker"); if (fail) throw failure; return { stdout: "Odysseus", stderr: "", exitCode: 0, killed: false }; + }, outputLock: { async withLock(_brain: string, work: () => unknown) { calls.push("output-lock"); return work(); } }, brain: "Odysseus brain", }; const start = source.indexOf(" async run(execution) {"), end = source.indexOf(" async launch(execution) {", start); const javascript = new Bun.Transpiler({ loader: "ts" }).transformSync("const operation = {" + source.slice(start, end) + "}; return operation.run;"); const operation = new Function(...Object.keys(dependencies), javascript)(...Object.values(dependencies)); - try { return { result: await operation({ purpose: "brain scratch prune", signal }), calls }; } + try { return { result: await operation({ purpose: "brain scratch prune", signal, outputAreas: areas, outputBytes: 4096, timeoutMs: 1000, publish, + assertAuthorized() { calls.push("authorize"); }, onExecute() { calls.push("on-execute"); } }), calls }; } catch (error) { return { error, calls }; } } try { const original = await execute(boundary, false), observed = await execute(variant, false); - expect(observed).toEqual(original); - expect(observed.calls).toEqual(["gate", "credential", "output-lock", "view", "output", "git", "worker", "publish", "output-cleanup", "view-cleanup"]); + expect(observed, "OWNED_PRUNE_AUTHORITY_AND_EXECUTION_PRESERVED").toEqual(original); + expect(observed.calls).toEqual(["authorize", "gate", "credential", "output-lock", "authorize", "view", "output", "git", "authorize", "authorize", "on-execute", "worker", "authorize", "publish", "output-cleanup", "view-cleanup"]); const failedOriginal = await execute(boundary, true), failedObserved = await execute(variant, true); expect(failedObserved.calls).toEqual(failedOriginal.calls); expect(failedObserved.error).toBe(failure); expect(failedOriginal.error).toBe(failure); expect(failedObserved.calls.slice(-2)).toEqual(["output-cleanup", "view-cleanup"]); + const unpublishedOriginal = await execute(boundary, false, false), unpublishedObserved = await execute(variant, false, false); + expect(unpublishedObserved, "OWNED_PRUNE_DISCARD_PRESERVED").toEqual(unpublishedOriginal); + expect(unpublishedObserved.calls).not.toContain("publish"); expect(diagnostics.every(record => Object.keys(record as object).join() === "fixtureOwnedPruneBoundary")).toBe(true); } finally { Object.defineProperty(console, "log", descriptor); } }); diff --git a/packages/ui-server/src/app.ts b/packages/ui-server/src/app.ts index 12f2bae5da..a0a1f3b8dd 100644 --- a/packages/ui-server/src/app.ts +++ b/packages/ui-server/src/app.ts @@ -4,7 +4,7 @@ import { createSharedStorageScan } from "./storage/shared-scan.js"; import { createStorageNotifier } from "./storage/notify.js"; import { createStorageGuidance } from "./storage/guidance.js"; import { createStorageSampling } from "./storage/sampling.js"; -import storagePackage from "../package.json"; +import storagePackage from "../package.json" with { type: "json" }; import { createStorageRemoval } from "./storage/removal.js"; import { createStorageRemovalRoutes } from "./routes/storage-removal.js"; import { createStorageMaintenance } from "./storage/maintenance.js"; diff --git a/packages/ui-server/src/storage/process.ts b/packages/ui-server/src/storage/process.ts index a61a8f2fd9..f6f59304cb 100644 --- a/packages/ui-server/src/storage/process.ts +++ b/packages/ui-server/src/storage/process.ts @@ -5,7 +5,7 @@ import { realpathSync, readFileSync } from "node:fs"; import type { MetadataRoot, MetadataResult } from "./metadata.js"; const inside = (root: string, path: string) => { const rel = relative(root, path); return !rel || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith("../")); }; /** Verify the installed public reader, never code or executables from content. */ -function coreReader(brainPath: string, subpath: "storage" | "queries"): string { +export function coreReader(brainPath: string, subpath: "storage" | "queries"): string { const manifestPath = realpathSync(fileURLToPath(import.meta.resolve("@schlessera/brain/package.json"))); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); if (manifest.name !== "@schlessera/brain") throw Error("storage-reader-unavailable"); diff --git a/packages/ui-server/src/storage/removal-facts.ts b/packages/ui-server/src/storage/removal-facts.ts index 4bd1ff18a6..5ef8e653ca 100644 --- a/packages/ui-server/src/storage/removal-facts.ts +++ b/packages/ui-server/src/storage/removal-facts.ts @@ -1,14 +1,13 @@ import { subprocessEnv } from "../config/env.js"; /** Repository evaluation stays in a disposable worker; even incidental Git writes are discarded. */ import { z } from "zod"; -import type { StorageAttachmentFacts } from "@schlessera/brain/storage"; import { MAINTENANCE_ENTRY, type HostBoundary } from "../brain/host-boundary.js"; const relative = z.string().min(1).max(4096).refine(value => !value.startsWith("/") && !/[\\\u0000-\u001f\u007f]/u.test(value) && value.split("/").every(part => part && part !== "." && part !== "..")); const hash = z.string().regex(/^[a-f0-9]{64}$/); const schema = z.object({ fingerprint: hash, complete: z.boolean(), scannedFiles: z.number().int().min(0).max(2000), unchecked: z.array(z.string().max(160)).max(16), files: z.array(z.object({ path: relative, git: z.enum(["tracked", "untracked", "ignored", "no-repository", "unknown"]), truncated: z.boolean(), references: z.array(z.object({ source: relative, hash, certainty: z.enum(["resolved", "possible"]) }).strict()).max(200) }).strict()).max(32) }).strict(); -export async function readRemovalFacts(boundary: Pick, root: string, paths: string[], signal: AbortSignal, assertAuthorized: () => void): Promise { +export async function readRemovalFacts(boundary: Pick, root: string, paths: string[], signal: AbortSignal, assertAuthorized: () => void): Promise> { if (!boundary.isolated()) throw Error("Attachment review requires the qualified worker boundary."); const result = await boundary.run({ argv: [process.execPath, "--config=/dev/null", "--no-env-file", MAINTENANCE_ENTRY], env: subprocessEnv("brainCli", { NO_COLOR: "1" }), credentials: false, gate: "none", purpose: "storage-removal-review", publish: false, outputBytes: 262144, timeoutMs: 30_000, diff --git a/packages/ui-server/src/storage/removal.ts b/packages/ui-server/src/storage/removal.ts index 165fa6ebd0..fabb1a811d 100644 --- a/packages/ui-server/src/storage/removal.ts +++ b/packages/ui-server/src/storage/removal.ts @@ -3,7 +3,8 @@ import { subprocessEnv } from "../config/env.js"; import type { Database } from "bun:sqlite"; import { realpathSync } from "node:fs"; import { createHash } from "node:crypto"; -import { inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment, storageAttachmentPath, storageAttachmentVacant, StorageAttachmentRefusal, type StorageAttachmentVersion, type StorageAttachmentFacts } from "@schlessera/brain/storage"; +import { createRequire } from "node:module"; +import type { StorageAttachmentVersion } from "@schlessera/brain/storage"; import type { JournalVersion, StorageRemovalInspection, StorageRemovalCapability, StorageRemovalReview, StorageRemovalReceipt, StorageRemovalFile } from "@schlessera/brain-ui-sdk/protocol"; import { BRAIN_LOCK_KEY, type KeyedLock } from "@schlessera/brain-ui-sdk/server"; import { GIT_LOCK_KEY } from "@schlessera/brain-ui-sdk/internal"; @@ -13,6 +14,7 @@ import { SELF, processAlive, type WriterGate } from "../brain/writer-gate.js"; import { StorageUnavailable, type StorageCoordinator } from "./coordinator.js"; import { readRemovalFacts } from "./removal-facts.js"; import { storageObservation } from "./maintenance.js"; +import { coreReader } from "./process.js"; export const STORAGE_REMOVAL_LIMITS = Object.freeze({ items: 32, fileBytes: 64 * 1024 * 1024, totalBytes: 128 * 1024 * 1024, milliseconds: 300_000, reviewMilliseconds: 60_000, receiptBytes: 262_144, receipts: 128 }); const REVIEW_TTL = 10 * 60_000, runningHere = new Set(); @@ -20,7 +22,7 @@ const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(va const prefix = (path: string, root: string) => root === "." || path === root || path.startsWith(`${root}/`); type PrivateFile = { path: string; version: StorageAttachmentVersion | null; consequences: string; retained: JournalVersion | null }; type Row = { id: string; principal_id: string; action: "remove" | "undo"; source_id: string | null; selection: string; fingerprint: string; journal_turn_id: string; state: StorageRemovalReceipt["state"]; pid: number; pid_start: string; receipt: string; private_versions: string }; -const factIdentity = (facts: StorageAttachmentFacts, path: string) => digest({ file: facts.files.find(file => file.path === path), complete: facts.complete, unchecked: facts.unchecked }); +const factIdentity = (facts: Awaited>, path: string) => digest({ file: facts.files.find(file => file.path === path), complete: facts.complete, unchecked: facts.unchecked }); const encode = (value: unknown) => { const text = JSON.stringify(value); if (Buffer.byteLength(text) > STORAGE_REMOVAL_LIMITS.receiptBytes) throw new StorageUnavailable(400, "Attachment receipt exceeds its bound."); return text.padEnd(STORAGE_REMOVAL_LIMITS.receiptBytes); }; const validId = (id: string) => /^[a-zA-Z0-9_-]{16,96}$/.test(id); @@ -32,6 +34,17 @@ export function createStorageRemoval(db: Database, options: { retainedRoot?: string; }) { const brain = realpathSync(options.brainPath), journal = options.journal; + // Core is an optional peer. Load its installed public entry only for removal, + // preserving server imports and unrelated features when it is absent. + let core: typeof import("@schlessera/brain/storage") | undefined; + function attachments() { + if (core) return core; + const loaded = createRequire(import.meta.url)(coreReader(brain, "storage")) as typeof import("@schlessera/brain/storage"); + for (const name of ["inspectStorageAttachment", "removeStorageAttachment", "restoreStorageAttachment", "storageAttachmentPath", "storageAttachmentVacant", "StorageAttachmentRefusal"] as const) { + if (typeof loaded[name] !== "function") throw new Error("Attachment recovery entry unavailable."); + } + return core = loaded; + } const record = (root: string, target: string, outcome: "success" | "no-space", error?: unknown) => { const code = (error as { code?: string; reason?: string } | undefined)?.code; const quota = code === "EDQUOT" || (error as { reason?: string } | undefined)?.reason === "journal_capacity"; @@ -46,11 +59,12 @@ export function createStorageRemoval(db: Database, options: { function capability(principal: string): StorageRemovalCapability { authorize(principal); reconcile(); const status = options.boundary.status(); const latest = db.query("SELECT * FROM turn_journal_storage_operations WHERE brain=? AND principal_id=? ORDER BY requested_at DESC,rowid DESC LIMIT 1").get(brain, principal) as Row | null; - const available = !closed && process.platform === "linux" && options.boundary.isolated() && status.available; + let available = !closed && process.platform === "linux" && options.boundary.isolated() && status.available; + if (available) { try { attachments(); } catch { available = false; } } return { available, reason: available ? null : "qualified_native_recovery_unavailable", limits: STORAGE_REMOVAL_LIMITS, latest: latest ? recover(latest) : null }; } function protectedReason(path: string): string | null { - if (!storageAttachmentPath(path)) return "unsupported_or_protected_attachment"; + if (!attachments().storageAttachmentPath(path)) return "unsupported_or_protected_attachment"; const status = options.boundary.status(); if (!status.available || status.profileFallback || !status.protectedPaths) return "protection_or_ownership_unknown"; if ([...options.protectedPaths, ...status.protectedPaths.map(value => value.path)].some(root => prefix(path, root))) return "protected_owner_path"; @@ -98,6 +112,7 @@ export function createStorageRemoval(db: Database, options: { const prior = select(id); if (prior) { if (prior.principal_id !== principal || prior.selection !== selection || prior.action !== action || prior.source_id !== sourceId) throw new StorageUnavailable(409, "Operation identity belongs to another selection."); return recover(prior); } if (!capability(principal).available) throw new StorageUnavailable(503, "This host cannot remove attachments yet."); + const { storageAttachmentPath, inspectStorageAttachment, storageAttachmentVacant, StorageAttachmentRefusal } = attachments(); if (reviews.size || active.size) throw new StorageUnavailable(409, "Another attachment review or operation is active."); const controller = new AbortController(), timeout = setTimeout(() => controller.abort(), STORAGE_REMOVAL_LIMITS.reviewMilliseconds); reviews.add(controller); let finish!: () => void; const done = new Promise(resolve => { finish = resolve; }); pendingReviews.add(done); @@ -149,7 +164,9 @@ export function createStorageRemoval(db: Database, options: { const revoked = setInterval(() => { try { check(); } catch { controller.abort(); } }, 100); let journalStarted = false, effects = false, unresolved = false; try { - check(); result.before = storageObservation(await options.storage.measure(row.principal_id, controller.signal), ["content", "git", "journal", "database"]); + check(); + const { inspectStorageAttachment, removeStorageAttachment, restoreStorageAttachment, storageAttachmentVacant, StorageAttachmentRefusal } = attachments(); + result.before = storageObservation(await options.storage.measure(row.principal_id, controller.signal), ["content", "git", "journal", "database"]); await exclusive(controller.signal, `storage-remove:${row.id}`, async () => { check(); if (Date.now() >= result.review.expiresAt) throw Error("review_expired"); const eligible = result.review.files.filter(file => file.eligible).map(file => file.path); diff --git a/scripts/check-ui-server-query-package.ts b/scripts/check-ui-server-query-package.ts index eed9b4c687..36ccc7e895 100644 --- a/scripts/check-ui-server-query-package.ts +++ b/scripts/check-ui-server-query-package.ts @@ -7,6 +7,8 @@ * (identity, range and operation checks) and returns core's terms; * - the packed graph routes, both the shipped TypeScript source and the * compiled default JavaScript, serve populated results through the peer; + * - Storage removal reports unavailable without core, and available with it + * on an otherwise qualified Linux host, through both shipped entry paths; * - no published ui-server declaration names `@schlessera/brain`, so a * consumer that does not install the optional peer can still typecheck it. */ @@ -65,10 +67,41 @@ for (const path of ${JSON.stringify([join(server, "src/brain/governance-policy.t if (invalid.source !== "last_valid" || invalid.hash !== valid.hash || !invalid.errors.length || invalid.policy.profiles.interactive !== "strict") throw new Error("POLICY_OPTIONAL_CORE_LAST_VALID"); } console.log("packed ui-server: source/default policy reads and last-valid fallback work without core"); +`); + const storageProbe = join(probeDir, "storage-peer.ts"); + await Bun.write(storageProbe, ` +import { createRequire } from "node:module"; +let peerPresent = false; +try { createRequire(import.meta.url).resolve("@schlessera/brain/package.json"); peerPresent = true; } catch {} +const expectedPeer = process.argv[2] === "present"; +if (peerPresent !== expectedPeer) throw new Error("STORAGE_OPTIONAL_CORE_FIXTURE"); +const unused = new Proxy({}, { get() { throw new Error("STORAGE_CAPABILITY_MUST_NOT_EXECUTE"); } }); +for (const variant of ["src", "dist"]) { + const extension = variant === "src" ? ".ts" : ".js"; + const root = ${JSON.stringify(server)} + "/" + variant; + const { createUiDb } = await import(root + "/db/client" + extension); + const { createStorageRemoval } = await import(root + "/storage/removal" + extension); + const db = createUiDb(":memory:"); + const removal = createStorageRemoval(db, { + brainPath: ${JSON.stringify(brainPath)}, authorized: () => true, protectedPaths: [], + boundary: { isolated: () => true, status: () => ({ available: true, protectedPaths: [] }), + run() { throw new Error("STORAGE_CAPABILITY_MUST_NOT_EXECUTE"); } }, + journal: unused, gate: unused, lock: unused, storage: unused, + }); + try { + const capability = removal.capability("odysseus"); + const expectedAvailable = expectedPeer && process.platform === "linux"; + if (capability.available !== expectedAvailable || capability.reason !== (expectedAvailable ? null : "qualified_native_recovery_unavailable")) { + throw new Error((expectedPeer ? "STORAGE_OPTIONAL_CORE_AVAILABLE" : "STORAGE_OPTIONAL_CORE_UNAVAILABLE") + ": " + variant + " " + JSON.stringify(capability)); + } + } finally { await removal.close(); db.close(); } +} +console.log("packed ui-server: source/default Storage removal capability with core " + (expectedPeer ? "present" : "absent")); `); renameSync(corePeer, heldCorePeer); - try { await run([policyProbe]); } + try { await run([policyProbe]); await run([storageProbe, "absent"]); } finally { renameSync(heldCorePeer, corePeer); } + await run([storageProbe, "present"]); // 1. Declarations: no reference to the optional peer, value or type. // A module reference — import/export from, import(), require() or a types diff --git a/scripts/native-proof/owned-prune-variant.ts b/scripts/native-proof/owned-prune-variant.ts index 0020125dea..1e1aeddfa2 100644 --- a/scripts/native-proof/owned-prune-variant.ts +++ b/scripts/native-proof/owned-prune-variant.ts @@ -10,11 +10,11 @@ export function ownedPruneVariant(cronSource: string, boundarySource: string) { const observer = (key: string, condition = "") => `const fixtureStarted = performance.now(); const fixtureStage = (stage: string) => { try { ${condition}console.log(JSON.stringify({ ${key}: { stage, elapsedMs: performance.now() - fixtureStarted } })); } catch { /* Observation cannot replace the operation. */ } };`; let cron = replace("cron", cronSource, " const { brain, log } = options;", " const { brain, log } = options;\n " + observer("fixtureOwnedPrune")); - cron = replace("cron", cron, " if (inFlight) return inFlight;", ' if (inFlight) { fixtureStage("tick:join-existing"); return inFlight; }'); + cron = replace("cron", cron, " if (tickFlight) return tickFlight;", ' if (tickFlight) { fixtureStage("tick:join-existing"); return tickFlight; }'); cron = replace("cron", cron, " const controller = new AbortController();", ' fixtureStage("tick:start-new");\n const controller = new AbortController();'); - cron = replace("cron", cron, " if (options.pins) await options.pins.refresh(controller.signal);", ' if (options.pins) { fixtureStage("pins:start"); await options.pins.refresh(controller.signal); fixtureStage("pins:done"); }'); - cron = replace("cron", cron, ' return brain.scratchPrune({ signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) });', ' fixtureStage("cli:start");\n const result = await brain.scratchPrune({ signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) });\n fixtureStage("cli:done"); return result;'); - cron = replace("cron", cron, " .catch((error: unknown) => {", ' .catch((error: unknown) => {\n fixtureStage("pass:error");'); + cron = replace("cron", cron, " try { await options.pins.refresh(controller.signal); }", ' try { fixtureStage("pins:start"); await options.pins.refresh(controller.signal); fixtureStage("pins:done"); }'); + cron = replace("cron", cron, ' return brain.scratchPrune({ ...(request ?? {}), signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) });', ' fixtureStage("cli:start");\n const result = await brain.scratchPrune({ ...(request ?? {}), signal: controller.signal, ...(options.pins ? { pinsFile: options.pins.file } : {}) });\n fixtureStage("cli:done"); return result;'); + cron = replace("cron", cron, " tickFlight = run().then(() => undefined).catch((error: unknown) => {", ' tickFlight = run().then(() => undefined).catch((error: unknown) => {\n fixtureStage("pass:error");'); cron = replace("cron", cron, " .finally(() => {", ' .finally(() => {\n fixtureStage("pass:settled");'); let boundary = replace("boundary", boundarySource, " async run(execution) {", ' async run(execution) {\n ' + observer("fixtureOwnedPruneBoundary", 'if (execution.purpose !== "brain scratch prune") return; ') + '\n fixtureStage("entry");'); const runStart = boundary.indexOf(" async run(execution) {"), runEnd = boundary.indexOf(" async launch(execution) {", runStart); @@ -24,11 +24,12 @@ export function ownedPruneVariant(cronSource: string, boundarySource: string) { run = replace("boundary", run, " const run = async () => {", ' const run = async () => {\n fixtureStage("output-lock:admitted");\n fixtureStage("view:start");'); const mark = (before: string, start: string, done: string) => { run = replace("boundary", run, before, ` fixtureStage("${start}");\n${before}\n fixtureStage("${done}");`); }; run = replace("boundary", run, " const view = await prepareInput(execution.signal);", ' const view = await prepareInput(execution.signal);\n fixtureStage("view:done");'); - mark(" outputs = await prepareMaintenanceOutput(brain, { startupView: view, folderPaths: hold?.folderPaths, signal: execution.signal });", "output:start", "output:done"); + mark(" outputs = await prepareMaintenanceOutput(brain, { startupView: view, folderPaths: hold?.folderPaths, signal: execution.signal, areas: execution.outputAreas });", "output:start", "output:done"); mark(" const git = await prepareMaintenanceGit(launch, outputs, execution.signal, options.log);", "git:start", "git:done"); mark(` const result = await runMaintenanceWorker({ ...launch, startupView: outputs.view, output: outputs.output, ...(git ? { git } : {}), stdin: execution.stdin, - ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}) });`, "worker:start", "worker:done"); - run = replace("boundary", run, " if (!result.killed) {", ' if (!result.killed) {\n fixtureStage("publish:start");'); + ...(execution.signal ? { signal: execution.signal } : {}), ...(execution.timeoutMs ? { timeoutMs: execution.timeoutMs } : {}), ...(execution.outputBytes ? { outputBytes: execution.outputBytes } : {}) }, + () => { execution.assertAuthorized?.(); execution.onExecute?.(); });`, "worker:start", "worker:done"); + run = replace("boundary", run, " if (!result.killed && execution.publish !== false) {", ' if (!result.killed && execution.publish !== false) {\n fixtureStage("publish:start");'); run = replace("boundary", run, " else await outputs.publish();", ' else await outputs.publish();\n fixtureStage("publish:done");'); run = replace("boundary", run, " } finally { try { await outputs?.cleanup(); } finally { await view.cleanup(); } }", ' } finally { try { fixtureStage("output-cleanup:start"); await outputs?.cleanup(); fixtureStage("output-cleanup:done"); } finally { fixtureStage("view-cleanup:start"); await view.cleanup(); fixtureStage("view-cleanup:done"); } }'); run = replace("boundary", run, " return outputLock.withLock(brain, () => execution.lock", ' fixtureStage("output-lock:wait");\n return outputLock.withLock(brain, () => execution.lock');