diff --git a/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs b/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs new file mode 100644 index 000000000..114e96d36 --- /dev/null +++ b/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs @@ -0,0 +1,129 @@ +import path from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import { expect, test } from '../fixtures/electron.mjs' + +const oldOperation = '11111111-1111-4111-8111-111111111111' +const deletedOperation = '22222222-2222-4222-8222-222222222222' +const unrelatedOperation = '33333333-3333-4333-8333-333333333333' + +function seedRecovery(databasePath) { + const database = new DatabaseSync(databasePath) + const now = '2026-10-15T00:00:00.000Z' + try { + database.exec('PRAGMA foreign_keys = ON') + for (const [id, title] of [['source', '此前的整理任务'], ['blocked', '新的写入任务'], ['unrelated', '其他任务']]) { + database.prepare('INSERT INTO conversations (id, title, active_branch_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?)').run(id, title, `branch-${id}`, now, now) + database.prepare('INSERT INTO conversation_branches (id, conversation_id, created_at) VALUES (?, ?, ?)').run(`branch-${id}`, id, now) + database.prepare(`INSERT INTO runs (id, conversation_id, branch_id, triggering_message_id, provider, model, purpose, status, started_at, completed_at) + VALUES (?, ?, ?, ?, 'fixture', 'fixture', 'chat', 'failed', ?, ?)`).run(`run-${id}`, id, `branch-${id}`, `input-${id}`, now, now) + } + for (const [id, runId, resource] of [ + [oldOperation, 'run-source', '/example/shared/report.md'], + [deletedOperation, 'deleted-run', '/example/shared/notes.md'], + [unrelatedOperation, 'run-unrelated', '/example/shared/report.md'], + ]) { + database.prepare(`INSERT INTO run_mutation_leases (invocation_id, run_id, segment_id, claims_json, status, reason, created_at, updated_at, lifetime) + VALUES (?, ?, ?, ?, 'hazard', 'MUTATION_INTERRUPTED', ?, ?, 'external')`).run(id, runId, `segment-${id}`, JSON.stringify([{ kind: 'file', canonicalPath: resource }]), now, now) + database.prepare('INSERT INTO run_mutation_processes (invocation_id, process_id, settlement, updated_at) VALUES (?, ?, \'unknown\', ?)') + .run(id, JSON.stringify({ kind: 'legacy-process', pid: 123456, job: 'not-user-facing', checkpoint: 'not-user-facing' }), now) + if (id !== unrelatedOperation) { + database.prepare(`INSERT INTO run_mutation_evidence (invocation_id, kind, outcome, reason, actor, related_run_id, created_at) + VALUES (?, 'observed', 'unknown', 'WRITE_BLOCKED', 'system', 'run-blocked', ?)`).run(id, now) + } + } + } + finally { database.close() } +} + +function readRecovery(databasePath) { + const database = new DatabaseSync(databasePath, { readOnly: true }) + try { + return { + leases: database.prepare('SELECT invocation_id, status, reason FROM run_mutation_leases ORDER BY invocation_id').all(), + decisions: database.prepare('SELECT invocation_id, kind, outcome, actor FROM run_mutation_evidence WHERE kind = \'waived\' ORDER BY sequence').all(), + runs: database.prepare('SELECT id FROM runs ORDER BY id').all(), + } + } + finally { database.close() } +} + +test('task recovery discovers older blockers and only waives the explicitly confirmed set', async ({ buddy }, testInfo) => { + test.setTimeout(120_000) + const instance = await buddy.createInstance('mutation-recovery') + const databasePath = path.join(instance.home, 'buddy/buddy.sqlite3') + await instance.launch() + await instance.stop() + seedRecovery(databasePath) + let { app, page, diagnostics } = await instance.launch() + await page.evaluate(async () => { + const api = window.lexoraDesktop.localChat + const { defaultBuddyId } = await api.buddies.list() + for (const id of ['source', 'blocked', 'unrelated']) { + await api.conversations.addMember(id, defaultBuddyId) + await api.conversations.setBuddy(id, defaultBuddyId) + } + }) + await page.locator('[data-task-id="blocked"] .desktop-task-sidebar__task').click() + const card = () => page.locator('.desktop-change-recovery:visible') + await expect(card()).toContainText('此前的整理任务') + await expect(card()).toContainText('已删除的任务') + await expect(card()).toContainText('/example/shared/report.md') + await expect(card()).not.toContainText('其他任务') + await expect(card()).not.toContainText(/123456|not-user-facing|PID|checkpoint/i) + await expect(card().getByRole('button', { name: /停止|结束/ })).toHaveCount(0) + await expect(card().getByRole('button', { name: '接受风险并恢复修改', exact: true })).toHaveCount(0) + + const initial = await page.evaluate(() => window.lexoraDesktop.localChat.changes.recovery.list({ conversationId: 'blocked' })) + expect(initial.map(item => item.invocationId).sort()).toEqual([oldOperation, deletedOperation]) + expect(initial.every(item => item.status === 'unknown' && item.revision)).toBe(true) + expect(readRecovery(databasePath).decisions).toEqual([]) + await card().getByRole('button', { name: '重新检查', exact: true }).click() + await expect(card().getByRole('button', { name: '重新检查', exact: true })).toBeEnabled() + await expect(card()).toContainText('暂时无法确认此前的操作是否已停止') + expect(readRecovery(databasePath).decisions).toEqual([]) + + for (const theme of ['light', 'dark']) { + await page.evaluate(theme => window.lexoraDesktop.themes.request({ action: 'preference', preference: { id: `lexora.themes.classic-${theme}` } }), theme) + await expect(page.locator('html')).toHaveAttribute('data-buddy-theme', theme) + await page.screenshot({ path: path.join(instance.artifactDirectory, `recovery-${theme}.png`), animations: 'disabled' }) + } + await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0].setSize(980, 640)) + await expect(card().getByRole('button', { name: '重新检查', exact: true })).toBeInViewport() + expect(await card().evaluate(element => element.scrollWidth <= element.clientWidth + 1)).toBe(true) + await page.screenshot({ path: path.join(instance.artifactDirectory, 'recovery-narrow.png'), animations: 'disabled' }) + + async function review() { + await card().getByRole('button', { name: '更多选项', exact: true }).click() + await page.locator('.n-dropdown-menu:visible').getByText('接受风险并恢复修改', { exact: true }).click() + } + await review() + const confirmation = page.locator('.n-dialog').filter({ hasText: '接受风险并恢复修改?' }) + await expect(confirmation).toContainText('仅对以下 2 项恢复修改') + await expect(confirmation).toContainText('不会重新运行旧命令') + await page.screenshot({ path: path.join(instance.artifactDirectory, 'recovery-confirmation.png'), animations: 'disabled' }) + await confirmation.getByRole('button', { name: '取消', exact: true }).click() + expect(readRecovery(databasePath).decisions).toEqual([]) + await review() + await confirmation.getByRole('button', { name: '接受风险并恢复修改', exact: true }).click() + await expect(confirmation).toBeHidden() + await expect(card()).toHaveCount(0) + await expect(page.getByText('已按你的选择接受风险并恢复所选位置的修改;此前操作仍未确认结束,旧命令不会自动重跑。', { exact: true })).toBeVisible() + await page.screenshot({ path: path.join(instance.artifactDirectory, 'recovery-waived.png'), animations: 'disabled' }) + const resolved = readRecovery(databasePath) + expect(resolved.leases).toEqual([ + { invocation_id: oldOperation, status: 'released', reason: 'USER_ACCEPTED_RISK' }, + { invocation_id: deletedOperation, status: 'released', reason: 'USER_ACCEPTED_RISK' }, + { invocation_id: unrelatedOperation, status: 'hazard', reason: 'MUTATION_INTERRUPTED' }, + ]) + expect(resolved.decisions.filter(item => item.outcome === 'applied').map(item => [item.invocation_id, item.kind, item.actor]).sort()) + .toEqual([[oldOperation, 'waived', 'user'], [deletedOperation, 'waived', 'user']]) + expect(resolved.runs.map(run => run.id)).toEqual(['run-blocked', 'run-source', 'run-unrelated']) + expect(diagnostics.console.filter(item => item.type === 'pageerror')).toEqual([]) + await instance.stop() + ;({ app, page, diagnostics } = await instance.launch()) + expect(await page.evaluate(() => window.lexoraDesktop.localChat.changes.recovery.list({ conversationId: 'blocked' }))).toEqual([]) + expect((await page.evaluate(() => window.lexoraDesktop.localChat.changes.recovery.list({ conversationId: 'unrelated' }))).map(item => item.invocationId)).toEqual([unrelatedOperation]) + expect(readRecovery(databasePath).decisions).toEqual(resolved.decisions) + expect(diagnostics.console.filter(item => item.type === 'pageerror')).toEqual([]) + await testInfo.attach('recovery-audit', { body: JSON.stringify(resolved, null, 2), contentType: 'application/json' }) +}) diff --git a/apps/buddy/electron/main/app/DesktopRuntimeHost.ts b/apps/buddy/electron/main/app/DesktopRuntimeHost.ts index 15fe0896f..412e68cae 100644 --- a/apps/buddy/electron/main/app/DesktopRuntimeHost.ts +++ b/apps/buddy/electron/main/app/DesktopRuntimeHost.ts @@ -130,6 +130,7 @@ export class DesktopRuntimeHost { ? join(app.isPackaged ? process.resourcesPath : app.getAppPath(), app.isPackaged ? shellSandbox.resource.to : `${shellSandbox.resource.from}/${currentTarget.id}`) : undefined, windowsSandbox: this.#sandboxExecutable(), + receiptDirectory: join(this.#environment.paths.buddyHome, 'process-receipts'), windowsShell: this.#windowsPowerShell, processControl: resolveBuddyProcessControl({ appPath: app.getAppPath(), isPackaged: app.isPackaged, resourcesPath: process.resourcesPath }), } diff --git a/apps/buddy/electron/main/local-chat/activity.ts b/apps/buddy/electron/main/local-chat/activity.ts index 5739b72aa..de7a59c78 100644 --- a/apps/buddy/electron/main/local-chat/activity.ts +++ b/apps/buddy/electron/main/local-chat/activity.ts @@ -1,6 +1,6 @@ import type { LocalChatIpcContext } from './registrar' import { artifactsRequestSchemas, artifactsRpc } from '../../../shared/artifacts/artifactApi' -import { changeOverviewRequestSchema, changesRequestSchemas, changesRpc } from '../../../shared/changes/changeApi' +import { changeOverviewRequestSchema, changesRecoveryRpc, changesRequestSchemas, changesRpc } from '../../../shared/changes/changeApi' import { notificationsRequestSchemas, notificationsRpc } from '../../../shared/notifications/notificationApi' import { approvalsRequestSchemas, approvalsRpc } from '../../../shared/permissions/approvalApi' import { conversationStatusRequestSchema, runsStatusRpc } from '../../../shared/runs/conversationStatusApi' @@ -12,6 +12,9 @@ import { LOCAL_CHAT_IPC_CHANNELS } from '../../shared/localChatApi' export function registerActivityIpc(context: LocalChatIpcContext): void { const { handle, request } = context + handle(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryList, (_event, input) => request(changesRecoveryRpc.list, changesRecoveryRpc.list.input.parse(input))) + handle(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryRecheck, (_event, input) => request(changesRecoveryRpc.recheck, changesRecoveryRpc.recheck.input.parse(input))) + handle(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryResolve, (_event, input) => request(changesRecoveryRpc.resolve, changesRecoveryRpc.resolve.input.parse(input))) handle(LOCAL_CHAT_IPC_CHANNELS.changesOverview, (_event, input) => request(changesRpc.overview, changeOverviewRequestSchema.parse(input))) handle(LOCAL_CHAT_IPC_CHANNELS.artifactsReadText, (_event, input) => request(artifactsRpc.readText, artifactsRequestSchemas.artifactText.parse(input))) diff --git a/apps/buddy/electron/main/sandbox/registerSandboxHostRpc.ts b/apps/buddy/electron/main/sandbox/registerSandboxHostRpc.ts index 34ee1755e..8befd5713 100644 --- a/apps/buddy/electron/main/sandbox/registerSandboxHostRpc.ts +++ b/apps/buddy/electron/main/sandbox/registerSandboxHostRpc.ts @@ -12,13 +12,14 @@ import { resolveSensitivePathRoots } from '../../../service/src/permissions/sens import sandboxProcessPath from '../../../service/src/sandbox/sandboxProcess?modulePath' import { Emitter } from '../../../shared/events/Emitter' import { sandboxLifecycleNotificationSchema } from '../../../shared/permissions/sandboxLifecycle' -import { SANDBOX_RPC_TIMEOUT_MS, sandboxCancelSchema, sandboxCommandSchema, sandboxNetworkRequestSchema, sandboxOutputSchema, sandboxPreparationSchema } from '../../../shared/permissions/shellSandbox' +import { SANDBOX_RPC_TIMEOUT_MS, sandboxCancelSchema, sandboxCommandSchema, sandboxNetworkRequestSchema, sandboxOutputSchema, sandboxPreparationSchema, sandboxSettlementAckSchema, sandboxSettlementSchema } from '../../../shared/permissions/shellSandbox' import { isLinux, isWindows, OPERATING_SYSTEM, SHELL_SANDBOX_BACKEND } from '../../../shared/platform/identifiers' import { BuddyServicePeer } from '../runtime/BuddyServicePeer' import { WindowsSandboxPreparation } from './WindowsSandboxPreparation' export interface SandboxHostOptions { buddyHome: string + receiptDirectory?: string protectedRoots?: string[] protectionScope?: string proxyUrl?: string @@ -34,6 +35,7 @@ export function registerSandboxHostRpc(peer: RuntimeRpcPeerContract, options: Sa const publish = (event: Parameters[0]) => changes.fire(Object.freeze(event)) const preparation = isWindows(process.platform) ? new WindowsSandboxPreparation({ ...options, protectedRoots: options.protectedRoots ?? [dirname(options.buddyHome)] }) : undefined const active = new Map() + const accepted = new Set() let disposed = false const disposers = [ peer.onRequest('host.sandbox.prepare', async (params) => { @@ -54,10 +56,11 @@ export function registerSandboxHostRpc(peer: RuntimeRpcPeerContract, options: Sa const input = sandboxCommandSchema.parse(params) if (!Object.values(OPERATING_SYSTEM).includes(process.platform)) return { ok: false, code: 'SANDBOX_UNAVAILABLE', notStarted: true } - if (active.has(input.requestId)) + if (accepted.has(input.requestId)) return { ok: false, code: 'SANDBOX_BUSY' } if (disposed || active.size >= 8) return { ok: false, code: 'SANDBOX_BUSY', notStarted: true } + accepted.add(input.requestId) preparation?.start() const controller = new AbortController() active.set(input.requestId, controller) @@ -143,6 +146,17 @@ export function registerSandboxHostRpc(peer: RuntimeRpcPeerContract, options: Sa if (output.requestId === input.requestId) peer.notify('host.sandbox.output', output) }) + processPeer.onRequest('sandbox.settlement', async (params) => { + const record = sandboxSettlementSchema.parse(params) + if (disposed || record.requestId !== input.requestId || record.executionId !== input.executionId || record.dispatchId !== input.dispatchId + || (record.launchState === 'authorized' && controller.signal.aborted)) { + throw new Error('SANDBOX_CANCELLED') + } + const result = sandboxSettlementAckSchema.parse(await peer.request('sandbox.settlement', record, 60_000)) + if (record.launchState === 'authorized' && (disposed || controller.signal.aborted)) + throw new Error('SANDBOX_CANCELLED') + return result + }) processPeer.onRequest('sandbox.network', async (params) => { const target = sandboxNetworkRequestSchema.parse(params) if (disposed || controller.signal.aborted || target.requestId !== input.requestId) @@ -156,6 +170,7 @@ export function registerSandboxHostRpc(peer: RuntimeRpcPeerContract, options: Sa sensitiveRoots: resolveSensitivePathRoots(), path: process.env.PATH ?? '', privateRoot: directory, + receiptDirectory: options.receiptDirectory, protectedRoots: options.protectedRoots ?? [dirname(options.buddyHome)], searchDirectory: options.searchDirectory, backend: windowsSandbox diff --git a/apps/buddy/electron/preload/local-chat/activity.ts b/apps/buddy/electron/preload/local-chat/activity.ts index dc709938a..40357daa8 100644 --- a/apps/buddy/electron/preload/local-chat/activity.ts +++ b/apps/buddy/electron/preload/local-chat/activity.ts @@ -24,6 +24,18 @@ export function createActivityApi(): Pick({ + list: input => ipcRenderer.invoke(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryList, { conversationId: input.conversationId }), + recheck: input => ipcRenderer.invoke(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryRecheck, { + conversationId: input.conversationId, + invocationIds: [...input.invocationIds], + }), + resolve: input => ipcRenderer.invoke(LOCAL_CHAT_IPC_CHANNELS.changesRecoveryResolve, { + conversationId: input.conversationId, + items: input.items.map(item => ({ invocationId: item.invocationId, revision: item.revision })), + decision: input.decision, + }), + }), onChanged: listener => subscribe(LOCAL_CHAT_IPC_CHANNELS.changesChanged, listener), overview: input => ipcRenderer.invoke(LOCAL_CHAT_IPC_CHANNELS.changesOverview, { ...input }), get: changeSetId => ipcRenderer.invoke(LOCAL_CHAT_IPC_CHANNELS.changesGet, { changeSetId }), diff --git a/apps/buddy/electron/shared/localChatApi.ts b/apps/buddy/electron/shared/localChatApi.ts index 3151d8436..125bb4e16 100644 --- a/apps/buddy/electron/shared/localChatApi.ts +++ b/apps/buddy/electron/shared/localChatApi.ts @@ -2,7 +2,7 @@ import type { LocalConversationStatus } from '@buddy-shared/runs/conversationSta import type { ArtifactChangeNotice, LocalArtifactText } from '../../shared/artifacts/artifactApi' import type { LocalAutomation, LocalAutomationCreateRequest, LocalAutomationListRequest, LocalAutomationMutationRequest, LocalAutomationOccurrenceListRequest, LocalAutomationOccurrencePage, LocalAutomationPage, LocalAutomationPreviewRequest, LocalAutomationPreviewResult, LocalAutomationRunNowResult, LocalAutomationUpdateRequest } from '../../shared/automation/automationApi' import type { BuddyCatalog, BuddyCreate, BuddyDefaultUpdate, BuddyMutation, BuddyToolOption, BuddyUpdate, LocalBuddy } from '../../shared/buddies/buddyApi' -import type { ChangeOverviewRequest, ChangeSetChangeNotice, LocalChangeOverview, LocalChangeSetDetail } from '../../shared/changes/changeApi' +import type { ChangeOverviewRequest, ChangeRecoveryListRequest, ChangeRecoveryRecheckRequest, ChangeRecoveryResolveRequest, ChangeSetChangeNotice, LocalChangeOverview, LocalChangeRecoveryItem, LocalChangeSetDetail } from '../../shared/changes/changeApi' import type { ConnectorChangeNotice, LocalConnector, LocalConnectorConfig, LocalConnectorCredential, LocalConnectorCredentialMutation } from '../../shared/connectors/connectorApi' import type { ConnectorRuntimeState, ConnectorToolSummary } from '../../shared/connectors/connectorState' import type { LocalChatCommandRequest, LocalStartTurnRequest, LocalTurnStart } from '../../shared/conversation/chatApi' @@ -75,6 +75,9 @@ export const LOCAL_CHAT_IPC_CHANNELS = { taskMarksSetRead: 'lexora:buddy:task-marks:set-read', taskMarksClear: 'lexora:buddy:task-marks:clear', changesOverview: 'lexora:buddy:changes:overview', + changesRecoveryList: 'lexora:buddy:changes:recovery:list', + changesRecoveryRecheck: 'lexora:buddy:changes:recovery:recheck', + changesRecoveryResolve: 'lexora:buddy:changes:recovery:resolve', spaceFilesMutate: 'lexora:buddy:space-files:mutate', spaceFilesHistory: 'lexora:buddy:space-files:history', spaceFilesQuery: 'lexora:buddy:space-files:query', @@ -493,6 +496,11 @@ export interface LocalChatApi { }) => Promise } changes: { + recovery: { + list: (input: ChangeRecoveryListRequest) => Promise + recheck: (input: ChangeRecoveryRecheckRequest) => Promise + resolve: (input: ChangeRecoveryResolveRequest) => Promise + } onChanged: (listener: (event: ChangeSetChangeNotice) => void) => () => void overview: (input: ChangeOverviewRequest) => Promise get: (changeSetId: string) => Promise diff --git a/apps/buddy/native/host/Cargo.toml b/apps/buddy/native/host/Cargo.toml index 55b745bd0..8aa92cca0 100644 --- a/apps/buddy/native/host/Cargo.toml +++ b/apps/buddy/native/host/Cargo.toml @@ -66,6 +66,10 @@ path = "__tests__/process_control_linux.rs" name = "supervised-shell-linux" path = "__tests__/supervised_shell_linux.rs" +[[test]] +name = "supervised-receipts-linux" +path = "__tests__/supervised_receipts_linux.rs" + [[test]] name = "sandbox-ipc-linux" path = "__tests__/sandbox_ipc_linux.rs" @@ -157,6 +161,8 @@ windows-sys = { workspace = true, features = [ "Win32_System_Rpc", "Win32_UI_Shell", "Win32_System_Diagnostics_ToolHelp", + "Win32_System_Diagnostics_Debug", + "Win32_System_SystemInformation", "Win32_System_Threading", "Win32_System_Time", "Win32_UI_Input_KeyboardAndMouse", diff --git a/apps/buddy/native/host/__tests__/process_recovery.rs b/apps/buddy/native/host/__tests__/process_recovery.rs index ff7434ca0..5db3387d8 100644 --- a/apps/buddy/native/host/__tests__/process_recovery.rs +++ b/apps/buddy/native/host/__tests__/process_recovery.rs @@ -1,6 +1,7 @@ use super::*; use serde_json::{Value, json}; +#[cfg(target_os = "linux")] fn boot() -> String { backend::boot_id().unwrap() } @@ -24,6 +25,7 @@ fn prior_checkpoint() -> Value { json!({"kind":"recovery-boot","version":1,"platform":backend::PLATFORM,"bootId":uuid::Uuid::new_v4().to_string()}) } +#[cfg(target_os = "linux")] #[test] fn a_complete_legacy_handshake_can_recover_after_a_checkpoint_boot_change() { assert_eq!( @@ -93,13 +95,14 @@ fn malformed_boot_ids_and_unknown_identity_versions_do_not_release() { "identity_invalid" ); let mut invalid_version = identity(); - invalid_version["version"] = json!(3); + invalid_version["version"] = json!(4); assert_eq!( inspect(invalid_version, Some(prior_checkpoint()))["reason"], "identity_version_unsupported" ); } +#[cfg(target_os = "linux")] #[test] fn checkpoint_reports_a_kernel_boot_id_not_a_wall_clock() { let mut output = Vec::new(); @@ -126,6 +129,7 @@ fn input_is_bounded_and_never_accepts_a_process_action() { } } +#[cfg(target_os = "linux")] #[test] fn same_boot_checkpoint_does_not_turn_pid_disappearance_into_release() { let checkpoint = diff --git a/apps/buddy/native/host/__tests__/process_recovery_windows.rs b/apps/buddy/native/host/__tests__/process_recovery_windows.rs index ab851bdf3..3aec7e749 100644 --- a/apps/buddy/native/host/__tests__/process_recovery_windows.rs +++ b/apps/buddy/native/host/__tests__/process_recovery_windows.rs @@ -35,13 +35,13 @@ fn windows_live_supervisor_keeps_an_empty_job_active() { } #[test] -fn windows_reused_pid_and_an_empty_held_job_prove_release_without_killing_the_reused_pid() { +fn windows_reused_pid_and_an_empty_held_job_cannot_prove_a_sealed_launch_domain() { let job = Job::host().unwrap(); let mut identity = job_identity(&job); identity["startTime"] = json!("0"); assert_eq!( inspect(identity, None), - json!({"protocol":1,"status":"released","reason":"job_empty"}) + json!({"protocol":1,"status":"unknown","reason":"job_boundary_unverified"}) ); } @@ -157,7 +157,7 @@ fn windows_reverification_observes_a_remaining_child_without_terminating_it() { loop { let result = inspect(identity.clone(), None); if result["reason"] != "job_active" || Instant::now() >= deadline { - assert_eq!(result["reason"], "job_empty"); + assert_eq!(result["reason"], "job_boundary_unverified"); break; } std::thread::sleep(Duration::from_millis(10)); @@ -188,6 +188,6 @@ fn windows_non_inheritable_observer_can_hold_the_job_after_supervisor_release() // SAFETY: DuplicateHandle returned a valid unique owned handle. let observer = unsafe { OwnedHandle::from_raw_handle(duplicate) }; drop(job); - assert_eq!(inspect(identity, None)["reason"], "job_empty"); + assert_eq!(inspect(identity, None)["reason"], "job_boundary_unverified"); drop(observer); } diff --git a/apps/buddy/native/host/__tests__/shell_sandbox.windows.mjs b/apps/buddy/native/host/__tests__/shell_sandbox.windows.mjs index 463e974de..40b47a24e 100644 --- a/apps/buddy/native/host/__tests__/shell_sandbox.windows.mjs +++ b/apps/buddy/native/host/__tests__/shell_sandbox.windows.mjs @@ -8,6 +8,7 @@ import { createServer } from 'node:net' import { homedir, networkInterfaces, tmpdir } from 'node:os' import { dirname, isAbsolute, join } from 'node:path' import process from 'node:process' +import { createInterface } from 'node:readline' import * as nativeTest from 'node:test' import { promisify } from 'node:util' @@ -29,12 +30,13 @@ async function listener(host = '127.0.0.1') { return { server, port: server.address().port } } -async function execute({ command, workspace, grants, port, onReady, onPreparing, measure = false, prepareOnly = false, denyReadScope = dirname(workspace), synchronizeDenials = false }) { +async function execute({ command, workspace, grants, port, onReady, onPreparing, measure = false, prepareOnly = false, authorizeLaunch = true, denyReadScope = dirname(workspace), synchronizeDenials = false }) { const privateRoot = await mkdtemp(join(dirname(workspace), 'private-')) const home = join(privateRoot, 'home') const temporary = join(privateRoot, 'tmp') await Promise.all([home, temporary].map(path => mkdir(path))) const shell = process.env.BUDDY_SANDBOX_TEST_SHELL ?? join(process.env.ProgramFiles, 'PowerShell', '7', 'pwsh.exe') + const launch = prepareOnly ? undefined : { executionId: randomUUID(), dispatchId: randomUUID(), receiptDirectory: await mkdtemp(join(dirname(workspace), 'receipts-')) } const startedAt = performance.now() let readyAt let preparingAt @@ -57,24 +59,51 @@ async function execute({ command, workspace, grants, port, onReady, onPreparing, let diagnostics = '' let ready = false let preparing = false + let supervisor + let protocol = Promise.resolve() + let protocolError child.stdout.on('data', data => chunks.push(data)) - child.stderr.on('data', (data) => { - diagnostics += data - if (releasedAt === undefined && diagnostics.includes('"type":"processesReleased"')) - releasedAt = performance.now() - if (!preparing && diagnostics.includes('"phase":"filesystem_preparing"')) { - preparing = true - preparingAt = performance.now() - onPreparing?.(child) - } - if (!ready && diagnostics.includes('"type":"ready"')) { - ready = true - readyAt = performance.now() - onReady?.(child) - } + const lines = createInterface({ input: child.stderr }) + lines.on('line', (line) => { + diagnostics += `${line}\n` + protocol = protocol.then(async () => { + const message = JSON.parse(line) + if (message.type === 'supervisor' && launch) { + assert.equal(supervisor, undefined, 'Duplicate sandbox supervisor') + assert.equal(message.protocol, 1) + assert.equal(message.identity.version, 3) + assert.equal(message.identity.pid, child.pid) + assert.equal(message.identity.executionId, launch.executionId) + assert.equal(message.identity.dispatchId, launch.dispatchId) + supervisor = message.identity + const prepared = JSON.parse(await readFile(join(launch.receiptDirectory, `${launch.executionId}.prepared.json`), 'utf8')) + assert.deepEqual(prepared, { version: 1, identity: supervisor, launchClosed: false }) + if (authorizeLaunch) + child.stdin.write(`${JSON.stringify({ executionId: launch.executionId, dispatchId: launch.dispatchId })}\n`) + else + child.stdin.end() + } + if (releasedAt === undefined && message.type === 'processesReleased') + releasedAt = performance.now() + if (!preparing && message.type === 'diagnostic' && message.phase === 'filesystem_preparing') { + preparing = true + preparingAt = performance.now() + onPreparing?.(child) + } + if (!ready && message.type === 'ready') { + if (launch) + assert.ok(supervisor, 'Command became ready before its supervisor handshake') + ready = true + readyAt = performance.now() + onReady?.(child) + } + }).catch((error) => { + protocolError ??= error + child.stdin.end() + }) }) child.stdin.on('error', () => {}) - child.stdin.write(`${JSON.stringify(prepareOnly ? { grants: identities, denyReadScope, synchronizeDenials } : { command, cwd: workspace, shell, privateRoot, proxyPort: port, grants: identities, denyReadScope, environment: { SystemRoot: process.env.SystemRoot, WINDIR: process.env.SystemRoot, ComSpec: join(process.env.SystemRoot, 'System32', 'cmd.exe'), PATHEXT: '.COM;.EXE;.BAT;.CMD', PSModulePath: join(dirname(shell), 'Modules'), PATH: `${dirname(shell)};${join(process.env.SystemRoot, 'System32')}`, HOME: home, USERPROFILE: home, TEMP: temporary, TMP: temporary, APPDATA: home, LOCALAPPDATA: home, POWERSHELL_TELEMETRY_OPTOUT: '1' } })}\n`) + child.stdin.write(`${JSON.stringify(prepareOnly ? { grants: identities, denyReadScope, synchronizeDenials } : { command, cwd: workspace, shell, privateRoot, proxyPort: port, grants: identities, denyReadScope, launch, environment: { SystemRoot: process.env.SystemRoot, WINDIR: process.env.SystemRoot, ComSpec: join(process.env.SystemRoot, 'System32', 'cmd.exe'), PATHEXT: '.COM;.EXE;.BAT;.CMD', PSModulePath: join(dirname(shell), 'Modules'), PATH: `${dirname(shell)};${join(process.env.SystemRoot, 'System32')}`, HOME: home, USERPROFILE: home, TEMP: temporary, TMP: temporary, APPDATA: home, LOCALAPPDATA: home, POWERSHELL_TELEMETRY_OPTOUT: '1' } })}\n`) const timer = setTimeout(() => child.stdin.end(), 60_000) const killTimer = setTimeout(() => child.kill(), 75_000) try { @@ -82,6 +111,14 @@ async function execute({ command, workspace, grants, port, onReady, onPreparing, child.once('error', reject) child.once('close', resolve) }) + await protocol + if (protocolError) + throw protocolError + if (launch && releasedAt !== undefined) { + assert.ok(supervisor, 'Sandbox reported release without a supervisor identity') + const released = JSON.parse(await readFile(join(launch.receiptDirectory, `${launch.executionId}.released.json`), 'utf8')) + assert.deepEqual(released, { version: 1, identity: supervisor, launchClosed: true }) + } const buffer = Buffer.concat(chunks) const output = buffer.subarray(0, 128).includes(0) ? buffer.toString('utf16le') : buffer.toString('utf8') await observed @@ -90,7 +127,10 @@ async function execute({ command, workspace, grants, port, onReady, onPreparing, finally { clearTimeout(timer) clearTimeout(killTimer) + lines.close() await rm(privateRoot, { recursive: true, force: true, maxRetries: 40, retryDelay: 50 }) + if (launch) + await rm(launch.receiptDirectory, { recursive: true, force: true }) } } @@ -120,6 +160,16 @@ nativeTest.test('Windows restricted-token behavior and known limits', { skip: pr const remote = Object.values(networkInterfaces()).flat().find(address => address?.family === 'IPv4' && !address.internal)?.address try { assert.ok(remote, 'Windows acceptance requires a non-loopback network interface') + await t.test('does not execute without a launch acknowledgement and seals its release receipt', async () => { + const marker = join(workspace, 'unacknowledged-command') + const result = await execute({ command: `[IO.File]::WriteAllText(${quote(marker)},'EXECUTED')`, workspace, port: proxy.port, grants: [{ path: workspace, access: 'write' }], authorizeLaunch: false }) + assert.notEqual(result.exitCode, 0, JSON.stringify(result)) + assert.match(result.diagnostics, /"type":"supervisor"/) + assert.match(result.diagnostics, /"type":"processesReleased"/) + assert(!result.diagnostics.includes('"phase":"filesystem_preparing"'), result.diagnostics) + assert(!result.diagnostics.includes('"type":"ready"'), result.diagnostics) + await assert.rejects(access(marker), { code: 'ENOENT' }) + }) await t.test('reconciles late command and workspace denials without releasing another configuration', async () => { const readGrant = { path: workspace, access: 'read' } const base = { workspace, port: proxy.port, grants: [readGrant] } @@ -273,7 +323,9 @@ foreach($target in @(@{key='proxy';host='127.0.0.1';port=${proxy.port}},@{key='o const result = await execute({ command: '[IO.File]::WriteAllText(\'must-not-run\',\'changed\')', workspace: current, port: proxy.port, grants: [{ path: current, access: 'write', device: String(metadata.dev), inode: String(metadata.ino) }] }) assert.notEqual(result.exitCode, 0, JSON.stringify(result)) assert(!result.diagnostics.includes('"type":"ready"'), result.diagnostics) - await assert.rejects(access(join(current, 'must-not-run'))) + const failure = result.diagnostics.trim().split('\n').map(line => JSON.parse(line)).find(message => message.type === 'diagnostic' && message.phase === 'filesystem') + assert.ok(failure, `Expected a directory identity failure after the launch handshake: ${result.diagnostics}`) + await assert.rejects(access(join(current, 'must-not-run')), { code: 'ENOENT' }) }) await t.test('supports PowerShell paths, file operations and native children', async () => { await writeFile(join(workspace, 'result.txt'), 'delete-fixture') diff --git a/apps/buddy/native/host/__tests__/supervised_receipts_linux.rs b/apps/buddy/native/host/__tests__/supervised_receipts_linux.rs new file mode 100644 index 000000000..2322c37d3 --- /dev/null +++ b/apps/buddy/native/host/__tests__/supervised_receipts_linux.rs @@ -0,0 +1,248 @@ +#![cfg(target_os = "linux")] + +use serde_json::{Value, json}; +use std::{ + io::{BufRead, BufReader, Write}, + path::Path, + process::{Child, Command, Stdio}, +}; + +fn private_root() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + root +} + +struct Execution { + child: Child, + control: BufReader, + identity: Value, + launch: Value, +} + +impl Execution { + fn new(root: &Path) -> Self { + let launch = json!({ + "executionId": uuid::Uuid::new_v4().to_string(), + "dispatchId": uuid::Uuid::new_v4().to_string(), + "receiptDirectory": root, + }); + let mut child = spawn(&launch); + let mut control = BufReader::new(child.stderr.take().unwrap()); + let mut line = String::new(); + control.read_line(&mut line).unwrap(); + let ready: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(ready["protocol"], 2, "{ready}"); + Self { + child, + control, + identity: ready["identity"].clone(), + launch, + } + } + + fn start(&mut self, command: &str) { + let request = json!({ + "shell": "/bin/bash", "args": ["--noprofile", "--norc", "-c"], + "command": command, "commandTransport": "argv", "cwd": "/tmp", + "environment": {"PATH": "/usr/bin:/bin"}, + "executionId": self.launch["executionId"], "dispatchId": self.launch["dispatchId"], + }); + writeln!(self.child.stdin.as_mut().unwrap(), "{request}").unwrap(); + } + + fn dispatch(&self) -> Value { + json!({"kind":"native-shell-dispatch","platform":"linux","version":3, + "executionId":self.launch["executionId"],"dispatchId":self.launch["dispatchId"]}) + } + + fn wait(&mut self) { + self.child.wait().unwrap(); + let mut remaining = String::new(); + std::io::Read::read_to_string(&mut self.control, &mut remaining).unwrap(); + } +} + +impl Drop for Execution { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn spawn(launch: &Value) -> Child { + Command::new(env!("CARGO_BIN_EXE_lexora-buddy-process-control")) + .args(["--supervise-shell", &launch.to_string()]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() +} + +fn verify(identity: Value, root: &Path) -> Value { + let mut query = Command::new(env!("CARGO_BIN_EXE_lexora-buddy-process-control")) + .arg("--reverify-owned-process") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + serde_json::to_writer( + query.stdin.take().unwrap(), + &json!({"identity":identity,"receiptDirectory":root}), + ) + .unwrap(); + let output = query.wait_with_output().unwrap(); + assert!(output.status.success()); + serde_json::from_slice(&output.stdout).unwrap() +} + +#[test] +fn closed_gate_has_a_durable_receipt_even_when_ready_was_not_persisted_by_the_caller() { + let root = private_root(); + let mut execution = Execution::new(root.path()); + assert_eq!( + verify(execution.dispatch(), root.path())["status"], + "active" + ); + drop(execution.child.stdin.take()); + execution.wait(); + assert_eq!( + verify(execution.dispatch(), root.path())["reason"], + "supervisor_receipt" + ); +} + +#[test] +fn receipt_survives_supervisor_exit_and_proves_detached_descendants_were_reaped() { + let root = private_root(); + let mut execution = Execution::new(root.path()); + execution.start("setsid sh -c 'sleep 60 >/dev/null 2>&1 & echo $!' & wait"); + let mut output = String::new(); + std::io::Read::read_to_string(&mut execution.child.stdout.take().unwrap(), &mut output) + .unwrap(); + execution.wait(); + let pid: u32 = output.trim().parse().unwrap(); + assert!(!Path::new(&format!("/proc/{pid}")).exists()); + assert_eq!( + verify(execution.dispatch(), root.path())["status"], + "released" + ); + let mut identity = execution.identity.clone(); + identity["kind"] = json!("native-shell"); + identity["platform"] = json!("linux"); + assert_eq!( + verify(identity.clone(), root.path())["reason"], + "supervisor_receipt" + ); + identity["startTime"] = json!("0"); + assert_eq!(verify(identity, root.path())["reason"], "receipt_invalid"); +} + +#[test] +fn malformed_receipt_owner_is_never_release_evidence() { + use std::os::unix::fs::PermissionsExt; + let root = private_root(); + let mut execution = Execution::new(root.path()); + drop(execution.child.stdin.take()); + execution.wait(); + let mut invalid_owners = vec![json!({ + "executionId": execution.launch["executionId"], + "dispatchId": execution.launch["dispatchId"], + })]; + for (key, value) in [ + ("version", json!(99)), + ("pid", json!(0)), + ("startTime", json!("invalid")), + ("bootId", json!("invalid")), + ("jobName", json!("unexpected")), + ] { + let mut identity = execution.identity.clone(); + identity[key] = value; + invalid_owners.push(identity); + } + for identity in invalid_owners { + for (phase, closed) in [("prepared", false), ("released", true)] { + let file = root.path().join(format!( + "{}.{}.json", + execution.launch["executionId"].as_str().unwrap(), + phase + )); + std::fs::write( + &file, + json!({"version":1, "identity":identity, "launchClosed":closed}).to_string(), + ) + .unwrap(); + std::fs::set_permissions(file, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + assert_eq!( + verify(execution.dispatch(), root.path())["reason"], + "receipt_invalid" + ); + } +} + +#[test] +fn duplicate_execution_never_enters_the_command_gate() { + let root = private_root(); + let mut execution = Execution::new(root.path()); + drop(execution.child.stdin.take()); + execution.wait(); + let output = spawn(&execution.launch).wait_with_output().unwrap(); + assert!(!output.status.success()); + assert!( + !String::from_utf8(output.stderr) + .unwrap() + .contains("\"type\":\"ready\"") + ); +} + +#[test] +fn killed_owner_without_a_receipt_stays_unknown() { + let root = private_root(); + let mut execution = Execution::new(root.path()); + execution.child.kill().unwrap(); + execution.wait(); + assert_eq!( + verify(execution.dispatch(), root.path())["reason"], + "descendants_unverified" + ); +} + +#[test] +fn malformed_or_missing_receipt_cannot_prove_release() { + let root = private_root(); + let other = private_root(); + let mut execution = Execution::new(root.path()); + drop(execution.child.stdin.take()); + execution.wait(); + assert_eq!( + verify(execution.dispatch(), other.path())["reason"], + "receipt_missing" + ); + let released = root.path().join(format!( + "{}.released.json", + execution.launch["executionId"].as_str().unwrap() + )); + std::fs::write(released, b"{").unwrap(); + assert_eq!( + verify(execution.dispatch(), root.path())["reason"], + "receipt_invalid" + ); +} + +#[test] +fn permissive_or_symlinked_receipt_root_is_rejected_before_ready() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let root = private_root(); + let mut launch = json!({"executionId":uuid::Uuid::new_v4().to_string(),"dispatchId":uuid::Uuid::new_v4().to_string(),"receiptDirectory":root.path()}); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(!spawn(&launch).wait_with_output().unwrap().status.success()); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let links = private_root(); + let link = links.path().join("link"); + symlink(root.path(), &link).unwrap(); + launch["receiptDirectory"] = json!(link); + assert!(!spawn(&launch).wait_with_output().unwrap().status.success()); +} diff --git a/apps/buddy/native/host/__tests__/windows_boot.rs b/apps/buddy/native/host/__tests__/windows_boot.rs new file mode 100644 index 000000000..5e4fb629f --- /dev/null +++ b/apps/buddy/native/host/__tests__/windows_boot.rs @@ -0,0 +1,51 @@ +use super::*; + +fn evidence(sequence: u32, uptime: u64) -> Evidence { + Evidence { + source: "windows-boot-sequence".into(), + version: 1, + architecture: "x64".into(), + build: 26100, + sequence, + uptime_millis: uptime.to_string(), + } +} + +#[test] +fn boot_attempt_without_kernel_uptime_reset_is_not_release_evidence() { + assert!(!evidence(2, 5000).proves_restart_from(&evidence(1, 4000))); + assert!(!evidence(2, 4000).proves_restart_from(&evidence(1, 4000))); + assert!(!evidence(1, 3000).proves_restart_from(&evidence(1, 4000))); +} + +#[test] +fn only_a_supported_same_build_sequence_change_with_lower_uptime_proves_restart() { + assert!(evidence(2, 3000).proves_restart_from(&evidence(1, 4000))); + let mut previous = evidence(1, 4000); + previous.build = 22631; + assert!(!evidence(2, 3000).proves_restart_from(&previous)); + previous.build = 99999; + assert!(!evidence(2, 3000).proves_restart_from(&previous)); +} + +#[test] +fn malformed_or_differently_versioned_evidence_is_not_accepted() { + let mut previous = evidence(1, 4000); + previous.uptime_millis = "-1".into(); + assert!(!evidence(2, 3000).proves_restart_from(&previous)); + previous = evidence(1, 4000); + previous.version = 2; + assert!(!evidence(2, 3000).proves_restart_from(&previous)); +} + +#[test] +fn shared_page_layout_rejects_unrecognized_os_and_builds() { + let mut bytes = [0; 0x2c8]; + assert!(decode(&bytes).is_err()); + bytes[0x26c..0x270].copy_from_slice(&10u32.to_le_bytes()); + bytes[0x260..0x264].copy_from_slice(&26100u32.to_le_bytes()); + bytes[0x2c4..0x2c8].copy_from_slice(&2u32.to_le_bytes()); + assert_eq!(decode(&bytes).unwrap(), (26100, 2)); + bytes[0x260..0x264].copy_from_slice(&99999u32.to_le_bytes()); + assert!(decode(&bytes).is_err()); +} diff --git a/apps/buddy/native/host/src/bin/process_control.rs b/apps/buddy/native/host/src/bin/process_control.rs index c0c1c5e54..4bcf32afe 100644 --- a/apps/buddy/native/host/src/bin/process_control.rs +++ b/apps/buddy/native/host/src/bin/process_control.rs @@ -71,8 +71,12 @@ fn main() -> ExitCode { }; } #[cfg(any(windows, target_os = "linux"))] - if command == "--supervise-shell" && arguments.next().is_none() { - return match lexora_buddy_host::supervised_shell::run() { + if command == "--supervise-shell" { + let configuration = arguments.next().and_then(|value| value.into_string().ok()); + if arguments.next().is_some() { + return ExitCode::FAILURE; + } + return match lexora_buddy_host::supervised_shell::run(configuration) { Ok(code) => ExitCode::from(code.clamp(0, 255) as u8), Err(error) => { eprintln!( diff --git a/apps/buddy/native/host/src/shell_sandbox.rs b/apps/buddy/native/host/src/shell_sandbox.rs index a8fde156c..0a1efbeb8 100644 --- a/apps/buddy/native/host/src/shell_sandbox.rs +++ b/apps/buddy/native/host/src/shell_sandbox.rs @@ -17,7 +17,7 @@ const SERVICE_NAME: &str = "LexoraBuddySandboxNetwork"; const PIPE_NAME: &str = r"\\.\pipe\LexoraBuddySandboxNetwork-v1"; const PROFILE_PREFIX: &str = "Lexora.Buddy.Sandbox."; const PROTOCOL_VERSION: u32 = 1; -const COMPONENT_VERSION: u32 = 4; +const COMPONENT_VERSION: u32 = 5; pub fn run() -> io::Result { let arguments: Vec = std::env::args().skip(1).collect(); diff --git a/apps/buddy/native/host/src/shell_sandbox/process.rs b/apps/buddy/native/host/src/shell_sandbox/process.rs index f2ce82b5b..614d25958 100644 --- a/apps/buddy/native/host/src/shell_sandbox/process.rs +++ b/apps/buddy/native/host/src/shell_sandbox/process.rs @@ -44,6 +44,7 @@ struct Request { #[serde(default)] deny_read_scope: Option, proxy_port: u16, + launch: crate::supervised_shell::receipt::Launch, } pub(super) fn run() -> io::Result { @@ -60,9 +61,8 @@ pub(super) fn run() -> io::Result { return Err(io::ErrorKind::InvalidInput.into()); } let request: Request = serde_json::from_slice(&bytes)?; - let cancellation = Cancellation::listen(input, Duration::from_secs(60)); - cancellation.check()?; validate(&request)?; + request.launch.validate()?; let (preparation, account, group, job) = at( "job", Lease::account( @@ -74,89 +74,123 @@ pub(super) fn run() -> io::Result { .collect(), ), )?; + let mut identity = crate::supervised_shell::windows::identity(&job)?; + identity["version"] = serde_json::json!(3); + identity["executionId"] = serde_json::json!(request.launch.execution_id); + identity["dispatchId"] = serde_json::json!(request.launch.dispatch_id); + let receipt = crate::supervised_shell::receipt::Receipt::prepare(&request.launch, &identity)?; eprintln!( "{}", - serde_json::json!({"type":"supervisor","protocol":1,"identity":crate::supervised_shell::windows::identity(&job)?}) - ); - let mut policy = request.grants; - policy.push(inspect(&request.private_root, Access::Write)?); - eprintln!( - "{}", - serde_json::json!({"type":"diagnostic","phase":"filesystem_preparing","nativeCode":null}) + serde_json::json!({"type":"supervisor","protocol":1,"identity":identity}) ); - let grants = at( - "filesystem", - Grants::apply( - &account, - &group, - &policy, - &cancellation, - request.deny_read_scope.as_deref().map(DenialUpdate::Merge), - ), - )?; - cancellation.check()?; - let script = Path::new(&request.private_root).join("command.ps1"); - let mut script_file = OpenOptions::new() - .write(true) - .create_new(true) - .open(&script)?; - script_file.write_all( + let mut launch_sealed = true; + let result = (|| -> io::Result { + let mut ack = Vec::new(); + (&mut input).take(1025).read_until(b'\n', &mut ack)?; + let accepted = serde_json::from_slice::(&ack).is_ok_and(|ack| { + ack.execution_id == request.launch.execution_id + && ack.dispatch_id == request.launch.dispatch_id + }); + if !accepted || ack.len() > 1024 || ack.last() != Some(&b'\n') { + return Err(io::ErrorKind::Interrupted.into()); + } + let cancellation = Cancellation::listen(input, Duration::from_secs(60)); + cancellation.check()?; + let mut policy = request.grants; + policy.push(inspect(&request.private_root, Access::Write)?); + eprintln!( + "{}", + serde_json::json!({"type":"diagnostic","phase":"filesystem_preparing","nativeCode":null}) + ); + let grants = at( + "filesystem", + Grants::apply( + &account, + &group, + &policy, + &cancellation, + request.deny_read_scope.as_deref().map(DenialUpdate::Merge), + ), + )?; + cancellation.check()?; + let script = Path::new(&request.private_root).join("command.ps1"); + let mut script_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&script)?; + script_file.write_all( b"\xef\xbb\xbftry { [Console]::OutputEncoding=[System.Text.Encoding]::UTF8 } catch {}\n", )?; - script_file.write_all(request.command.as_bytes())?; - script_file.write_all( - b"\nif (-not $?) { if ($LASTEXITCODE) { exit $LASTEXITCODE } else { exit 1 } }\n", - )?; - drop(script_file); - cancellation.check()?; - let (lease, child) = at( - "launch", - Lease::connect(LaunchRequest { - shell: request.shell, - script: script.to_string_lossy().into_owned(), - cwd: request.cwd, - environment: request.environment, - capabilities: grants.capabilities.clone(), - job: job.0.as_raw_handle() as usize, - // SAFETY: stdout belongs to the supervisor; the authenticated broker duplicates it without inheriting control input. - output: unsafe { GetStdHandle(STD_OUTPUT_HANDLE) } as usize, - }), - )?; - preparation.finish()?; - cancellation.check()?; - eprintln!( - "{}", - serde_json::json!({"type":"ready","protocol":super::PROTOCOL_VERSION}) - ); - // SAFETY: The process is still suspended, already confined to its restricted token and kill-on-close job. - if unsafe { ResumeThread(child.thread.as_raw_handle()) } == u32::MAX { - return Err(io::Error::last_os_error()); - } - let deadline = Instant::now() + Duration::from_secs(86400); - let result = loop { - if cancellation.cancelled() || !lease.alive() || Instant::now() >= deadline { - break Err(io::ErrorKind::Interrupted.into()); - } - // SAFETY: The process handle is live and queryable throughout this bounded wait. - if unsafe { WaitForSingleObject(child.process.as_raw_handle(), 25) } == WAIT_OBJECT_0 { - let mut code = 0; - // SAFETY: The process finished and the exit-code output is writable. - boolean(unsafe { GetExitCodeProcess(child.process.as_raw_handle(), &mut code) })?; - break Ok(code as i32); + script_file.write_all(request.command.as_bytes())?; + script_file.write_all( + b"\nif (-not $?) { if ($LASTEXITCODE) { exit $LASTEXITCODE } else { exit 1 } }\n", + )?; + drop(script_file); + cancellation.check()?; + launch_sealed = false; + let (lease, child) = at( + "launch", + Lease::connect(LaunchRequest { + shell: request.shell, + script: script.to_string_lossy().into_owned(), + cwd: request.cwd, + environment: request.environment, + capabilities: grants.capabilities.clone(), + job: job.0.as_raw_handle() as usize, + // SAFETY: stdout belongs to the supervisor; the authenticated broker duplicates it without inheriting control input. + output: unsafe { GetStdHandle(STD_OUTPUT_HANDLE) } as usize, + }), + )?; + preparation.finish()?; + cancellation.check()?; + eprintln!( + "{}", + serde_json::json!({"type":"ready","protocol":super::PROTOCOL_VERSION}) + ); + // SAFETY: The process is still suspended, already confined to its restricted token and kill-on-close job. + if unsafe { ResumeThread(child.thread.as_raw_handle()) } == u32::MAX { + return Err(io::Error::last_os_error()); } - }; + let deadline = Instant::now() + Duration::from_secs(86400); + let result = loop { + if cancellation.cancelled() || !lease.alive() || Instant::now() >= deadline { + break Err(io::ErrorKind::Interrupted.into()); + } + // SAFETY: The process handle is live and queryable throughout this bounded wait. + if unsafe { WaitForSingleObject(child.process.as_raw_handle(), 25) } == WAIT_OBJECT_0 { + let mut code = 0; + // SAFETY: The process finished and the exit-code output is writable. + boolean(unsafe { GetExitCodeProcess(child.process.as_raw_handle(), &mut code) })?; + break Ok(code as i32); + } + }; + job.finish()?; + drop(child); + lease.finish()?; + launch_sealed = true; + drop(grants); + result + })(); job.finish()?; - eprintln!( - "{}", - serde_json::json!({"type":"processesReleased","protocol":1}) - ); - drop(child); - drop(job); - lease.finish()?; - drop(grants); + if launch_sealed { + if let Some(receipt) = &receipt { + receipt.release()?; + } + eprintln!( + "{}", + serde_json::json!({"type":"processesReleased","protocol":1}) + ); + } result } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LaunchAck { + execution_id: String, + dispatch_id: String, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Preparation { diff --git a/apps/buddy/native/host/src/supervised_shell.rs b/apps/buddy/native/host/src/supervised_shell.rs index fd4fc4ae0..315a9db77 100644 --- a/apps/buddy/native/host/src/supervised_shell.rs +++ b/apps/buddy/native/host/src/supervised_shell.rs @@ -9,6 +9,7 @@ use std::{ use serde::Deserialize; +pub(crate) mod receipt; pub mod recovery; #[cfg(target_os = "linux")] @@ -30,6 +31,8 @@ struct Request { command_transport: Transport, cwd: String, environment: BTreeMap, + execution_id: Option, + dispatch_id: Option, } #[derive(Deserialize)] @@ -39,12 +42,26 @@ enum Transport { Stdin, } -pub fn run() -> io::Result { +pub fn run(configuration: Option) -> io::Result { + let launch = configuration + .map(|value| serde_json::from_str::(&value)) + .transpose()?; + if let Some(launch) = &launch { + launch.validate()?; + } let supervision = backend::initialize()?; - let identity = backend::identity(&supervision)?; + let mut identity = backend::identity(&supervision)?; + let receipt = if let Some(launch) = &launch { + identity["version"] = serde_json::json!(3); + identity["executionId"] = serde_json::json!(launch.execution_id); + identity["dispatchId"] = serde_json::json!(launch.dispatch_id); + receipt::Receipt::prepare(launch, &identity)? + } else { + None + }; eprintln!( "{}", - serde_json::json!({"type":"ready","protocol":1,"identity":identity}) + serde_json::json!({"type":"ready","protocol":if launch.is_some() {2} else {1},"identity":identity}) ); let mut input = BufReader::new(io::stdin()); let mut bytes = Vec::new(); @@ -52,6 +69,9 @@ pub fn run() -> io::Result { .take(2 * 1024 * 1024 + 1) .read_until(b'\n', &mut bytes)?; if bytes.len() > 2 * 1024 * 1024 || bytes.last() != Some(&b'\n') { + if let Some(receipt) = &receipt { + receipt.release()?; + } released(); return Err(io::ErrorKind::InvalidInput.into()); } @@ -75,10 +95,23 @@ pub fn run() -> io::Result { let request = match request { Ok(request) => request, Err(error) => { + if let Some(receipt) = &receipt { + receipt.release()?; + } released(); return Err(error.into()); } }; + if launch.as_ref().is_some_and(|launch| { + request.execution_id.as_ref() != Some(&launch.execution_id) + || request.dispatch_id.as_ref() != Some(&launch.dispatch_id) + }) { + if let Some(receipt) = &receipt { + receipt.release()?; + } + released(); + return Err(io::ErrorKind::InvalidInput.into()); + } let cancelled = Arc::new(AtomicBool::new(false)); let cancellation = Arc::clone(&cancelled); std::thread::spawn(move || { @@ -86,7 +119,7 @@ pub fn run() -> io::Result { let _ = input.read(&mut byte); cancellation.store(true, Ordering::Release); }); - backend::run(&supervision, &request, &cancelled) + backend::run(&supervision, &request, &cancelled, receipt.as_ref()) } fn released() { diff --git a/apps/buddy/native/host/src/supervised_shell/linux.rs b/apps/buddy/native/host/src/supervised_shell/linux.rs index 481e460f6..d315493b5 100644 --- a/apps/buddy/native/host/src/supervised_shell/linux.rs +++ b/apps/buddy/native/host/src/supervised_shell/linux.rs @@ -31,9 +31,13 @@ pub(super) fn run( _supervision: &Supervision, request: &Request, cancelled: &AtomicBool, + receipt: Option<&super::receipt::Receipt>, ) -> io::Result { let result = execute(request, cancelled); cleanup()?; + if let Some(receipt) = receipt { + receipt.release()?; + } super::released(); result } diff --git a/apps/buddy/native/host/src/supervised_shell/receipt.rs b/apps/buddy/native/host/src/supervised_shell/receipt.rs new file mode 100644 index 000000000..b3123287e --- /dev/null +++ b/apps/buddy/native/host/src/supervised_shell/receipt.rs @@ -0,0 +1,211 @@ +use cap_std::fs::{Dir, OpenOptions}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + fs, + io::{self, Read, Write}, + path::Path, +}; + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Launch { + pub execution_id: String, + pub dispatch_id: String, + pub receipt_directory: Option, +} + +impl Launch { + pub(crate) fn validate(&self) -> io::Result<()> { + if !super::recovery::valid_uuid(&self.execution_id) + || !super::recovery::valid_uuid(&self.dispatch_id) + { + return Err(io::ErrorKind::InvalidInput.into()); + } + Ok(()) + } +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Record { + version: u32, + identity: Value, + launch_closed: bool, +} + +pub(crate) struct Receipt { + directory: Dir, + execution_id: String, + identity: Value, + #[cfg(windows)] + _guard: crate::windows_fs::ParentGuard, +} + +impl Receipt { + pub(crate) fn prepare(launch: &Launch, identity: &Value) -> io::Result> { + launch.validate()?; + let Some(path) = &launch.receipt_directory else { + return Ok(None); + }; + let directory = open_directory(path, true)?; + let receipt = Self { + directory, + execution_id: launch.execution_id.clone(), + identity: identity.clone(), + #[cfg(windows)] + _guard: crate::windows_fs::ParentGuard::new(&Path::new(path).join("receipt"))?, + }; + receipt.write("prepared", false)?; + Ok(Some(receipt)) + } + + pub(crate) fn release(&self) -> io::Result<()> { + self.write("released", true) + } + + fn write(&self, phase: &str, launch_closed: bool) -> io::Result<()> { + let name = format!("{}.{}.json", self.execution_id, phase); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.mode(0o600); + } + let temporary = format!("{name}.pending"); + let mut file = self + .directory + .open_with(if launch_closed { &temporary } else { &name }, &options)?; + serde_json::to_writer( + &mut file, + &Record { + version: 1, + identity: self.identity.clone(), + launch_closed, + }, + )?; + file.write_all(b"\n")?; + file.sync_all()?; + drop(file); + if launch_closed { + self.directory.rename(&temporary, &self.directory, &name)?; + } + #[cfg(unix)] + self.directory.open(".")?.sync_all()?; + Ok(()) + } +} + +fn open_directory(path: &str, create: bool) -> io::Result { + let path = Path::new(path); + if !path.is_absolute() { + return Err(io::ErrorKind::InvalidInput.into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::{DirBuilderExt, MetadataExt}; + if create { + match fs::DirBuilder::new().mode(0o700).create(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_dir() + || metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & 0o077 != 0 + { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let directory = Dir::open_ambient_dir(path, cap_std::ambient_authority())?; + let opened = directory.dir_metadata()?; + use cap_std::fs::MetadataExt as _; + if opened.dev() != metadata.dev() || opened.ino() != metadata.ino() { + return Err(io::ErrorKind::PermissionDenied.into()); + } + Ok(directory) + } + #[cfg(windows)] + { + if !create && !fs::symlink_metadata(path)?.is_dir() { + return Err(io::ErrorKind::PermissionDenied.into()); + } + crate::private_directories::ensure(&[path.to_string_lossy().into_owned()]) + .map_err(io::Error::other)?; + Ok(Dir::from_std_file(crate::windows_fs::open_directory(path)?)) + } +} + +pub(super) fn recover( + root: &str, + execution_id: &str, + dispatch_id: &str, +) -> io::Result<(Value, bool)> { + if !super::recovery::valid_uuid(execution_id) || !super::recovery::valid_uuid(dispatch_id) { + return Err(io::ErrorKind::InvalidInput.into()); + } + let directory = open_directory(root, false)?; + let prepared = read(&directory, &format!("{execution_id}.prepared.json"))?; + if prepared.version != 1 + || prepared.launch_closed + || prepared.identity["executionId"] != execution_id + || prepared.identity["dispatchId"] != dispatch_id + { + return Err(io::ErrorKind::InvalidData.into()); + } + let released = match read(&directory, &format!("{execution_id}.released.json")) { + Ok(record) => { + if record.version != 1 || !record.launch_closed || record.identity != prepared.identity + { + return Err(io::ErrorKind::InvalidData.into()); + } + true + } + Err(error) if error.kind() == io::ErrorKind::NotFound => false, + Err(error) => return Err(error), + }; + Ok((prepared.identity, released)) +} + +fn read(directory: &Dir, name: &str) -> io::Result { + let metadata = directory.symlink_metadata(name)?; + if !metadata.is_file() || metadata.len() > 8192 { + return Err(io::ErrorKind::InvalidData.into()); + } + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + if metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & 0o077 != 0 + || metadata.nlink() != 1 + { + return Err(io::ErrorKind::PermissionDenied.into()); + } + } + let file = directory.open(name)?; + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + let opened = file.metadata()?; + if opened.dev() != metadata.dev() + || opened.ino() != metadata.ino() + || opened.uid() != rustix::process::geteuid().as_raw() + || opened.mode() & 0o077 != 0 + || opened.nlink() != 1 + { + return Err(io::ErrorKind::PermissionDenied.into()); + } + } + #[cfg(windows)] + if crate::windows_fs::is_name_surrogate(&file)? { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let mut bytes = Vec::new(); + file.take(8193).read_to_end(&mut bytes)?; + if bytes.len() > 8192 { + return Err(io::ErrorKind::InvalidData.into()); + } + Ok(serde_json::from_slice(&bytes)?) +} diff --git a/apps/buddy/native/host/src/supervised_shell/recovery.rs b/apps/buddy/native/host/src/supervised_shell/recovery.rs index 447539d1a..fa276c3e7 100644 --- a/apps/buddy/native/host/src/supervised_shell/recovery.rs +++ b/apps/buddy/native/host/src/supervised_shell/recovery.rs @@ -8,11 +8,14 @@ mod backend; #[cfg(windows)] #[path = "recovery/windows.rs"] pub(crate) mod backend; +#[cfg(any(windows, test))] +#[path = "recovery/windows_boot.rs"] +pub(crate) mod windows_boot; const MAX_INPUT_BYTES: u64 = 8192; pub(crate) const WINDOWS_JOB_PREFIX: &str = r"Global\Lexora.Buddy.Shell.v2."; -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Identity { kind: String, @@ -23,24 +26,45 @@ pub(super) struct Identity { version: Option, boot_id: Option, job_name: Option, + execution_id: Option, + dispatch_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + boot_evidence: Option, #[serde(rename = "requestId")] _request_id: Option, } #[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub(super) struct Checkpoint { +pub(super) struct LegacyCheckpoint { kind: String, version: u32, platform: String, boot_id: String, } +#[derive(Deserialize, Serialize)] +#[serde(untagged)] +pub(super) enum Checkpoint { + Legacy(LegacyCheckpoint), + Windows(WindowsCheckpoint), +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct WindowsCheckpoint { + kind: String, + version: u32, + platform: String, + boot_evidence: serde_json::Value, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct Request { identity: Identity, checkpoint: Option, + receipt_directory: Option, } #[derive(Serialize)] @@ -66,19 +90,38 @@ impl Verification { } pub fn checkpoint(mut output: impl Write) -> io::Result<()> { - let result = backend::boot_id().map(|boot_id| Checkpoint { - kind: "recovery-boot".into(), - version: 1, - platform: backend::PLATFORM.into(), - boot_id, - }); - match result { - Ok(checkpoint) => serde_json::to_writer(&mut output, &checkpoint)?, - Err(_) => { - serde_json::to_writer(&mut output, &Verification::unknown("boot_id_unavailable"))? + #[cfg(windows)] + { + let result = windows_boot::capture().map(|boot| WindowsCheckpoint { + kind: "recovery-boot".into(), + version: 2, + platform: backend::PLATFORM.into(), + boot_evidence: serde_json::to_value(boot).unwrap_or_default(), + }); + match result { + Ok(checkpoint) => serde_json::to_writer(&mut output, &checkpoint)?, + Err(_) => { + serde_json::to_writer(&mut output, &Verification::unknown("boot_id_unavailable"))? + } } + Ok(()) + } + #[cfg(not(windows))] + { + let result = backend::boot_id().map(|boot_id| LegacyCheckpoint { + kind: "recovery-boot".into(), + version: 1, + platform: backend::PLATFORM.into(), + boot_id, + }); + match result { + Ok(checkpoint) => serde_json::to_writer(&mut output, &checkpoint)?, + Err(_) => { + serde_json::to_writer(&mut output, &Verification::unknown("boot_id_unavailable"))? + } + } + Ok(()) } - Ok(()) } pub fn reverify(input: impl Read, mut output: impl Write) -> io::Result<()> { @@ -95,7 +138,74 @@ pub fn reverify(input: impl Read, mut output: impl Write) -> io::Result<()> { Ok(()) } -fn verify(request: Request) -> Verification { +fn verify(mut request: Request) -> Verification { + if request.identity.version == Some(3) { + let identity = &request.identity; + if identity.platform != backend::PLATFORM { + return Verification::unknown("platform_mismatch"); + } + if !matches!( + identity.kind.as_str(), + "native-shell" | "sandbox-job" | "native-shell-dispatch" | "sandbox-dispatch" + ) { + return Verification::unknown("unsupported_process"); + } + let (Some(execution_id), Some(dispatch_id)) = + (&identity.execution_id, &identity.dispatch_id) + else { + return Verification::unknown("identity_incomplete"); + }; + if !valid_uuid(execution_id) || !valid_uuid(dispatch_id) { + return Verification::unknown("identity_invalid"); + } + if let Some(root) = &request.receipt_directory { + match super::receipt::recover(root, execution_id, dispatch_id) { + Ok((mut prepared, released)) => { + let dispatch = matches!( + identity.kind.as_str(), + "native-shell-dispatch" | "sandbox-dispatch" + ); + if !dispatch + && (prepared["pid"].as_u64() != identity.pid.map(u64::from) + || prepared["startTime"].as_str() != identity.start_time.as_deref() + || prepared["jobName"].as_str() != identity.job_name.as_deref() + || prepared["bootId"].as_str() != identity.boot_id.as_deref() + || prepared.get("bootEvidence") != identity.boot_evidence.as_ref()) + { + return Verification::unknown("receipt_invalid"); + } + prepared["kind"] = serde_json::json!(if identity.platform == "win32" + && identity.kind == "sandbox-job" + { + "sandbox-job" + } else { + "native-shell" + }); + prepared["platform"] = serde_json::json!(identity.platform); + let owner: Identity = match serde_json::from_value(prepared) { + Ok(identity) => identity, + Err(_) => return Verification::unknown("receipt_invalid"), + }; + if !valid_receipt_owner(&owner) { + return Verification::unknown("receipt_invalid"); + } + if released { + return Verification::new("released", "supervisor_receipt"); + } + request.identity = owner; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Verification::unknown("receipt_missing"); + } + Err(_) => return Verification::unknown("receipt_invalid"), + } + } else if matches!( + identity.kind.as_str(), + "native-shell-dispatch" | "sandbox-dispatch" + ) { + return Verification::unknown("receipt_root_unavailable"); + } + } let identity = &request.identity; if !matches!(identity.kind.as_str(), "native-shell" | "sandbox-job") || (identity.kind == "sandbox-job" && identity.platform != "win32") @@ -111,18 +221,23 @@ fn verify(request: Request) -> Verification { || !value.bytes().all(|byte| byte.is_ascii_digit()) || value.parse::().is_err() }) - || identity.nonce.as_deref().is_none_or(str::is_empty) + || (identity.version != Some(3) && identity.nonce.as_deref().is_none_or(str::is_empty)) { return Verification::unknown("identity_incomplete"); } - if identity.version.is_some_and(|version| version != 2) { + if identity + .version + .is_some_and(|version| version != 2 && version != 3) + { return Verification::unknown("identity_version_unsupported"); } if identity .boot_id .as_deref() .is_some_and(|value| !valid_uuid(value)) - || (identity.version == Some(2) && identity.boot_id.is_none()) + || (identity.version == Some(2) + && identity.platform != "win32" + && identity.boot_id.is_none()) { return Verification::unknown("identity_invalid"); } @@ -139,33 +254,123 @@ fn verify(request: Request) -> Verification { { return Verification::unknown("identity_incomplete"); } - let current_boot = match backend::boot_id() { - Ok(boot) => boot, - Err(_) => return Verification::unknown("boot_id_unavailable"), - }; - if identity - .boot_id - .as_ref() - .is_some_and(|boot| *boot != current_boot) + #[cfg(windows)] { - return Verification::new("released", "boot_changed"); + verify_windows_boot(identity, request.checkpoint) } - if let Some(checkpoint) = request.checkpoint { - if checkpoint.kind != "recovery-boot" - || checkpoint.version != 1 - || checkpoint.platform != identity.platform - || !valid_uuid(&checkpoint.boot_id) + #[cfg(not(windows))] + { + let current_boot = match backend::boot_id() { + Ok(boot) => boot, + Err(_) => return Verification::unknown("boot_id_unavailable"), + }; + if identity + .boot_id + .as_ref() + .is_some_and(|boot| *boot != current_boot) { - return Verification::unknown("checkpoint_invalid"); + return Verification::new("released", "boot_changed"); + } + if let Some(checkpoint) = request.checkpoint { + let Checkpoint::Legacy(checkpoint) = checkpoint else { + return Verification::unknown("checkpoint_invalid"); + }; + if checkpoint.kind != "recovery-boot" + || checkpoint.version != 1 + || checkpoint.platform != identity.platform + || !valid_uuid(&checkpoint.boot_id) + { + return Verification::unknown("checkpoint_invalid"); + } + if checkpoint.boot_id != current_boot { + return Verification::new("released", "boot_changed"); + } + } + backend::verify(identity) + } +} + +fn valid_receipt_owner(identity: &Identity) -> bool { + if identity.version != Some(3) + || identity.platform != backend::PLATFORM + || identity.pid.is_none_or(|pid| pid == 0) + || identity.start_time.as_deref().is_none_or(|value| { + value.is_empty() + || !value.bytes().all(|byte| byte.is_ascii_digit()) + || value.parse::().is_err() + }) + || identity + .execution_id + .as_deref() + .is_none_or(|value| !valid_uuid(value)) + || identity + .dispatch_id + .as_deref() + .is_none_or(|value| !valid_uuid(value)) + || identity + .boot_id + .as_deref() + .is_some_and(|value| !valid_uuid(value)) + { + return false; + } + #[cfg(target_os = "linux")] + { + identity.boot_id.is_some() + && identity.job_name.is_none() + && identity.boot_evidence.is_none() + } + #[cfg(windows)] + { + identity.job_name.as_deref().is_some_and(|name| { + name.strip_prefix(WINDOWS_JOB_PREFIX) + .is_some_and(valid_uuid) + }) && identity.boot_evidence.as_ref().is_none_or(|value| { + serde_json::from_value::(value.clone()) + .is_ok_and(|evidence| evidence.valid()) + }) + } +} + +#[cfg(windows)] +fn verify_windows_boot(identity: &Identity, checkpoint: Option) -> Verification { + let saved = match checkpoint { + Some(Checkpoint::Windows(checkpoint)) => { + if checkpoint.kind != "recovery-boot" + || checkpoint.version != 2 + || checkpoint.platform != "win32" + { + return Verification::unknown("checkpoint_invalid"); + } + Some(checkpoint.boot_evidence) + } + Some(Checkpoint::Legacy(checkpoint)) => { + if checkpoint.kind != "recovery-boot" + || checkpoint.version != 1 + || checkpoint.platform != "win32" + || !valid_uuid(&checkpoint.boot_id) + { + return Verification::unknown("checkpoint_invalid"); + } + None } - if checkpoint.boot_id != current_boot { + None => None, + }; + let previous = saved.as_ref().or(identity.boot_evidence.as_ref()); + if let Some(previous) = previous { + let previous: windows_boot::Evidence = + match serde_json::from_value::(previous.clone()) { + Ok(evidence) if evidence.valid() => evidence, + _ => return Verification::unknown("checkpoint_invalid"), + }; + if windows_boot::capture().is_ok_and(|current| current.proves_restart_from(&previous)) { return Verification::new("released", "boot_changed"); } } backend::verify(identity) } -fn valid_uuid(value: &str) -> bool { +pub(super) fn valid_uuid(value: &str) -> bool { uuid::Uuid::parse_str(value) .is_ok_and(|id| !id.is_nil() && id.hyphenated().to_string() == value) } diff --git a/apps/buddy/native/host/src/supervised_shell/recovery/windows.rs b/apps/buddy/native/host/src/supervised_shell/recovery/windows.rs index 339c431d1..0701265b0 100644 --- a/apps/buddy/native/host/src/supervised_shell/recovery/windows.rs +++ b/apps/buddy/native/host/src/supervised_shell/recovery/windows.rs @@ -13,47 +13,6 @@ use crate::supervised_shell::windows::{boolean, owned, wide}; pub(super) const PLATFORM: &str = "win32"; -#[repr(C)] -#[derive(Default)] -struct BootEnvironment { - identifier: windows_sys::core::GUID, - firmware_type: u32, - flags: u64, -} - -#[link(name = "ntdll")] -unsafe extern "system" { - fn NtQuerySystemInformation( - class: u32, - information: *mut std::ffi::c_void, - length: u32, - returned: *mut u32, - ) -> i32; -} - -pub(crate) fn boot_id() -> io::Result { - let mut information = BootEnvironment::default(); - let mut returned = 0; - // SAFETY: SystemBootEnvironmentInformation (90) writes this fixed, correctly aligned native structure. - let status = unsafe { - NtQuerySystemInformation( - 90, - ptr::from_mut(&mut information).cast(), - size_of::() as u32, - &mut returned, - ) - }; - if status < 0 || returned != size_of::() as u32 { - return Err(io::ErrorKind::Unsupported.into()); - } - let guid = information.identifier; - let id = uuid::Uuid::from_fields(guid.data1, guid.data2, guid.data3, &guid.data4); - if id.is_nil() { - return Err(io::ErrorKind::InvalidData.into()); - } - Ok(id.hyphenated().to_string()) -} - pub(crate) fn start_time(process: HANDLE) -> io::Result { let mut creation = FILETIME::default(); let mut exit = FILETIME::default(); @@ -82,7 +41,7 @@ pub(super) fn verify(identity: &Identity) -> Verification { if supervisor_active { return Verification::new("active", "supervisor_active"); } - if identity.version != Some(2) { + if !matches!(identity.version, Some(2 | 3)) { return Verification::unknown("legacy_identity"); } let Some(name) = identity.job_name.as_deref().filter(|name| { @@ -134,7 +93,7 @@ pub(super) fn verify(identity: &Identity) -> Verification { return query_error(&error); } if accounting.ActiveProcesses == 0 { - Verification::new("released", "job_empty") + Verification::unknown("job_boundary_unverified") } else { Verification::new("active", "job_active") } diff --git a/apps/buddy/native/host/src/supervised_shell/recovery/windows_boot.rs b/apps/buddy/native/host/src/supervised_shell/recovery/windows_boot.rs new file mode 100644 index 000000000..8696ec067 --- /dev/null +++ b/apps/buddy/native/host/src/supervised_shell/recovery/windows_boot.rs @@ -0,0 +1,102 @@ +use std::io; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Evidence { + source: String, + version: u32, + architecture: String, + build: u32, + sequence: u32, + uptime_millis: String, +} + +impl Evidence { + pub(super) fn valid(&self) -> bool { + self.source == "windows-boot-sequence" + && self.version == 1 + && self.architecture == "x64" + && supported_build(self.build) + && self.sequence != 0 + && self.uptime_millis.parse::().is_ok() + } + + pub(super) fn proves_restart_from(&self, previous: &Self) -> bool { + self.valid() + && previous.valid() + && self.architecture == previous.architecture + && self.build == previous.build + && self.sequence != previous.sequence + && self.uptime_millis.parse::().ok() < previous.uptime_millis.parse::().ok() + } +} + +fn supported_build(build: u32) -> bool { + build == 26100 +} + +#[cfg(all(windows, target_arch = "x86_64"))] +pub(crate) fn capture() -> io::Result { + use windows_sys::Win32::System::{ + Diagnostics::Debug::ReadProcessMemory, SystemInformation::GetTickCount64, + Threading::GetCurrentProcess, + }; + + fn read() -> io::Result<(u32, u32)> { + let mut bytes = [0u8; 0x2c8]; + let mut copied = 0; + // SAFETY: ReadProcessMemory validates the fixed x64 shared-page mapping. No untrusted pointer is dereferenced. + let result = unsafe { + ReadProcessMemory( + GetCurrentProcess(), + 0x7ffe0000usize as *const _, + bytes.as_mut_ptr().cast(), + bytes.len(), + &mut copied, + ) + }; + if result == 0 || copied != bytes.len() { + return Err(io::Error::last_os_error()); + } + decode(&bytes) + } + + let (build, sequence) = read()?; + // SAFETY: This public query returns kernel uptime, not adjustable wall-clock time. + let uptime_millis = unsafe { GetTickCount64() }.to_string(); + if read()? != (build, sequence) { + return Err(io::ErrorKind::InvalidData.into()); + } + Ok(Evidence { + source: "windows-boot-sequence".into(), + version: 1, + architecture: "x64".into(), + build, + sequence, + uptime_millis, + }) +} + +#[cfg(all(windows, not(target_arch = "x86_64")))] +pub(crate) fn capture() -> io::Result { + Err(io::ErrorKind::Unsupported.into()) +} + +#[cfg(any(target_arch = "x86_64", test))] +fn decode(bytes: &[u8; 0x2c8]) -> io::Result<(u32, u32)> { + let word = |offset: usize| { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap_or_default()) + }; + let build = word(0x260); + let sequence = word(0x2c4); + if word(0x26c) != 10 || word(0x270) != 0 || !supported_build(build) || sequence == 0 { + return Err(io::ErrorKind::Unsupported.into()); + } + Ok((build, sequence)) +} + +#[cfg(test)] +#[path = "../../../__tests__/windows_boot.rs"] +mod tests; diff --git a/apps/buddy/native/host/src/supervised_shell/windows.rs b/apps/buddy/native/host/src/supervised_shell/windows.rs index d33e3dff5..cfc64d4ec 100644 --- a/apps/buddy/native/host/src/supervised_shell/windows.rs +++ b/apps/buddy/native/host/src/supervised_shell/windows.rs @@ -28,18 +28,30 @@ pub(super) fn initialize() -> io::Result { } pub(crate) fn identity(job: &Job) -> io::Result { - use super::recovery::backend::{boot_id, start_time}; + use super::recovery::backend::start_time; // SAFETY: This pseudo handle denotes the live supervisor. let start = start_time(unsafe { GetCurrentProcess() })?; - Ok(serde_json::json!({ + let mut identity = serde_json::json!({ "version":2, "pid":std::process::id(), "startTime":start.to_string(), - "bootId":boot_id()?, "jobName":job.name() - })) + "jobName":job.name() + }); + if let Ok(boot) = super::recovery::windows_boot::capture() { + identity["bootEvidence"] = serde_json::to_value(boot)?; + } + Ok(identity) } -pub(super) fn run(job: &Job, request: &Request, cancelled: &AtomicBool) -> io::Result { +pub(super) fn run( + job: &Job, + request: &Request, + cancelled: &AtomicBool, + receipt: Option<&super::receipt::Receipt>, +) -> io::Result { let result = execute(request, cancelled, job); job.finish()?; + if let Some(receipt) = receipt { + receipt.release()?; + } super::released(); result } diff --git a/apps/buddy/package.json b/apps/buddy/package.json index 3e26293d5..faf238b7a 100644 --- a/apps/buddy/package.json +++ b/apps/buddy/package.json @@ -49,7 +49,7 @@ "test:watch": "vitest" }, "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.79", + "@anthropic-ai/sandbox-runtime": "0.0.80", "@aws-sdk/client-s3": "3.1151.0", "@aws-sdk/util-format-url": "3.972.50", "@earendil-works/pi-ai": "1.3.0", diff --git a/apps/buddy/platform/process/__tests__/ownedProcessRecovery.spec.ts b/apps/buddy/platform/process/__tests__/ownedProcessRecovery.spec.ts index c267fcbb8..e3138ddc8 100644 --- a/apps/buddy/platform/process/__tests__/ownedProcessRecovery.spec.ts +++ b/apps/buddy/platform/process/__tests__/ownedProcessRecovery.spec.ts @@ -21,7 +21,7 @@ describe.skipIf(!['linux', 'win32'].includes(process.platform))('owned process r it.each([ { status: 'released', reason: 'boot_changed' }, - { status: 'released', reason: 'job_empty' }, + { status: 'released', reason: 'supervisor_receipt' }, { status: 'active', reason: 'supervisor_active' }, { status: 'active', reason: 'job_active' }, { status: 'unknown', reason: 'access_denied' }, diff --git a/apps/buddy/platform/process/__tests__/supervisedShellLaunch.spec.ts b/apps/buddy/platform/process/__tests__/supervisedShellLaunch.spec.ts new file mode 100644 index 000000000..d246e29f4 --- /dev/null +++ b/apps/buddy/platform/process/__tests__/supervisedShellLaunch.spec.ts @@ -0,0 +1,111 @@ +import type { OwnedProcessSettlement } from '../supervisedShellProcess' +import { access, mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { deferred } from '@buddy-tests/deferred' +import { describe, expect, it } from 'vitest' +import { verifyOwnedProcessRelease } from '../ownedProcessRecovery' +import { runSupervisedShellProcess } from '../supervisedShellProcess' + +describe.skipIf(process.platform !== 'linux')('durable native launch gate', () => { + async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'lexora-launch-')) + const receiptDirectory = join(root, 'receipts') + const records: OwnedProcessSettlement[] = [] + return { + root, + records, + receiptDirectory, + run(onSettlement: (record: OwnedProcessSettlement) => void | Promise, signal?: AbortSignal) { + return runSupervisedShellProcess({ + shell: '/bin/bash', + args: ['--noprofile', '--norc', '-c'], + command: 'printf executed > marker', + cwd: root, + environment: { PATH: '/usr/bin:/bin' }, + }, { + receiptDirectory, + signal, + onData() {}, + async onSettlement(record) { + records.push(record) + await onSettlement(record) + }, + }) + }, + dispose: () => rm(root, { recursive: true, force: true }), + } + } + + it('does not start until asynchronous identity persistence has completed', async () => { + const f = await fixture() + const ready = deferred() + const persisted = deferred() + try { + const result = f.run(async (record) => { + if (record.launchState === 'authorized') { + ready.resolve() + await persisted.promise + } + }) + await ready.promise + await expect(access(join(f.root, 'marker'))).rejects.toThrow() + persisted.resolve() + await expect(result).resolves.toEqual({ exitCode: 0 }) + expect(await readFile(join(f.root, 'marker'), 'utf8')).toBe('executed') + const identity = f.records.find(record => record.launchState === 'authorized')!.processId + await expect(verifyOwnedProcessRelease(identity, { receiptDirectory: f.receiptDirectory })) + .resolves + .toEqual({ status: 'released', reason: 'supervisor_receipt' }) + } + finally { + persisted.resolve() + await f.dispose() + } + }) + + it('ignores late persistence ACK after cancellation and retains a verifiable sealed gate', async () => { + const f = await fixture() + const ready = deferred() + const persisted = deferred() + const controller = new AbortController() + try { + const result = f.run(async (record) => { + if (record.launchState === 'authorized') { + ready.resolve() + await persisted.promise + } + }, controller.signal) + const rejected = expect(result).rejects.toThrow() + await ready.promise + controller.abort() + persisted.resolve() + await rejected + await expect(access(join(f.root, 'marker'))).rejects.toThrow() + const dispatch = f.records[0]!.processId + await expect(verifyOwnedProcessRelease(dispatch, { receiptDirectory: f.receiptDirectory })) + .resolves + .toEqual({ status: 'released', reason: 'supervisor_receipt' }) + } + finally { + persisted.resolve() + await f.dispose() + } + }) + + it('can import the native receipt after final persistence fails', async () => { + const f = await fixture() + try { + await expect(f.run((record) => { + if (record.launchState === 'settled') + throw new Error('persistence unavailable') + })).rejects.toThrow('persistence unavailable') + const dispatch = f.records[0]!.processId + await expect(verifyOwnedProcessRelease(dispatch, { receiptDirectory: f.receiptDirectory })) + .resolves + .toEqual({ status: 'released', reason: 'supervisor_receipt' }) + } + finally { await f.dispose() } + }) +}) diff --git a/apps/buddy/platform/process/ownedProcessRecovery.ts b/apps/buddy/platform/process/ownedProcessRecovery.ts index 9cc2e2d3f..f3ded0f7b 100644 --- a/apps/buddy/platform/process/ownedProcessRecovery.ts +++ b/apps/buddy/platform/process/ownedProcessRecovery.ts @@ -12,12 +12,31 @@ const supervisorIdentityFields = { startTime: z.string().regex(/^\d+$/).max(20).pipe(z.string().refine(value => BigInt(value) <= 0xFFFF_FFFF_FFFF_FFFFn)), } +const windowsBootEvidenceSchema = z.object({ + source: z.literal('windows-boot-sequence'), + version: z.literal(1), + architecture: z.literal('x64'), + build: z.number().int().positive(), + sequence: z.number().int().positive().max(0xFFFF_FFFF), + uptimeMillis: z.string().regex(/^\d+$/).max(20), +}).strict() + export const supervisedProcessIdentitySchema = z.union([ + z.object({ + ...supervisorIdentityFields, + version: z.literal(3), + bootEvidence: windowsBootEvidenceSchema.optional(), + bootId: kernelUuidSchema.optional(), + executionId: z.uuid(), + dispatchId: z.uuid(), + jobName: z.string().startsWith('Global\\Lexora.Buddy.Shell.v2.').optional(), + }).strict(), z.object({ ...supervisorIdentityFields, bootId: kernelUuidSchema.optional() }).strict(), z.object({ ...supervisorIdentityFields, version: z.literal(2), - bootId: kernelUuidSchema, + bootEvidence: windowsBootEvidenceSchema.optional(), + bootId: kernelUuidSchema.optional(), jobName: z.string().startsWith('Global\\Lexora.Buddy.Shell.v2.').refine(value => kernelUuidSchema.safeParse(value.slice('Global\\Lexora.Buddy.Shell.v2.'.length)).success).optional(), }).strict(), ]) @@ -44,26 +63,35 @@ const unknownReasons = [ 'helper_unavailable', 'helper_failed', 'invalid_response', + 'receipt_missing', + 'receipt_invalid', + 'receipt_root_unavailable', ] as const const verificationSchema = z.discriminatedUnion('status', [ - z.object({ protocol: z.literal(1), status: z.literal('released'), reason: z.enum(['boot_changed', 'job_empty']) }).strict(), + z.object({ protocol: z.literal(1), status: z.literal('released'), reason: z.enum(['boot_changed', 'supervisor_receipt']) }).strict(), z.object({ protocol: z.literal(1), status: z.literal('active'), reason: z.enum(['supervisor_active', 'job_active']) }).strict(), z.object({ protocol: z.literal(1), status: z.literal('unknown'), reason: z.enum(unknownReasons) }).strict(), ]) -const checkpointSchema = z.object({ +const checkpointSchema = z.union([z.object({ kind: z.literal('recovery-boot'), version: z.literal(1), platform: z.enum([OPERATING_SYSTEM.Linux, OPERATING_SYSTEM.Windows]), bootId: kernelUuidSchema, -}).strict() +}).strict(), z.object({ + kind: z.literal('recovery-boot'), + version: z.literal(2), + platform: z.literal(OPERATING_SYSTEM.Windows), + bootEvidence: windowsBootEvidenceSchema, +}).strict()]) export type OwnedProcessVerification = Omit, 'protocol'> interface RecoveryOptions { signal?: AbortSignal executable?: string + receiptDirectory?: string } function supportedPlatform() { @@ -105,7 +133,7 @@ export async function captureProcessRecoveryCheckpoint(options: RecoveryOptions } } -export async function reverifyOwnedProcess( +export async function verifyOwnedProcessRelease( processId: string, options: RecoveryOptions & { checkpoint?: string } = {}, ): Promise { @@ -129,7 +157,7 @@ export async function reverifyOwnedProcess( } } try { - const result = await query('--reverify-owned-process', { identity, checkpoint }, options) + const result = await query('--reverify-owned-process', { identity, checkpoint, receiptDirectory: options.receiptDirectory }, options) if (!result) return { status: 'unknown', reason: 'helper_unavailable' } if (result.code !== 0) @@ -144,3 +172,5 @@ export async function reverifyOwnedProcess( return { status: 'unknown', reason: 'helper_failed' } } } + +export const reverifyOwnedProcess = verifyOwnedProcessRelease diff --git a/apps/buddy/platform/process/supervisedShellProcess.ts b/apps/buddy/platform/process/supervisedShellProcess.ts index 875a5ec72..12bbe552d 100644 --- a/apps/buddy/platform/process/supervisedShellProcess.ts +++ b/apps/buddy/platform/process/supervisedShellProcess.ts @@ -1,5 +1,4 @@ import type { Buffer } from 'node:buffer' -import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { spawn } from 'node:child_process' import { randomUUID } from 'node:crypto' import process from 'node:process' @@ -10,10 +9,20 @@ import { supervisedProcessIdentitySchema } from './ownedProcessRecovery' export interface OwnedProcessSettlement { processId: string settlement: 'pending' | 'released' | 'unknown' + executionId?: string + dispatchId?: string + launchState?: 'prepared' | 'authorized' | 'settled' +} + +export interface OwnedProcessSupervision { + executionId?: string + dispatchId?: string + receiptDirectory?: string + onSettlement: (settlement: OwnedProcessSettlement) => void | Promise } const diagnosticSchema = z.discriminatedUnion('type', [ - z.object({ type: z.literal('ready'), protocol: z.literal(1), identity: supervisedProcessIdentitySchema }).strict(), + z.object({ type: z.literal('ready'), protocol: z.literal(2), identity: supervisedProcessIdentitySchema }).strict(), z.object({ type: z.literal('released'), protocol: z.literal(1) }).strict(), z.object({ type: z.literal('failed'), code: z.literal('SHELL_SUPERVISION_FAILED'), nativeCode: z.number().nullable() }).strict(), ]) @@ -25,11 +34,10 @@ export async function runSupervisedShellProcess(input: { commandTransport?: 'argv' | 'stdin' cwd: string environment: NodeJS.ProcessEnv -}, options: { +}, options: OwnedProcessSupervision & { signal?: AbortSignal timeout?: number onData: (data: Buffer) => void - onSettlement: (settlement: OwnedProcessSettlement) => void onProcess?: (state: { phase: 'exited' | 'closed', pid?: number, exitCode: number | null }) => void executable?: string }): Promise<{ exitCode: number | null }> { @@ -39,63 +47,66 @@ export async function runSupervisedShellProcess(input: { throw new Error('SHELL_SUPERVISION_UNAVAILABLE') if (options.timeout !== undefined && (!Number.isFinite(options.timeout) || options.timeout <= 0 || options.timeout * 1000 > 2_147_483_647)) throw new Error('Invalid shell timeout') + const executionId = z.uuid().parse(options.executionId ?? randomUUID()) + const dispatchId = z.uuid().parse(options.dispatchId ?? randomUUID()) + const dispatchProcessId = JSON.stringify({ kind: 'native-shell-dispatch', platform: process.platform, version: 3, executionId, dispatchId }) + const settle = (processId: string, settlement: OwnedProcessSettlement['settlement'], launchState: OwnedProcessSettlement['launchState']) => + options.onSettlement({ processId, settlement, executionId, dispatchId, launchState }) + await settle(dispatchProcessId, 'pending', 'prepared') + options.signal?.throwIfAborted() return new Promise((resolve, reject) => { - const nonce = randomUUID() - const dispatchId = JSON.stringify({ kind: 'native-shell-dispatch', platform: process.platform, nonce }) - options.onSettlement({ processId: dispatchId, settlement: 'pending' }) - let child: ChildProcessWithoutNullStreams - try { - child = spawn(executable, ['--supervise-shell'], { cwd: input.cwd, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) - } - catch (error) { - options.onSettlement({ processId: dispatchId, settlement: 'released' }) - throw error - } + const child = spawn(executable, ['--supervise-shell', JSON.stringify({ executionId, dispatchId, receiptDirectory: options.receiptDirectory })], { cwd: input.cwd, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) let commandDispatched = false - let processId = JSON.stringify({ kind: 'native-shell', platform: process.platform, pid: child.pid, nonce, startTime: null }) - let ready = false + let processId: string | undefined let released = false let failed = false let timedOut = false + let closed = false + let cancelled = false let launchError: Error | undefined let forceKill: ReturnType | undefined let timeout: ReturnType | undefined let exited = false + let protocol = Promise.resolve() const closeUnknownOutput = () => { if (exited && child.stderr.readableEnded && !released) child.stdout.destroy() } const cancel = () => { + cancelled = true child.stdin.end() - forceKill ??= setTimeout(() => child.kill(), 10_000) - forceKill.unref() + if (!closed) { + forceKill ??= setTimeout(() => child.kill(), 10_000) + forceKill.unref() + } } const lines = createInterface({ input: child.stderr }) lines.on('line', (line) => { - try { + protocol = protocol.then(async () => { const diagnostic = diagnosticSchema.parse(JSON.parse(line)) if (diagnostic.type === 'ready') { - if (ready || diagnostic.identity.pid !== child.pid) + if (processId || diagnostic.identity.pid !== child.pid + || !('executionId' in diagnostic.identity) || diagnostic.identity.executionId !== executionId || diagnostic.identity.dispatchId !== dispatchId) { throw new Error('SHELL_SUPERVISION_INVALID_IDENTITY') - ready = true - processId = JSON.stringify({ kind: 'native-shell', platform: process.platform, ...diagnostic.identity, nonce }) - options.onSettlement({ processId, settlement: 'pending' }) - if (!options.signal?.aborted && !timedOut) { - commandDispatched = true - child.stdin.write(`${JSON.stringify({ ...input, commandTransport: input.commandTransport ?? 'argv', environment: Object.fromEntries(Object.entries(input.environment).filter((entry): entry is [string, string] => entry[1] !== undefined)) })}\n`) } - else { cancel() } + processId = JSON.stringify({ kind: 'native-shell', platform: process.platform, ...diagnostic.identity }) + if (closed || cancelled || options.signal?.aborted || timedOut) + return cancel() + await settle(processId, 'pending', 'authorized') + if (closed || cancelled || options.signal?.aborted || timedOut) + return cancel() + commandDispatched = true + child.stdin.write(`${JSON.stringify({ ...input, executionId, dispatchId, commandTransport: input.commandTransport ?? 'argv', environment: Object.fromEntries(Object.entries(input.environment).filter((entry): entry is [string, string] => entry[1] !== undefined)) })}\n`) } if (diagnostic.type === 'released') released = true if (diagnostic.type === 'failed') failed = true - } - catch (error) { + }).catch((error) => { launchError = error instanceof Error ? error : new Error('SHELL_SUPERVISION_FAILED') failed = true cancel() - } + }) }) child.stdout.on('data', options.onData) child.stdin.on('error', () => {}) @@ -109,27 +120,28 @@ export async function runSupervisedShellProcess(input: { closeUnknownOutput() }) child.once('close', (exitCode) => { - options.onProcess?.({ phase: 'closed', pid: child.pid, exitCode }) + closed = true options.signal?.removeEventListener('abort', cancel) clearTimeout(timeout) clearTimeout(forceKill) lines.close() - try { + void (async () => { + await protocol + options.onProcess?.({ phase: 'closed', pid: child.pid, exitCode }) const settlement = released || !commandDispatched ? 'released' : 'unknown' - if (ready) - options.onSettlement({ processId, settlement }) - options.onSettlement({ processId: dispatchId, settlement }) + if (processId) + await settle(processId, settlement, 'settled') + await settle(dispatchProcessId, settlement, 'settled') if (options.signal?.aborted) - reject(options.signal.reason ?? new Error('aborted')) - else if (timedOut) - reject(new Error(`timeout:${options.timeout}`)) - else if (launchError) - reject(launchError) - else if (!ready || failed || !released) - reject(new Error('SHELL_SUPERVISION_FAILED')) - else resolve({ exitCode }) - } - catch (error) { reject(error) } + throw options.signal.reason ?? new Error('aborted') + if (timedOut) + throw new Error(`timeout:${options.timeout}`) + if (launchError) + throw launchError + if (!processId || failed || !released) + throw new Error('SHELL_SUPERVISION_FAILED') + return { exitCode } + })().then(resolve, reject) }) options.signal?.addEventListener('abort', cancel, { once: true }) if (options.timeout !== undefined) { diff --git a/apps/buddy/platform/process/windowsSandboxProcess.ts b/apps/buddy/platform/process/windowsSandboxProcess.ts index 59ce18381..c14ce4097 100644 --- a/apps/buddy/platform/process/windowsSandboxProcess.ts +++ b/apps/buddy/platform/process/windowsSandboxProcess.ts @@ -1,5 +1,6 @@ import type { Buffer } from 'node:buffer' import type { SandboxProcessState } from '../../shared/permissions/sandboxLifecycle' +import type { OwnedProcessSupervision } from './supervisedShellProcess' import { spawn } from 'node:child_process' import { randomUUID } from 'node:crypto' import process from 'node:process' @@ -36,7 +37,7 @@ export async function runWindowsSandboxProcess(executable: string, request: Wind onData: (data: Buffer) => void onStarted: () => void onProcess?: (process: SandboxProcessState) => void -}): Promise { +} & Partial): Promise { return runProcess(executable, 'run', request, options) } @@ -51,10 +52,23 @@ async function runProcess(executable: string, mode: 'run' | 'prepare', request: onData: (data: Buffer) => void onStarted: () => void onProcess?: (process: SandboxProcessState) => void -}): Promise { +} & Partial): Promise { + options.signal.throwIfAborted() + const executionId = options.executionId ?? randomUUID() + const dispatchId = options.dispatchId ?? randomUUID() + const dispatchProcessId = JSON.stringify({ kind: 'native-shell-dispatch', platform: 'win32', version: 3, executionId, dispatchId }) + const settle = async (processId: string, settlement: 'pending' | 'released' | 'unknown', launchState: 'prepared' | 'authorized' | 'settled') => { + await options.onSettlement?.({ processId, settlement, launchState, executionId, dispatchId }) + } + if (mode === 'run') + await settle(dispatchProcessId, 'pending', 'prepared') options.signal.throwIfAborted() return new Promise((resolve, reject) => { let ready = false + let closed = false + let cancelled = false + let authorized = false + let protocol = Promise.resolve() let failed = false let processesReleased = false let processId: string | undefined @@ -74,20 +88,34 @@ async function runProcess(executable: string, mode: 'run' | 'prepare', request: closeUnknownOutput() }) const cancel = () => { + cancelled = true options.onProcess?.({ phase: 'cleanup_requested', pid: child.pid }) child.stdin.end() - forceKill ??= setTimeout(() => child.kill(), 10_000) - forceKill.unref() + if (!closed) { + forceKill ??= setTimeout(() => child.kill(), 10_000) + forceKill.unref() + } } const lines = createInterface({ input: child.stderr }) lines.on('line', (line) => { - try { + protocol = protocol.then(async () => { const diagnostic = diagnosticSchema.parse(JSON.parse(line)) if (diagnostic.type === 'supervisor') { if (processId || diagnostic.identity.pid !== child.pid) throw new Error('Invalid sandbox supervisor identity') - processId = JSON.stringify({ kind: 'sandbox-job', platform: 'win32', ...diagnostic.identity, nonce: randomUUID() }) + processId = JSON.stringify({ kind: 'sandbox-job', platform: 'win32', ...diagnostic.identity, ...mode === 'prepare' ? { nonce: randomUUID() } : {} }) options.onProcess?.({ phase: 'spawned', pid: child.pid, processId }) + if (mode === 'run') { + if (!('executionId' in diagnostic.identity) || diagnostic.identity.executionId !== executionId || diagnostic.identity.dispatchId !== dispatchId) + throw new Error('Invalid sandbox launch identity') + if (closed || cancelled || options.signal.aborted) + return cancel() + await settle(processId, 'pending', 'authorized') + if (closed || cancelled || options.signal.aborted) + return cancel() + authorized = true + child.stdin.write(`${JSON.stringify({ executionId, dispatchId })}\n`) + } } if (diagnostic.type === 'ready' && !ready) { ready = true @@ -100,11 +128,10 @@ async function runProcess(executable: string, mode: 'run' | 'prepare', request: failed = true if (diagnostic.type === 'processesReleased') processesReleased = true - } - catch { + }).catch(() => { failed = true cancel() - } + }) }) child.stdout.on('data', options.onData) child.stdin.on('error', () => {}) @@ -112,19 +139,29 @@ async function runProcess(executable: string, mode: 'run' | 'prepare', request: launchError = error }) child.once('close', (code) => { - options.onProcess?.({ phase: 'closed', pid: child.pid, exitCode: code }) - options.onProcess?.({ phase: processesReleased ? 'resources_released' : 'cleanup_failed', pid: child.pid, processId }) - options.signal.removeEventListener('abort', cancel) - clearTimeout(forceKill) - lines.close() - if (launchError) - reject(launchError) - else if (!ready || failed) - reject(new ShellSandboxError(ready ? 'SANDBOX_FAILED' : 'SANDBOX_UNAVAILABLE')) - else resolve(code) + closed = true + void (async () => { + await protocol + options.onProcess?.({ phase: 'closed', pid: child.pid, exitCode: code }) + options.onProcess?.({ phase: processesReleased ? 'resources_released' : 'cleanup_failed', pid: child.pid, processId }) + options.signal.removeEventListener('abort', cancel) + clearTimeout(forceKill) + lines.close() + if (mode === 'run') { + const state = processesReleased || !authorized ? 'released' : 'unknown' + if (processId) + await settle(processId, state, 'settled') + await settle(dispatchProcessId, state, 'settled') + } + if (launchError) + throw launchError + if (!ready || failed) + throw new ShellSandboxError(ready ? 'SANDBOX_FAILED' : 'SANDBOX_UNAVAILABLE') + return code + })().then(resolve, reject) }) options.signal.addEventListener('abort', cancel, { once: true }) - child.stdin.write(`${JSON.stringify(request)}\n`) + child.stdin.write(`${JSON.stringify(mode === 'run' ? { ...request, launch: { executionId, dispatchId, receiptDirectory: options.receiptDirectory } } : request)}\n`) if (options.signal.aborted) cancel() }) diff --git a/apps/buddy/service/src/BuddyService.ts b/apps/buddy/service/src/BuddyService.ts index ea9e757c0..b2bc3deb9 100644 --- a/apps/buddy/service/src/BuddyService.ts +++ b/apps/buddy/service/src/BuddyService.ts @@ -9,6 +9,7 @@ import type { BuddyRuntime } from './BuddyRuntime' import type { RunEventLogPort } from './events/RunEventPorts' import type { BuddyServiceRpcServer } from './rpc/BuddyServiceRpcServer' import type { BuddyServiceErrorCode } from './rpc/runtimeRequest' +import { randomUUID } from 'node:crypto' import { mkdir } from 'node:fs/promises' import { join } from 'node:path' import process from 'node:process' @@ -86,6 +87,7 @@ import { registerBuddiesRpc } from './buddies/registerBuddiesRpc' import { ChangeCaptureService } from './changes/ChangeCaptureService' import { createChangeSetRepository } from './changes/changeSetRepository' import { MutationCoordinator } from './changes/MutationCoordinator' +import { createMutationEvidenceRepository } from './changes/mutationEvidenceRepository' import { MutationRecoveryService } from './changes/MutationRecoveryService' import { createMutationRepository } from './changes/mutationRepository' import { registerChangeRpc } from './changes/registerChangeRpc' @@ -423,15 +425,25 @@ export async function startBuddyService( }) return service }) + const mutationEvidence = createMutationEvidenceRepository(options.database) + const recoverySourceId = randomUUID() + let recoveryRevision = 0 + const notifyRecovery = (runId: string) => { + const run = runs.findById(runId) + if (run) + options.rpc.notify(changesChanged.method, { sourceId: recoverySourceId, revision: ++recoveryRevision, conversationId: run.conversationId, runId }) + } const mutations = await host.start('runtime.mutations', async ({ defer }) => { - const service = new MutationCoordinator({ repository: createMutationRepository(options.database), resources: fileResources }) + const service = new MutationCoordinator({ repository: createMutationRepository(options.database), evidence: mutationEvidence, resources: fileResources }) + const notices = service.onDidChange(event => notifyRecovery(event.runId)) + defer(() => notices.dispose()) defer(() => service.dispose()) await service.initialize() return service }, ['runtime.file_engine']) + const mutationRecovery = new MutationRecoveryService({ mutations, record, evidence: mutationEvidence, receiptDirectory: join(options.buddyHome, 'process-receipts'), context: { conversations, runs }, onChange: notifyRecovery }) await host.step('runtime.mutation_recovery', async () => { - const recovery = new MutationRecoveryService({ mutations, record }) - await recovery.recheck({ signal: AbortSignal.timeout(5_000) }) + await mutationRecovery.recheck({ signal: AbortSignal.timeout(5_000) }) }) const changeCaptureService = await host.start('runtime.change_capture', ({ defer }) => { const service = new ChangeCaptureService({ paths, repository: createChangeSetRepository(options.database), onListenerError: () => record({ event: 'observer.failed', component: 'runtime.change_capture', level: 'warn' }) }) @@ -894,6 +906,7 @@ export async function startBuddyService( attachmentService, changeCaptureService, mutations, + receiptDirectory: join(options.buddyHome, 'process-receipts'), directoryGrants, recordPermissions: record, createCapabilities: createBuddyCapabilityFactory(currentPlatform, { @@ -1411,6 +1424,7 @@ export async function startBuddyService( runs, rpc: options.rpc, service: changeCaptureService, + recovery: mutationRecovery, }), ) register( diff --git a/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts b/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts index 2c64d545b..42a3a0640 100644 --- a/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts +++ b/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts @@ -52,6 +52,7 @@ export interface BuddySessionExtensionServices { changeCaptureService: Pick fileResources: FileResourceCoordinator mutations: MutationCoordinator + receiptDirectory?: string createCapabilities: BuddyCapabilityFactory directoryGrants: Pick recordPermissions?: ApplicationDiagnosticReporter @@ -178,6 +179,7 @@ export async function createBuddySessionExtensions( sandbox: services.shellSandbox, directoryPermissions: sandboxDirectories, mutations: services.mutations, + receiptDirectory: services.receiptDirectory, assertExecutionAllowed: ({ run }) => services.delegation?.assertShellExecutionAllowed(run.runId), })) if (services.delegation) { diff --git a/apps/buddy/service/src/changes/MutationCoordinator.ts b/apps/buddy/service/src/changes/MutationCoordinator.ts index 333f77323..b1210490c 100644 --- a/apps/buddy/service/src/changes/MutationCoordinator.ts +++ b/apps/buddy/service/src/changes/MutationCoordinator.ts @@ -1,5 +1,6 @@ import type { FileResourceCoordinator } from '../files/FileResourceCoordinator' -import type { MutationRepository, OwnedProcessSettlement } from './mutationRepository' +import type { MutationEvidenceRepository } from './mutationEvidenceRepository' +import type { MutationExecutionUpdate, MutationRepository, OwnedProcessSettlement } from './mutationRepository' import { AsyncLocalStorage } from 'node:async_hooks' import { randomUUID } from 'node:crypto' import { containsCanonicalPath, filePaths } from '../../../platform/filesystem/filePaths' @@ -50,7 +51,7 @@ export class MutationCoordinatorError extends Error { readonly code: 'MUTATION_RESOURCE_UNSAFE' | 'MUTATION_COORDINATOR_STOPPED' | 'MUTATION_INVOCATION_CONFLICT' constructor(code: MutationCoordinatorError['code']) { - super(code === 'MUTATION_RESOURCE_UNSAFE' ? `${code}: An earlier operation has an unverified outcome. Conflicting writes remain blocked. Check application logs for details.` : code) + super(code === 'MUTATION_RESOURCE_UNSAFE' ? `${code}: An earlier operation has an unverified outcome. Conflicting writes remain blocked. Review the recovery options in the task.` : code) this.code = code this.name = 'MutationCoordinatorError' } @@ -66,25 +67,29 @@ export class MutationCoordinator { readonly #resources: Pick | undefined readonly #settlements = new Map>() readonly #repository: MutationRepository | undefined + readonly #evidence: MutationEvidenceRepository | undefined + readonly #resolutions = new Map>() readonly #changes = new Emitter<{ runId: string }>(() => console.error('MUTATION_OBSERVER_FAILED')) readonly onDidChange = this.#changes.event #lastWriter: { runId: string, segmentId: string } | undefined #sequence = 0 #disposed = false - constructor(options: { repository?: MutationRepository, resources?: Pick } = {}) { + constructor(options: { repository?: MutationRepository, evidence?: MutationEvidenceRepository, resources?: Pick } = {}) { this.#resources = options.resources + this.#evidence = options.evidence const repository = options.repository this.#repository = repository if (!repository) return for (const stored of repository.listUnreleased()) { - if (stored.processes.length ? stored.processes.every(process => process.settlement === 'released') : stored.lifetime === 'runtime') { + const knownReleased = stored.processes.length ? stored.processes.every(process => process.settlement === 'released') : stored.lifetime === 'runtime' + if (knownReleased && !this.#evidence) { repository.settle(stored.invocationId, 'released', null) continue } this.#processes.set(stored.invocationId, new Map(stored.processes.map(process => [process.processId, process.settlement]))) - this.restoreHazard({ runId: stored.runId, invocationId: stored.invocationId, claims: stored.claims, reason: stored.reason ?? 'MUTATION_INTERRUPTED' }) + this.restoreHazard({ runId: stored.runId, invocationId: stored.invocationId, claims: stored.claims, reason: knownReleased && !stored.processes.length ? 'RUNTIME_OWNERSHIP_ENDED' : stored.reason ?? 'MUTATION_INTERRUPTED' }) } } @@ -135,6 +140,13 @@ export class MutationCoordinator { async initialize(): Promise { await this.#resources?.reconcile([...this.#hazards.values()], id => this.#repository?.isReleased(id) === true) + for (const hazard of this.#hazards.values()) { + const decision = this.#evidence?.decision(hazard.invocationId) + if (decision?.outcome === 'requested' && (decision.kind === 'waived' || decision.kind === 'quiesced')) + await this.#resolveProtection(hazard.invocationId, decision.kind).catch(() => {}) + else if (this.#canConfirmReleased(hazard.invocationId)) + await this.confirmReleased(hazard.invocationId).catch(() => {}) + } } getHazards(): readonly MutationHazard[] { @@ -146,8 +158,8 @@ export class MutationCoordinator { } async waitForSettlements(): Promise { - while (this.#settlements.size) - await Promise.allSettled(this.#settlements.values()) + while (this.#settlements.size || this.#resolutions.size) + await Promise.allSettled([...this.#settlements.values(), ...this.#resolutions.values()]) } getRunState(runId: string): MutationRunState { @@ -167,11 +179,14 @@ export class MutationCoordinator { return lease ? Object.freeze({ runId: lease.runId, invocationId: lease.invocationId }) : null } - recordOwnedProcessSettlement(input: { invocationId?: string, processId: string, settlement: 'pending' | 'released' | 'unknown' }): boolean { + recordOwnedProcessSettlement(input: { invocationId?: string, processId: string, settlement: 'pending' | 'released' | 'unknown' } & MutationExecutionUpdate): boolean { const invocationId = input.invocationId ?? this.#invocation.getStore()?.invocationId if (!invocationId || (!this.#active.has(invocationId) && !this.#hazards.has(invocationId))) return false - if (input.settlement === 'pending' && (this.#disposed || this.#closing.has(invocationId) || !this.#active.has(invocationId))) + const launching = input.launchState === 'prepared' || input.launchState === 'authorized' + if (launching && input.settlement !== 'pending') + return false + if ((input.settlement === 'pending' || launching) && (this.#disposed || this.#closing.has(invocationId) || !this.#active.has(invocationId))) return false return this.#recordProcess({ ...input, invocationId }) } @@ -182,13 +197,14 @@ export class MutationCoordinator { return this.#recordProcess({ invocationId, processId, settlement: 'pending' }) } - #recordProcess(input: { invocationId: string, processId: string, settlement: OwnedProcessSettlement }): boolean { + #recordProcess(input: { invocationId: string, processId: string, settlement: OwnedProcessSettlement } & MutationExecutionUpdate): boolean { const { invocationId } = input const processes = this.#processes.get(invocationId) ?? new Map() - if (processes.get(input.processId) === 'released') - return true - this.#repository?.recordProcess(invocationId, input.processId, input.settlement) - processes.set(input.processId, input.settlement) + const alreadyReleased = processes.get(input.processId) === 'released' + if (alreadyReleased && (!input.launchState || input.launchState === 'authorized')) + return input.launchState !== 'authorized' + this.#repository?.recordProcess(invocationId, input.processId, input.settlement, input) + processes.set(input.processId, alreadyReleased ? 'released' : input.settlement) this.#processes.set(invocationId, processes) if (this.#hazards.has(invocationId) && [...processes.values()].every(value => value === 'released')) void this.confirmReleased(invocationId).catch(() => {}) @@ -227,17 +243,54 @@ export class MutationCoordinator { this.#drain() } - async confirmReleased(invocationId: string): Promise { + confirmReleased(invocationId: string): Promise { + return this.#resolveProtection(invocationId, 'quiesced') + } + + waiveHazard(invocationId: string): Promise { + if (!this.#evidence) + return Promise.reject(new Error('MUTATION_RECOVERY_AUDIT_UNAVAILABLE')) + return this.#resolveProtection(invocationId, 'waived') + } + + #canConfirmReleased(invocationId: string): boolean { + const state = this.getInvocationSettlement(invocationId) + return state === 'released' || (state === 'none' && ['FILE_RESOURCE_SETTLEMENT_FAILED', 'RUNTIME_OWNERSHIP_ENDED'].includes(this.#hazards.get(invocationId)?.reason ?? '')) + } + + #resolveProtection(invocationId: string, kind: 'quiesced' | 'waived'): Promise { + const pending = this.#resolutions.get(invocationId) + if (pending) + return pending const hazard = this.#hazards.get(invocationId) if (!hazard) - return - if (this.#resources) - await this.#settleResource(invocationId, false) - this.#repository?.settle(invocationId, 'released', null) - this.#hazards.delete(invocationId) - this.#processes.delete(invocationId) - this.#publish(hazard.runId) - this.#drain() + return Promise.resolve() + if (kind === 'quiesced' && !this.#canConfirmReleased(invocationId)) + return Promise.reject(new Error('PROCESS_RELEASE_EVIDENCE_MISSING')) + const prior = this.#evidence?.decision(invocationId) + if (prior?.outcome !== 'requested' || prior.kind !== kind) + this.#evidence?.append({ invocationId, kind, outcome: 'requested', actor: kind === 'waived' ? 'user' : 'system', reason: kind === 'waived' ? 'USER_ACCEPTED_RISK' : 'PROCESS_RELEASE_CONFIRMED' }) + const finish = async () => { + if (this.#resources) + await this.#settleResource(invocationId, false) + if (kind === 'quiesced' && !this.#canConfirmReleased(invocationId)) { + await this.#resources?.reconcile([hazard], () => false) + throw new Error('PROCESS_RELEASE_EVIDENCE_MISSING') + } + if (this.#evidence) + this.#evidence.applyResolution(invocationId, kind) + else + this.#repository?.settle(invocationId, 'released', null) + this.#hazards.delete(invocationId) + this.#processes.delete(invocationId) + this.#publish(hazard.runId) + for (const runId of this.#evidence?.blockedRuns(invocationId) ?? []) + this.#publish(runId) + this.#drain() + } + const resolution = finish().finally(() => this.#resolutions.delete(invocationId)) + this.#resolutions.set(invocationId, resolution) + return resolution } async dispose(): Promise { @@ -253,10 +306,16 @@ export class MutationCoordinator { #drain(): void { for (let index = 0; index < this.#pending.length;) { const pending = this.#pending[index]! - if ([...this.#hazards.values()].some(hazard => claimsOverlap(hazard.claims, pending.claims))) { + const blockers = [...this.#hazards.values()].filter(hazard => claimsOverlap(hazard.claims, pending.claims)) + if (blockers.length) { this.#pending.splice(index, 1) pending.unsubscribe() - pending.reject(new MutationCoordinatorError('MUTATION_RESOURCE_UNSAFE')) + try { + for (const hazard of blockers) + this.#evidence?.recordBlocker(hazard.invocationId, pending.runId) + pending.reject(new MutationCoordinatorError('MUTATION_RESOURCE_UNSAFE')) + } + catch (error) { pending.reject(error) } this.#publish(pending.runId) continue } diff --git a/apps/buddy/service/src/changes/MutationRecoveryService.ts b/apps/buddy/service/src/changes/MutationRecoveryService.ts index 3087d314b..3aceea6df 100644 --- a/apps/buddy/service/src/changes/MutationRecoveryService.ts +++ b/apps/buddy/service/src/changes/MutationRecoveryService.ts @@ -1,9 +1,15 @@ +import type { ChangeRecoveryListRequest, ChangeRecoveryRecheckRequest, ChangeRecoveryResolveRequest, LocalChangeRecoveryItem } from '../../../shared/changes/changeApi' import type { ApplicationDiagnosticReporter } from '../../../shared/diagnostics/applicationDiagnostic' +import type { ConversationRepository } from '../storage/conversationRepository' +import type { RunRepository } from '../storage/runRepository' import type { MutationCoordinator, MutationHazard } from './MutationCoordinator' +import type { MutationEvidenceRepository } from './mutationEvidenceRepository' +import { createHash } from 'node:crypto' import process from 'node:process' import { z } from 'zod' import { captureProcessRecoveryCheckpoint, reverifyOwnedProcess, supervisedProcessIdentitySchema } from '../../../platform/process/ownedProcessRecovery' import { safeDiagnosticReporter } from '../../../shared/diagnostics/applicationDiagnostic' +import { BuddyServiceError } from '../rpc/runtimeRequest' const identitySchema = z.object({ kind: z.string(), @@ -18,31 +24,116 @@ export interface MutationRecoveryOptions { verify?: typeof reverifyOwnedProcess checkpoint?: typeof captureProcessRecoveryCheckpoint record?: ApplicationDiagnosticReporter + evidence?: MutationEvidenceRepository + receiptDirectory?: string + context?: { + conversations: Pick + runs: Pick + } + onChange?: (runId: string) => void } export class MutationRecoveryService { readonly #options: MutationRecoveryOptions readonly #record: ApplicationDiagnosticReporter + #tail: Promise = Promise.resolve() constructor(options: MutationRecoveryOptions) { this.#options = options this.#record = safeDiagnosticReporter(options.record) } - async recheck(input: { signal?: AbortSignal } = {}): Promise { + recheck(input: { signal?: AbortSignal, invocationIds?: readonly string[] } = {}): Promise { const signal = input.signal ?? AbortSignal.timeout(15_000) - for (const hazard of this.#options.mutations.getHazards()) { - if (signal.aborted) - break - try { - await this.#recheck(hazard, signal) - } - catch (error) { + return this.#serialize(async () => { + for (const hazard of this.#options.mutations.getHazards()) { if (signal.aborted) break - throw error + if (input.invocationIds && !input.invocationIds.includes(hazard.invocationId)) + continue + try { + await this.#recheck(hazard, signal) + } + catch { + if (signal.aborted) + break + this.#observe(hazard, 'unknown', 'RECOVERY_VERIFICATION_FAILED') + } } - } + }) + } + + list(input: ChangeRecoveryListRequest): LocalChangeRecoveryItem[] { + const context = this.#options.context + const conversation = context?.conversations.findById(input.conversationId) + if (!context || !conversation || conversation.deletedAt) + throw new BuddyServiceError('VALIDATION_FAILED') + const runIds = new Set(context.runs.listIdsForConversation(input.conversationId)) + return this.#options.mutations.getHazards().flatMap((hazard) => { + if (!runIds.has(hazard.runId) && !this.#options.evidence?.blockedRuns(hazard.invocationId).some(runId => runIds.has(runId))) + return [] + const run = context.runs.findById(hazard.runId) + const owner = run ? context.conversations.findById(run.conversationId) : null + const latest = this.#options.evidence?.latest(hazard.invocationId) + const status = latest?.outcome === 'active' ? 'active' as const : 'unknown' as const + const reason = latest?.reason ?? hazard.reason + const processes = [...this.#options.mutations.getOwnedProcesses(hazard.invocationId)].sort((a, b) => a.processId.localeCompare(b.processId)) + return [{ + invocationId: hazard.invocationId, + runId: hazard.runId, + conversationId: owner && !owner.deletedAt ? owner.id : null, + taskTitle: owner && !owner.deletedAt ? owner.title ?? '' : '', + occurredAt: this.#options.evidence?.createdAt(hazard.invocationId) ?? run?.startedAt ?? new Date().toISOString(), + affectedResources: hazard.claims.map(claim => claim.kind === 'host' ? 'host' : claim.canonicalPath), + reason, + status, + revision: createHash('sha256').update(JSON.stringify({ hazard, processes, status, reason })).digest('hex'), + }] + }) + } + + async recheckVisible(input: ChangeRecoveryRecheckRequest, signal?: AbortSignal): Promise { + const visible = new Set(this.list(input).map(item => item.invocationId)) + if (input.invocationIds.some(id => !visible.has(id))) + throw new BuddyServiceError('VALIDATION_FAILED') + await this.recheck({ signal, invocationIds: input.invocationIds }) + return this.list(input) + } + + resolve(input: ChangeRecoveryResolveRequest, signal?: AbortSignal): Promise { + return this.#serialize(async () => { + signal?.throwIfAborted() + const visible = new Map(this.list(input).map(item => [item.invocationId, item])) + if (input.items.some((item) => { + const current = visible.get(item.invocationId) + return !current || current.status !== 'unknown' || current.revision !== item.revision + })) { + throw new BuddyServiceError('VALIDATION_FAILED') + } + if (!this.#options.evidence) + throw new BuddyServiceError('RUNTIME_UNAVAILABLE') + for (const item of input.items) { + signal?.throwIfAborted() + const current = this.list(input).find(candidate => candidate.invocationId === item.invocationId) + if (!current || current.status !== 'unknown' || current.revision !== item.revision) + throw new BuddyServiceError('VALIDATION_FAILED') + await this.#options.mutations.waiveHazard(item.invocationId) + this.#options.onChange?.(visible.get(item.invocationId)!.runId) + } + return this.list(input) + }) + } + + #serialize(operation: () => Promise): Promise { + const result = this.#tail.catch(() => {}).then(operation) + this.#tail = result + return result + } + + #observe(hazard: MutationHazard, outcome: 'active' | 'unknown', reason: string): void { + this.#options.evidence?.append({ invocationId: hazard.invocationId, kind: 'observed', outcome, reason, actor: 'system' }) + this.#record({ event: outcome === 'active' ? 'mutation.recovery.active' : 'mutation.recovery.protected', level: 'warn', runId: hazard.runId, operationId: hazard.invocationId, errorCode: reason }) + this.#options.onChange?.(hazard.runId) } async #recheck(hazard: MutationHazard, signal: AbortSignal): Promise { @@ -52,16 +143,21 @@ export class MutationRecoveryService { const records = mutations.getOwnedProcesses(hazard.invocationId).map(record => ({ ...record, identity: parseIdentity(record.processId) })) const checkpoint = records.find(record => record.identity?.kind === 'recovery-boot') const owned = records.filter(record => record.identity?.kind === 'native-shell' || record.identity?.kind === 'sandbox-job') + const verifiable = records.filter(record => owned.includes(record) || (record.identity?.version === 3 && ['native-shell-dispatch', 'sandbox-dispatch'].includes(record.identity.kind))) const released = new Set(records.filter(record => record.settlement === 'released').map(record => record.processId)) let active = false + let reason = owned.length ? 'PROCESS_RELEASE_EVIDENCE_MISSING' : records.length ? 'DISPATCH_IDENTITY_MISSING' : 'PROCESS_IDENTITY_MISSING' const verify = this.#options.verify ?? reverifyOwnedProcess - for (const record of owned) { + for (const record of verifiable) { if (released.has(record.processId)) continue if (signal.aborted) return - const outcome = await verify(record.processId, { signal, checkpoint: checkpoint?.processId }) + const outcome = await verify(record.processId, { signal, checkpoint: checkpoint?.processId, receiptDirectory: this.#options.receiptDirectory }) + if (outcome.status === 'active' || (!active && outcome.status === 'unknown')) + reason = outcome.reason active ||= outcome.status === 'active' + this.#options.evidence?.append({ invocationId: hazard.invocationId, kind: outcome.status === 'released' ? 'quiesced' : 'observed', outcome: outcome.status === 'released' ? 'applied' : outcome.status, reason: outcome.reason, actor: 'system', processId: record.processId }) if (outcome.status === 'released') released.add(record.processId) } @@ -72,11 +168,13 @@ export class MutationRecoveryService { continue const identity = record.identity if (identity?.kind === 'native-shell-dispatch') { - const children = owned.filter(child => identity.nonce && child.identity?.nonce === identity.nonce) + const children = owned.filter(child => identity.version === 3 + ? identity.executionId && identity.dispatchId && child.identity?.executionId === identity.executionId && child.identity?.dispatchId === identity.dispatchId + : identity.nonce && child.identity?.nonce === identity.nonce) if (children.length && children.every(child => released.has(child.processId))) released.add(record.processId) } - if (identity?.kind === 'sandbox-dispatch') { + if (identity?.kind === 'sandbox-dispatch' && identity.requestId) { const children = records.filter(child => child.identity?.requestId === identity.requestId && child.processId !== record.processId && child.identity?.kind !== 'recovery-boot') const legacyChildren = sandboxDispatches.length === 1 && owned.every(child => !child.identity?.requestId) ? owned : [] const associated = children.length ? children : legacyChildren @@ -86,13 +184,14 @@ export class MutationRecoveryService { } const withoutCheckpoint = records.filter(record => record.identity?.kind !== 'recovery-boot') const complete = withoutCheckpoint.length > 0 && withoutCheckpoint.every(record => released.has(record.processId)) - if (complete || hazard.reason === 'FILE_RESOURCE_SETTLEMENT_FAILED') { + if (complete || (hazard.reason === 'FILE_RESOURCE_SETTLEMENT_FAILED' && withoutCheckpoint.every(record => released.has(record.processId)))) { for (const record of records) mutations.recordOwnedProcessSettlement({ invocationId: hazard.invocationId, processId: record.processId, settlement: 'released' }) try { signal.throwIfAborted() await waitForVerification(mutations.confirmReleased(hazard.invocationId), signal) this.#record({ event: 'mutation.recovery.released', level: 'info', runId: hazard.runId, operationId: hazard.invocationId }) + this.#options.onChange?.(hazard.runId) } catch { signal.throwIfAborted() @@ -113,7 +212,7 @@ export class MutationRecoveryService { this.#record({ event: 'mutation.recovery.checkpoint_saved', level: 'info', runId: hazard.runId, operationId: hazard.invocationId }) } } - this.#record({ event: active ? 'mutation.recovery.active' : 'mutation.recovery.protected', level: 'warn', runId: hazard.runId, operationId: hazard.invocationId, errorCode: 'PROCESS_RELEASE_EVIDENCE_MISSING' }) + this.#observe(hazard, active ? 'active' : 'unknown', reason) } } diff --git a/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts b/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts index 90b832f64..ad6fd24cf 100644 --- a/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts @@ -164,7 +164,9 @@ describe('mutation ownership', () => { const restored = setup() restored.restoreHazard(state.hazards[0]!) await expect(restored.acquire({ ...input, runId: 'other', invocationId: 'other-1' })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) - restored.confirmReleased('host-1') + await expect(restored.confirmReleased('host-1')).rejects.toThrow('PROCESS_RELEASE_EVIDENCE_MISSING') + restored.recordOwnedProcessSettlement({ invocationId: 'host-1', processId: 'verified-original', settlement: 'released' }) + await restored.confirmReleased('host-1') const recovered = await restored.acquire({ ...input, runId: 'other', invocationId: 'other-2' }) expect(restored.getRunState('failed').hazards).toEqual([]) recovered.release() diff --git a/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts b/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts index fe2a36649..f74da77e1 100644 --- a/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts @@ -28,6 +28,25 @@ async function fixture(processIds: string[]) { } describe('mutation recovery evidence', () => { + it('retains malformed dispatch identities instead of associating absent request identifiers', async () => { + const f = await fixture([JSON.stringify({ kind: 'sandbox-dispatch' }), native('old')]) + f.mutations.recordOwnedProcessSettlement({ invocationId: 'invocation', processId: native('old'), settlement: 'released' }) + const recovery = new MutationRecoveryService({ mutations: f.mutations, checkpoint: async () => null }) + await recovery.recheck() + expect(f.repository.listUnreleased()).toHaveLength(1) + }) + + it('retains a child reported while verification was awaiting its original snapshot', async () => { + const f = await fixture([native('old')]) + const recovery = new MutationRecoveryService({ mutations: f.mutations, verify: async () => { + f.mutations.recordOwnedProcessSettlement({ invocationId: 'invocation', processId: 'late-child', settlement: 'unknown' }) + return { status: 'released', reason: 'supervisor_receipt' } + } }) + await recovery.recheck() + expect(f.repository.listUnreleased()[0]?.processes).toContainEqual({ processId: 'late-child', settlement: 'unknown' }) + expect(f.mutations.getHazards()).toHaveLength(1) + }) + it('releases dispatch and job together only after every associated process is proven released', async () => { const f = await fixture([dispatch('request'), JSON.stringify({ kind: 'native-shell-dispatch', nonce: 'nonce', requestId: 'request' }), native('nonce', 'request')]) const recovery = new MutationRecoveryService({ mutations: f.mutations, verify: async () => ({ status: 'released', reason: 'boot_changed' }) }) diff --git a/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts b/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts new file mode 100644 index 000000000..d4e6f6f90 --- /dev/null +++ b/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts @@ -0,0 +1,209 @@ +import type { DatabaseSync } from 'node:sqlite' +import { randomUUID } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { createConversationRepository } from '../../storage/conversationRepository' +import { openBuddyDatabase } from '../../storage/database' +import { createRunRepository } from '../../storage/runRepository' +import { MutationCoordinator } from '../MutationCoordinator' +import { createMutationEvidenceRepository } from '../mutationEvidenceRepository' +import { MutationRecoveryService } from '../MutationRecoveryService' +import { createMutationRepository } from '../mutationRepository' + +const cleanups: (() => void | Promise)[] = [] +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup() +}) +const signal = new AbortController().signal + +function setup(database: DatabaseSync) { + const repository = createMutationRepository(database) + const evidence = createMutationEvidenceRepository(database) + const mutations = new MutationCoordinator({ repository, evidence }) + cleanups.push(() => mutations.dispose()) + return { repository, evidence, mutations } +} + +async function memoryFixture() { + const database = openBuddyDatabase({ databasePath: ':memory:' }) + cleanups.push(() => database.close()) + return { database, ...setup(database) } +} + +async function hazard(mutations: MutationCoordinator, invocationId = 'old', runId = 'old-run') { + const lease = await mutations.acquire({ invocationId, runId, lifetime: 'external', claims: [{ kind: 'host' }], signal }) + mutations.recordOwnedProcessSettlement({ invocationId, processId: JSON.stringify({ kind: 'sandbox-dispatch', requestId: invocationId }), settlement: 'unknown' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + await mutations.waitForSettlements() +} + +function context(database: DatabaseSync) { + const conversations = createConversationRepository(database) + const runs = createRunRepository(database) + const now = '2026-10-01T00:00:00.000Z' + for (const id of ['old', 'next', 'unrelated']) { + conversations.create({ id, branchId: id, createdAt: now, approvalPolicy: 'policy', executionProfile: 'workspace_write', spaceId: null, title: id }) + conversations.createMessage({ id: `question-${id}`, conversationId: id, branchId: id, role: 'user', runId: null, createdAt: now, content: { text: id } }) + runs.create({ id: `${id}-run`, conversationId: id, branchId: id, triggeringMessageId: `question-${id}`, provider: 'fixture', model: 'fixture', purpose: 'chat', status: 'failed', startedAt: now, approvalPolicy: 'policy', executionProfile: 'workspace_write', piSessionFile: null }) + } + return { conversations, runs } +} + +describe('durable mutation recovery decisions', () => { + it('migrates six legacy records and keeps a waiver distinct from process release after reopening twice', async () => { + const root = await mkdtemp(join(tmpdir(), 'mutation-recovery-')) + cleanups.push(() => rm(root, { recursive: true, force: true })) + const databasePath = join(root, 'buddy.sqlite3') + let database = openBuddyDatabase({ databasePath }) + let f = setup(database) + for (let index = 0; index < 6; index++) { + const id = `legacy-${index}` + const lease = await f.mutations.acquire({ invocationId: id, runId: id, lifetime: 'external', claims: [{ kind: 'tree', canonicalPath: join(root, id) }], signal }) + f.mutations.recordOwnedProcessSettlement({ invocationId: id, processId: JSON.stringify(index === 5 ? { kind: 'sandbox-dispatch', requestId: randomUUID() } : { kind: 'native-shell', platform: 'win32', pid: index + 1, startTime: '1', nonce: randomUUID() }), settlement: 'unknown' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + } + await f.mutations.dispose() + database.exec('DROP TABLE run_mutation_evidence; DROP TABLE run_mutation_executions; PRAGMA user_version = 30') + database.close() + database = openBuddyDatabase({ databasePath }) + f = setup(database) + expect(f.repository.listUnreleased()).toHaveLength(6) + for (const item of f.mutations.getHazards()) await f.mutations.waiveHazard(item.invocationId) + expect(database.prepare('SELECT count(*) AS count FROM run_mutation_processes WHERE settlement = \'unknown\'').get()).toEqual({ count: 6 }) + expect(database.prepare('SELECT count(*) AS count FROM run_mutation_evidence WHERE kind = \'waived\' AND outcome = \'applied\' AND actor = \'user\'').get()).toEqual({ count: 6 }) + await f.mutations.dispose() + database.close() + database = openBuddyDatabase({ databasePath }) + cleanups.push(() => database.close()) + f = setup(database) + await f.mutations.initialize() + expect(f.mutations.getHazards()).toEqual([]) + expect(database.prepare('PRAGMA quick_check').get()).toEqual({ quick_check: 'ok' }) + expect(database.prepare('PRAGMA foreign_key_check').all()).toEqual([]) + const next = await f.mutations.acquire({ invocationId: 'next', runId: 'next', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + next.release() + }) + + it('resumes only the recorded decision after native protection release fails', async () => { + const f = await memoryFixture() + let fail = true + const resources = { + acquire: async () => {}, + reconcile: async () => {}, + release: async (_id: string, protectedState: boolean) => { + if (!protectedState && fail) + throw new Error('NATIVE_RELEASE_FAILED') + }, + } + const first = new MutationCoordinator({ repository: f.repository, evidence: f.evidence, resources }) + cleanups.push(() => first.dispose()) + await hazard(first) + await expect(first.waiveHazard('old')).rejects.toThrow('NATIVE_RELEASE_FAILED') + expect(first.getHazards()).toHaveLength(1) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'waived', outcome: 'requested' }) + await first.dispose() + fail = false + const restored = new MutationCoordinator({ repository: f.repository, evidence: f.evidence, resources }) + cleanups.push(() => restored.dispose()) + await restored.initialize() + expect(restored.getHazards()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'waived', outcome: 'applied' }) + expect(f.database.prepare('SELECT settlement FROM run_mutation_processes').get()).toEqual({ settlement: 'unknown' }) + }) + + it('limits cross-task recovery to recorded blockers and rejects stale or active decisions', async () => { + const f = await memoryFixture() + const scope = context(f.database) + await hazard(f.mutations) + const recovery = new MutationRecoveryService({ mutations: f.mutations, evidence: f.evidence, context: scope, checkpoint: async () => null }) + expect(recovery.list({ conversationId: 'next' })).toEqual([]) + await expect(f.mutations.acquire({ invocationId: 'next-attempt', runId: 'next-run', lifetime: 'runtime', claims: [{ kind: 'host' }], signal })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const item = recovery.list({ conversationId: 'next' })[0]! + expect(item).toMatchObject({ invocationId: 'old', conversationId: 'old' }) + expect(recovery.list({ conversationId: 'unrelated' })).toEqual([]) + await expect(recovery.resolve({ conversationId: 'unrelated', items: [item], decision: 'waived' })).rejects.toMatchObject({ code: 'VALIDATION_FAILED' }) + f.evidence.append({ invocationId: 'old', actor: 'system', kind: 'observed', outcome: 'active', reason: 'supervisor_active' }) + await expect(recovery.resolve({ conversationId: 'next', items: [item], decision: 'waived' })).rejects.toMatchObject({ code: 'VALIDATION_FAILED' }) + f.evidence.append({ invocationId: 'old', actor: 'system', kind: 'observed', outcome: 'unknown', reason: 'job_not_found' }) + await expect(recovery.resolve({ conversationId: 'next', items: [item], decision: 'waived' })).rejects.toMatchObject({ code: 'VALIDATION_FAILED' }) + const current = recovery.list({ conversationId: 'next' })[0]! + await recovery.resolve({ conversationId: 'next', items: [current], decision: 'waived' }) + expect(recovery.list({ conversationId: 'next' })).toEqual([]) + expect(f.database.prepare('SELECT settlement FROM run_mutation_processes').get()).toEqual({ settlement: 'unknown' }) + }) + + it('atomically binds launch authorization and never rolls it back to prepared', async () => { + const f = await memoryFixture() + const executionId = randomUUID() + const dispatchId = randomUUID() + const lease = await f.mutations.acquire({ invocationId: 'run', runId: 'run', lifetime: 'external', claims: [{ kind: 'host' }], signal }) + const record = { invocationId: 'run', processId: 'dispatch', settlement: 'pending' as const, executionId, dispatchId } + f.mutations.recordOwnedProcessSettlement({ ...record, launchState: 'prepared' }) + f.mutations.recordOwnedProcessSettlement({ ...record, processId: 'identity', launchState: 'authorized' }) + expect(() => f.mutations.recordOwnedProcessSettlement({ ...record, processId: 'second-launch', launchState: 'authorized' })).toThrow('MUTATION_EXECUTION_CONFLICT') + f.mutations.recordOwnedProcessSettlement({ ...record, launchState: 'prepared' }) + expect(f.database.prepare('SELECT launch_state FROM run_mutation_executions').get()).toEqual({ launch_state: 'authorized' }) + expect(() => f.mutations.recordOwnedProcessSettlement({ ...record, processId: 'wrong', dispatchId: randomUUID(), launchState: 'authorized' })).toThrow('MUTATION_EXECUTION_CONFLICT') + expect(f.mutations.getOwnedProcesses('run').some(process => process.processId === 'wrong')).toBe(false) + f.mutations.recordOwnedProcessSettlement({ ...record, processId: 'identity', settlement: 'released', launchState: 'settled' }) + expect(() => f.mutations.recordOwnedProcessSettlement({ ...record, processId: 'late-start', launchState: 'authorized' })).toThrow('MUTATION_EXECUTION_CONFLICT') + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + await f.mutations.waitForSettlements() + }) + + it('does not apply a queued user decision cancelled before durable acceptance', async () => { + const f = await memoryFixture() + const scope = context(f.database) + const lease = await f.mutations.acquire({ invocationId: 'old', runId: 'old-run', lifetime: 'external', claims: [{ kind: 'host' }], signal }) + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: JSON.stringify({ kind: 'native-shell', platform: 'linux', nonce: 'nonce', pid: 1, startTime: '1' }), settlement: 'unknown' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + const gate = Promise.withResolvers() + const entered = Promise.withResolvers() + const recovery = new MutationRecoveryService({ mutations: f.mutations, evidence: f.evidence, context: scope, checkpoint: async () => null, verify: async () => { + entered.resolve() + await gate.promise + return { status: 'unknown', reason: 'legacy_identity' } + } }) + const checking = recovery.recheck() + await entered.promise + const controller = new AbortController() + const pending = recovery.resolve({ conversationId: 'old', items: recovery.list({ conversationId: 'old' }), decision: 'waived' }, controller.signal) + const assertion = expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + controller.abort() + gate.resolve() + await checking + await assertion + expect(f.evidence.decision('old')).toBeNull() + expect(f.mutations.getHazards()).toHaveLength(1) + }) + + it('keeps verified recovery requested until native protection has actually settled after restart', async () => { + const f = await memoryFixture() + await hazard(f.mutations) + f.database.prepare('UPDATE run_mutation_processes SET settlement = \'released\'').run() + f.evidence.append({ invocationId: 'old', kind: 'quiesced', outcome: 'requested', actor: 'system', reason: 'PROCESS_RELEASE_CONFIRMED' }) + const restored = new MutationCoordinator({ repository: f.repository, evidence: f.evidence, resources: { + acquire: async () => {}, + reconcile: async () => {}, + release: async () => { + expect(f.repository.isReleased('old')).toBe(false) + expect(f.evidence.decision('old')).toMatchObject({ outcome: 'requested' }) + }, + } }) + cleanups.push(() => restored.dispose()) + expect(restored.getHazards()).toHaveLength(1) + await restored.initialize() + expect(restored.getHazards()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) + }) + + it('cannot waive without a durable audit repository', async () => { + const mutations = new MutationCoordinator() + cleanups.push(() => mutations.dispose()) + await hazard(mutations) + await expect(mutations.waiveHazard('old')).rejects.toThrow('MUTATION_RECOVERY_AUDIT_UNAVAILABLE') + expect(mutations.getHazards()).toHaveLength(1) + }) +}) diff --git a/apps/buddy/service/src/changes/__tests__/registerChangeRpc.spec.ts b/apps/buddy/service/src/changes/__tests__/registerChangeRpc.spec.ts index 9d7749ab2..3f4147e4c 100644 --- a/apps/buddy/service/src/changes/__tests__/registerChangeRpc.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/registerChangeRpc.spec.ts @@ -10,6 +10,8 @@ import { openBuddyDatabase } from '../../storage/database' import { createRunRepository } from '../../storage/runRepository' import { ChangeCaptureService } from '../ChangeCaptureService' import { createChangeSetRepository } from '../changeSetRepository' +import { MutationCoordinator } from '../MutationCoordinator' +import { MutationRecoveryService } from '../MutationRecoveryService' import { registerChangeRpc } from '../registerChangeRpc' describe('branch changes overview', () => { @@ -39,7 +41,7 @@ describe('branch changes overview', () => { conversations.createBranch({ id: 'child', conversationId: 'conversation', parentBranchId: 'root', forkedFromMessageId: 'answer-first', createdAt: now, activate: true }) turn('child-only', 'child', '2026-09-09T00:02:00.000Z') const handlers = new Map() - const dispose = registerChangeRpc({ conversations, runs, service, rpc: { onRequest: (method, handler) => { + const dispose = registerChangeRpc({ conversations, runs, service, recovery: new MutationRecoveryService({ mutations: new MutationCoordinator() }), rpc: { onRequest: (method, handler) => { handlers.set(method, handler) return () => handlers.delete(method) } } }) diff --git a/apps/buddy/service/src/changes/mutationEvidenceRepository.ts b/apps/buddy/service/src/changes/mutationEvidenceRepository.ts new file mode 100644 index 000000000..b3dfd928a --- /dev/null +++ b/apps/buddy/service/src/changes/mutationEvidenceRepository.ts @@ -0,0 +1,92 @@ +import type { DatabaseSync } from 'node:sqlite' +import { withTransaction } from '../storage/database' + +export interface MutationEvidence { + invocationId: string + kind: 'constructive' | 'quiesced' | 'observed' | 'waived' + outcome: 'active' | 'unknown' | 'requested' | 'applied' + reason: string + actor: 'system' | 'user' + processId?: string + relatedRunId?: string +} + +export interface StoredMutationEvidence extends MutationEvidence { + sequence: number + createdAt: string +} + +export interface MutationEvidenceRepository { + append: (evidence: MutationEvidence) => void + latest: (invocationId: string) => StoredMutationEvidence | null + decision: (invocationId: string) => StoredMutationEvidence | null + recordBlocker: (invocationId: string, runId: string) => void + blockedRuns: (invocationId: string) => string[] + applyResolution: (invocationId: string, kind: 'quiesced' | 'waived') => void + createdAt: (invocationId: string) => string | null +} + +interface EvidenceRow { + sequence: number + invocation_id: string + kind: MutationEvidence['kind'] + outcome: MutationEvidence['outcome'] + reason: string + actor: MutationEvidence['actor'] + process_id: string | null + related_run_id: string | null + created_at: string +} + +export function createMutationEvidenceRepository(database: DatabaseSync): MutationEvidenceRepository { + const insert = database.prepare(` + INSERT INTO run_mutation_evidence (invocation_id, kind, outcome, reason, actor, process_id, related_run_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (invocation_id, related_run_id) WHERE related_run_id IS NOT NULL DO NOTHING + `) + const latest = database.prepare(` + SELECT * FROM run_mutation_evidence WHERE invocation_id = ? AND related_run_id IS NULL AND process_id IS NULL + ORDER BY sequence DESC LIMIT 1 + `) + const decision = database.prepare(` + SELECT * FROM run_mutation_evidence WHERE invocation_id = ? AND outcome IN ('requested', 'applied') AND process_id IS NULL AND related_run_id IS NULL + ORDER BY sequence DESC LIMIT 1 + `) + const blockedRuns = database.prepare('SELECT related_run_id FROM run_mutation_evidence WHERE invocation_id = ? AND related_run_id IS NOT NULL') + const createdAt = database.prepare('SELECT created_at FROM run_mutation_leases WHERE invocation_id = ?') + const settle = database.prepare('UPDATE run_mutation_leases SET status = \'released\', reason = ?, updated_at = ? WHERE invocation_id = ?') + const append = (evidence: MutationEvidence) => { + insert.run(evidence.invocationId, evidence.kind, evidence.outcome, evidence.reason, evidence.actor, evidence.processId ?? null, evidence.relatedRunId ?? null, new Date().toISOString()) + } + return { + append, + latest: invocationId => toEvidence(latest.get(invocationId) as EvidenceRow | undefined), + decision: invocationId => toEvidence(decision.get(invocationId) as EvidenceRow | undefined), + recordBlocker: (invocationId, runId) => append({ invocationId, kind: 'observed', outcome: 'unknown', actor: 'system', reason: 'WRITE_BLOCKED', relatedRunId: runId }), + blockedRuns: invocationId => (blockedRuns.all(invocationId) as { related_run_id: string }[]).map(row => row.related_run_id), + createdAt: invocationId => (createdAt.get(invocationId) as { created_at: string } | undefined)?.created_at ?? null, + applyResolution(invocationId, kind) { + withTransaction(database, () => { + append({ invocationId, kind, outcome: 'applied', actor: kind === 'waived' ? 'user' : 'system', reason: kind === 'waived' ? 'USER_ACCEPTED_RISK' : 'PROCESS_RELEASE_CONFIRMED' }) + if (Number(settle.run(kind === 'waived' ? 'USER_ACCEPTED_RISK' : null, new Date().toISOString(), invocationId).changes) !== 1) + throw new Error('MUTATION_LEASE_NOT_FOUND') + }) + }, + } +} + +function toEvidence(row: EvidenceRow | undefined): StoredMutationEvidence | null { + return row + ? { + sequence: row.sequence, + invocationId: row.invocation_id, + kind: row.kind, + outcome: row.outcome, + reason: row.reason, + actor: row.actor, + processId: row.process_id ?? undefined, + relatedRunId: row.related_run_id ?? undefined, + createdAt: row.created_at, + } + : null +} diff --git a/apps/buddy/service/src/changes/mutationRepository.ts b/apps/buddy/service/src/changes/mutationRepository.ts index 8242415ca..4c76c58ef 100644 --- a/apps/buddy/service/src/changes/mutationRepository.ts +++ b/apps/buddy/service/src/changes/mutationRepository.ts @@ -1,9 +1,16 @@ import type { DatabaseSync } from 'node:sqlite' import type { MutationClaim, MutationIdentity, MutationLifetime } from './MutationCoordinator' import { z } from 'zod' +import { withTransaction } from '../storage/database' export type OwnedProcessSettlement = 'pending' | 'released' | 'unknown' +export interface MutationExecutionUpdate { + executionId?: string + dispatchId?: string + launchState?: 'prepared' | 'authorized' | 'settled' +} + export interface StoredMutation extends MutationIdentity { segmentId: string lifetime: MutationLifetime @@ -16,7 +23,7 @@ export interface StoredMutation extends MutationIdentity { export interface MutationRepository { createLease: (lease: MutationIdentity & { segmentId: string, claims: readonly MutationClaim[], lifetime: MutationLifetime }) => void settle: (invocationId: string, status: 'released' | 'hazard', reason: string | null) => void - recordProcess: (invocationId: string, processId: string, settlement: OwnedProcessSettlement) => void + recordProcess: (invocationId: string, processId: string, settlement: OwnedProcessSettlement, execution?: MutationExecutionUpdate) => void listUnreleased: () => StoredMutation[] isReleased: (invocationId: string) => boolean } @@ -50,6 +57,18 @@ export function createMutationRepository(database: DatabaseSync): MutationReposi ON CONFLICT (invocation_id, process_id) DO UPDATE SET settlement = excluded.settlement, updated_at = excluded.updated_at WHERE run_mutation_processes.settlement <> 'released' `) + const executionSchema = z.object({ executionId: z.uuid(), dispatchId: z.uuid(), launchState: z.enum(['prepared', 'authorized', 'settled']) }) + const updateExecution = database.prepare(` + INSERT INTO run_mutation_executions (execution_id, invocation_id, dispatch_id, launch_state, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (execution_id) DO UPDATE SET + launch_state = CASE WHEN excluded.launch_state = 'prepared' THEN run_mutation_executions.launch_state ELSE excluded.launch_state END, + updated_at = excluded.updated_at + WHERE run_mutation_executions.invocation_id = excluded.invocation_id + AND run_mutation_executions.dispatch_id = excluded.dispatch_id + AND (run_mutation_executions.launch_state <> 'settled' OR excluded.launch_state = 'settled') + AND (excluded.launch_state <> 'authorized' OR run_mutation_executions.launch_state = 'prepared') + `) const listUnreleased = database.prepare('SELECT * FROM run_mutation_leases WHERE status <> \'released\' ORDER BY created_at, invocation_id') const findStatus = database.prepare('SELECT status FROM run_mutation_leases WHERE invocation_id = ?') const listProcesses = database.prepare('SELECT process_id, settlement FROM run_mutation_processes WHERE invocation_id = ? ORDER BY process_id') @@ -65,8 +84,16 @@ export function createMutationRepository(database: DatabaseSync): MutationReposi if (Number(settle.run(status, reason, new Date().toISOString(), invocationId).changes) !== 1) throw new Error('MUTATION_LEASE_NOT_FOUND') }, - recordProcess(invocationId, processId, settlement) { - recordProcess.run(invocationId, processId, settlement, new Date().toISOString()) + recordProcess(invocationId, processId, settlement, execution) { + withTransaction(database, () => { + const now = new Date().toISOString() + if (execution?.launchState) { + const value = executionSchema.parse(execution) + if (Number(updateExecution.run(value.executionId, invocationId, value.dispatchId, value.launchState, now, now).changes) !== 1) + throw new Error('MUTATION_EXECUTION_CONFLICT') + } + recordProcess.run(invocationId, processId, settlement, now) + }) }, listUnreleased() { return (listUnreleased.all() as unknown as MutationRow[]).map(row => ({ diff --git a/apps/buddy/service/src/changes/registerChangeRpc.ts b/apps/buddy/service/src/changes/registerChangeRpc.ts index 00ed031f5..a58dd6d42 100644 --- a/apps/buddy/service/src/changes/registerChangeRpc.ts +++ b/apps/buddy/service/src/changes/registerChangeRpc.ts @@ -2,12 +2,14 @@ import type { RuntimeRequestRegistrar } from '../rpc/runtimeRequest' import type { ConversationRepository } from '../storage/conversationRepository' import type { RunRepository } from '../storage/runRepository' import type { ChangeCaptureService } from './ChangeCaptureService' -import { changesRpc } from '../../../shared/changes/changeApi' +import type { MutationRecoveryService } from './MutationRecoveryService' +import { changesRecoveryRpc, changesRpc } from '../../../shared/changes/changeApi' import { BuddyServiceError, registerRuntimeRequest } from '../rpc/runtimeRequest' import { ChangeCaptureError } from './ChangeCaptureService' export function registerChangeRpc(options: { rpc: RuntimeRequestRegistrar + recovery: MutationRecoveryService service: Pick conversations: Pick runs: Pick @@ -38,8 +40,14 @@ export function registerChangeRpc(options: { throw new BuddyServiceError('VALIDATION_FAILED') return result }) + const listRecovery = registerRuntimeRequest(options.rpc, changesRecoveryRpc.list, input => options.recovery.list(input)) + const recheckRecovery = registerRuntimeRequest(options.rpc, changesRecoveryRpc.recheck, (input, signal) => options.recovery.recheckVisible(input, signal)) + const resolveRecovery = registerRuntimeRequest(options.rpc, changesRecoveryRpc.resolve, (input, signal) => options.recovery.resolve(input, signal)) return () => { get() overview() + listRecovery() + recheckRecovery() + resolveRecovery() } } diff --git a/apps/buddy/service/src/sandbox/ShellSandboxClient.ts b/apps/buddy/service/src/sandbox/ShellSandboxClient.ts index 95d23dbce..0b97c5daf 100644 --- a/apps/buddy/service/src/sandbox/ShellSandboxClient.ts +++ b/apps/buddy/service/src/sandbox/ShellSandboxClient.ts @@ -6,10 +6,11 @@ import type { SandboxCommand, SandboxNetworkTarget, SandboxResult } from '../../ import type { RuntimeRpcPeerContract } from '../../../shared/runtime/rpcPeer' import { Buffer } from 'node:buffer' import { randomUUID } from 'node:crypto' +import process from 'node:process' import { Emitter } from '../../../shared/events/Emitter' import { copyEventSnapshot } from '../../../shared/events/eventSnapshot' import { sandboxLifecycleNotificationSchema } from '../../../shared/permissions/sandboxLifecycle' -import { SANDBOX_RPC_TIMEOUT_MS, sandboxNetworkRequestSchema, sandboxOutputSchema, sandboxResultSchema, ShellSandboxError } from '../../../shared/permissions/shellSandbox' +import { SANDBOX_RPC_TIMEOUT_MS, sandboxNetworkRequestSchema, sandboxOutputSchema, sandboxResultSchema, sandboxSettlementSchema, ShellSandboxError } from '../../../shared/permissions/shellSandbox' type ExecOptions = Parameters[2] @@ -26,6 +27,7 @@ interface PendingCommand { approve: (target: SandboxNetworkTarget) => Promise cancel: () => void settled: Promise + record: (settlement: OwnedProcessSettlement) => Promise } export class ShellSandboxClient { @@ -39,10 +41,22 @@ export class ShellSandboxClient { constructor(peer: RuntimeRpcPeerContract) { this.#peer = peer - this.#dispose = peer.onRequest('sandbox.network', async (params) => { + const network = peer.onRequest('sandbox.network', async (params) => { const { requestId, ...target } = sandboxNetworkRequestSchema.parse(params) return { allowed: await this.#pending.get(requestId)?.approve(target) ?? false } }) + const settlement = peer.onRequest('sandbox.settlement', async (params) => { + const { requestId, ...record } = sandboxSettlementSchema.parse(params) + const pending = this.#pending.get(requestId) + if (!pending || this.#disposed) + throw new ShellSandboxError('SANDBOX_CANCELLED') + await pending.record(record) + return { accepted: true } + }) + this.#dispose = () => { + network() + settlement() + } } async prepare(roots: readonly string[]): Promise { @@ -52,20 +66,20 @@ export class ShellSandboxClient { } async exec( - input: Omit, + input: Omit, options: ExecOptions, approveNetwork: (target: SandboxNetworkTarget) => Promise, - supervision?: { onSettlement: (settlement: OwnedProcessSettlement) => void }, + supervision?: { onSettlement: (settlement: OwnedProcessSettlement) => void | Promise }, ): Promise<{ exitCode: number | null }> { if (this.#disposed) throw new ShellSandboxError('SANDBOX_UNAVAILABLE') options.signal?.throwIfAborted() const requestId = randomUUID() - const dispatchId = JSON.stringify({ kind: 'sandbox-dispatch', requestId }) + const executionId = randomUUID() + const dispatchId = randomUUID() + const dispatchProcessId = JSON.stringify({ kind: 'sandbox-dispatch', platform: process.platform, version: 3, requestId, executionId, dispatchId }) const processes = new Map() - let proof = false let noCommandStarted = false - let settlementError: unknown let dispatched = false let returned = false let cancelled = false @@ -89,25 +103,6 @@ export class ShellSandboxClient { backendRevision = parsed.data.revision const { requestId: _, ...event } = parsed.data publish({ kind: 'backend', event }) - try { - const state = event.snapshot.process - if (state?.processId) { - const settlement = state.phase === 'resources_released' ? 'released' : state.phase === 'cleanup_failed' ? 'unknown' : 'pending' - const processId = associateRequest(state.processId, requestId) - if (processes.get(processId) !== 'released') { - processes.set(processId, settlement) - supervision?.onSettlement({ processId, settlement }) - } - if (settlement === 'released') - proof = true - } - if (event.kind === 'settled' && !event.snapshot.started && !state?.pid) - noCommandStarted = true - } - catch (error) { - settlementError = error - cancel() - } } return } @@ -121,22 +116,38 @@ export class ShellSandboxClient { const settled = new Promise((resolve) => { settle = resolve }) - this.#pending.set(requestId, { approve: approveNetwork, cancel, settled }) + this.#pending.set(requestId, { + approve: approveNetwork, + cancel, + settled, + async record(record) { + if (record.executionId !== executionId || record.dispatchId !== dispatchId || returned + || (record.launchState === 'authorized' && (cancelled || options.signal?.aborted))) { + throw new ShellSandboxError('SANDBOX_CANCELLED') + } + const processId = associateRequest(record.processId, requestId) + await supervision?.onSettlement({ ...record, processId }) + processes.set(processId, record.settlement) + if (record.launchState === 'authorized' && (cancelled || options.signal?.aborted)) + throw new ShellSandboxError('SANDBOX_CANCELLED') + }, + }) options.signal?.addEventListener('abort', cancel, { once: true }) try { publish({ kind: 'requested' }) options.signal?.throwIfAborted() + if (this.#disposed || cancelled) + throw new ShellSandboxError('SANDBOX_CANCELLED') + await supervision?.onSettlement({ processId: dispatchProcessId, settlement: 'pending', executionId, dispatchId, launchState: 'prepared' }) + options.signal?.throwIfAborted() if (this.#disposed || cancelled) throw new ShellSandboxError('SANDBOX_CANCELLED') dispatched = true - supervision?.onSettlement({ processId: dispatchId, settlement: 'pending' }) - const result = sandboxResultSchema.parse(await this.#peer.request('host.sandbox.exec', { ...input, requestId }, SANDBOX_RPC_TIMEOUT_MS)) + const result = sandboxResultSchema.parse(await this.#peer.request('host.sandbox.exec', { ...input, requestId, executionId, dispatchId }, SANDBOX_RPC_TIMEOUT_MS)) returned = true if (!result.ok && result.notStarted) noCommandStarted = true publish({ kind: 'returned', result }) - if (settlementError) - throw settlementError options.signal?.throwIfAborted() if (!result.ok) throw new ShellSandboxError(result.code) @@ -154,9 +165,10 @@ export class ShellSandboxClient { if (dispatched) { for (const [processId, settlement] of processes) { if (settlement !== 'released') - supervision?.onSettlement({ processId, settlement: 'unknown' }) + await supervision?.onSettlement({ processId, settlement: 'unknown', executionId, dispatchId, launchState: 'settled' }) } - supervision?.onSettlement({ processId: dispatchId, settlement: proof || (returned && noCommandStarted) ? 'released' : 'unknown' }) + const proof = processes.size > 0 && [...processes.values()].every(state => state === 'released') + await supervision?.onSettlement({ processId: dispatchProcessId, settlement: proof || (returned && noCommandStarted && processes.size === 0) ? 'released' : 'unknown', executionId, dispatchId, launchState: 'settled' }) } } finally { diff --git a/apps/buddy/service/src/sandbox/__tests__/ShellSandboxClient.spec.ts b/apps/buddy/service/src/sandbox/__tests__/ShellSandboxClient.spec.ts index 8a3f85983..c2a3b5ac8 100644 --- a/apps/buddy/service/src/sandbox/__tests__/ShellSandboxClient.spec.ts +++ b/apps/buddy/service/src/sandbox/__tests__/ShellSandboxClient.spec.ts @@ -12,6 +12,10 @@ function fixture() { const notifications: Array<{ method: string, params: unknown }> = [] const listeners = new Set<(method: string, params: unknown) => void>() let requestId = '' + let executionId = '' + let dispatchId = '' + const requested = deferred() + const handlers = new Map unknown>() const peer: RuntimeRpcPeerContract = { notify: (method, params) => { notifications.push({ method, params }) }, onNotification: (listener) => { @@ -20,9 +24,17 @@ function fixture() { listeners.delete(listener) } }, - onRequest: () => () => {}, + onRequest: (method, handler) => { + handlers.set(method, handler) + return () => { + handlers.delete(method) + } + }, request: async (_method, params) => { requestId = (params as SandboxCommand).requestId + executionId = (params as SandboxCommand).executionId! + dispatchId = (params as SandboxCommand).dispatchId! + requested.resolve() return response.promise }, close: () => {}, @@ -32,17 +44,70 @@ function fixture() { const events: SandboxClientEvent[] = [] client.onDidChange(event => events.push(event)) const execute = (supervision?: Parameters[3]) => client.exec({ command: 'private command text', cwd: '/workspace', roots: ['/workspace'], workspaceRoots: [], resourceReadRoots: [], additionalDirectories: [], readOnly: true }, { onData: () => {}, signal: controller.signal }, async () => false, supervision) - return { client, controller, events, execute, notifications, response, backend(revision: number, kind: string, result?: unknown, process?: unknown) { + return { client, controller, events, execute, notifications, response, requested: requested.promise, record(processId: string, settlement: 'pending' | 'released' | 'unknown') { + return handlers.get('sandbox.settlement')!({ requestId, executionId, dispatchId, processId, settlement, launchState: settlement === 'pending' ? 'authorized' : 'settled' }) + }, backend(revision: number, kind: string, result?: unknown, process?: unknown) { for (const listener of listeners) listener('host.sandbox.lifecycle', { requestId, revision, kind, snapshot: { phase: kind === 'settled' ? 'finished' : 'running', started: true, waiting: 0, cancellation: 'none', ...result ? { result } : {}, ...process ? { process } : {} } }) } } } describe('sandbox client facts', () => { + it('acknowledges startup only after the service persistence callback resolves', async () => { + const f = fixture() + const persisted = deferred() + const entered = deferred() + const pending = f.execute({ async onSettlement(record) { + if (record.launchState === 'authorized') { + entered.resolve() + await persisted.promise + } + } }) + await f.requested + let acknowledged = false + const acknowledgement = Promise.resolve(f.record('native-identity', 'pending')).then((value) => { + acknowledged = true + return value + }) + await entered.promise + expect(acknowledged).toBe(false) + persisted.resolve() + await expect(acknowledgement).resolves.toEqual({ accepted: true }) + await f.record('native-identity', 'released') + f.response.resolve({ ok: true, exitCode: 0 }) + await pending + await f.client.dispose() + }) + + it('rejects a late startup acknowledgement after cancellation', async () => { + const f = fixture() + const persisted = deferred() + const entered = deferred() + const pending = f.execute({ async onSettlement(record) { + if (record.launchState === 'authorized') { + entered.resolve() + await persisted.promise + } + } }) + const rejected = expect(pending).rejects.toThrow() + await f.requested + const acknowledgement = Promise.resolve(f.record('native-identity', 'pending')) + const denied = expect(acknowledgement).rejects.toThrow('SANDBOX_CANCELLED') + await entered.promise + f.controller.abort() + persisted.resolve() + await denied + f.response.resolve({ ok: false, code: 'SANDBOX_CANCELLED' }) + await rejected + await f.client.dispose() + }) + it('releases only a trusted pre-dispatch rejection and retains unproven failures', async () => { for (const notStarted of [true, false]) { const f = fixture() const states = new Map() - const pending = f.execute({ onSettlement: ({ processId, settlement }) => states.set(processId, settlement) }) + const pending = f.execute({ onSettlement: ({ processId, settlement }) => { + states.set(processId, settlement) + } }) f.response.resolve({ ok: false, code: 'SANDBOX_UNAVAILABLE', ...(notStarted ? { notStarted: true } : {}) }) await expect(pending).rejects.toThrow('SANDBOX_UNAVAILABLE') expect([...states.values()]).toEqual([notStarted ? 'released' : 'unknown']) @@ -53,8 +118,11 @@ describe('sandbox client facts', () => { it('requires a trusted tree release receipt even when the backend returns success', async () => { const f = fixture() const states = new Map() - const pending = f.execute({ onSettlement: ({ processId, settlement }) => states.set(processId, settlement) }) - f.backend(1, 'started') + const pending = f.execute({ onSettlement: ({ processId, settlement }) => { + states.set(processId, settlement) + } }) + await f.requested + f.backend(1, 'process', undefined, { phase: 'resources_released', pid: 1234, processId: JSON.stringify({ kind: 'native-shell', pid: 1234 }) }) f.backend(2, 'settled', { ok: true, exitCode: 0 }) f.response.resolve({ ok: true, exitCode: 0 }) await pending @@ -66,11 +134,16 @@ describe('sandbox client facts', () => { const f = fixture() const states = new Map() const identity = JSON.stringify({ kind: 'native-shell', pid: 1234, startTime: '12345', bootId: 'fixture' }) - const pending = f.execute({ onSettlement: ({ processId, settlement }) => states.set(processId, settlement) }) + const pending = f.execute({ onSettlement: ({ processId, settlement }) => { + states.set(processId, settlement) + } }) const rejected = expect(pending).rejects.toThrow() + await f.requested + await f.record(identity, 'pending') f.backend(1, 'process', undefined, { phase: 'spawned', pid: 1234, processId: identity }) f.controller.abort() expect([...states.values()]).toEqual(['pending', 'pending']) + await f.record(identity, 'released') f.backend(2, 'process', undefined, { phase: 'resources_released', pid: 1234, processId: identity }) f.response.resolve({ ok: false, code: 'SANDBOX_CANCELLED' }) await rejected @@ -82,6 +155,7 @@ describe('sandbox client facts', () => { const records: import('../../../../shared/diagnostics/applicationDiagnostic').ApplicationDiagnostic[] = [] const subscription = observeSandboxClient(f.client, event => records.push(event)) const pending = diagnosticContext.run({ runId: 'run-fixture', toolCallId: 'tool-fixture' }, f.execute) + await f.requested f.backend(1, 'started') f.response.resolve({ ok: true, exitCode: 0 }) await pending @@ -94,6 +168,7 @@ describe('sandbox client facts', () => { it('keeps successful settlement distinct from cleanup and ignores stale backend messages', async () => { const f = fixture() const pending = f.execute() + await f.requested f.backend(2, 'started') f.backend(1, 'started') f.backend(3, 'settled', { ok: true, exitCode: 0 }) @@ -112,6 +187,7 @@ describe('sandbox client facts', () => { const f = fixture() const pending = f.execute() const rejected = expect(pending).rejects.toThrow() + await f.requested f.controller.abort() expect(f.events.map(event => event.kind)).toEqual(['requested', 'cancel-requested']) f.backend(1, 'settled', { ok: true, exitCode: 0 }) diff --git a/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts b/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts index 61bd07467..e54806bf7 100644 --- a/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts +++ b/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts @@ -43,9 +43,10 @@ async function fixture() { await capability.extension!.factory({ registerTool: (tool: ToolDefinition) => { tools.set(tool.name, tool) } } as unknown as Parameters[0]) - const invoke = (command: string) => mutations.withInvocation(lease, () => tools.get('bash')!.execute('shell-call', { command }, controller.signal, undefined, undefined!)) + const invokeWithoutContext = (command: string) => tools.get('bash')!.execute('shell-call', { command }, controller.signal, undefined, undefined!) + const invoke = (command: string) => mutations.withInvocation(lease, () => invokeWithoutContext(command)) const processes = () => database.prepare('SELECT process_id, settlement FROM run_mutation_processes').all() as { process_id: string, settlement: string }[] - return { database, directory, repository, mutations, controller, lease, invoke, processes, async dispose() { + return { database, directory, repository, mutations, controller, lease, invoke, invokeWithoutContext, processes, async dispose() { await mutations.dispose() database.close() await rm(directory, { recursive: true, force: true }) @@ -114,6 +115,16 @@ describe.skipIf(process.platform !== 'linux')('real shell process ownership', () } }) + it('does not launch an untracked command when a production mutation coordinator is present', async () => { + const f = await fixture() + try { + await expect(f.invokeWithoutContext('touch untracked-start')).rejects.toThrow('SHELL_INVOCATION_REQUIRED') + await expect(access(join(f.directory, 'untracked-start'))).rejects.toThrow() + expect(f.processes()).toEqual([]) + } + finally { await f.dispose() } + }) + it('does not launch a command for an invocation whose lease has already ended', async () => { const f = await fixture() try { diff --git a/apps/buddy/service/src/sandbox/__tests__/shellSandbox.spec.ts b/apps/buddy/service/src/sandbox/__tests__/shellSandbox.spec.ts index c765b4c47..2a2d1e99f 100644 --- a/apps/buddy/service/src/sandbox/__tests__/shellSandbox.spec.ts +++ b/apps/buddy/service/src/sandbox/__tests__/shellSandbox.spec.ts @@ -9,6 +9,7 @@ import { join, resolve } from 'node:path' import process from 'node:process' import { setTimeout } from 'node:timers/promises' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { verifyOwnedProcessRelease } from '../../../../platform/process/ownedProcessRecovery' import { resolveSandboxDependencies } from '../../../../platform/process/sandboxDependencies' import { createSandboxDirectory } from '../../../../platform/process/sandboxDirectory' import { createSandboxEnvironment } from '../../../../platform/process/sandboxEnvironment' @@ -102,11 +103,32 @@ describe.skipIf(process.platform !== 'linux')('linux shell enforcement', () => { expect(new Set(supervised.map(process => process.pid)).size).toBe(1) const dispatch = JSON.parse(dispatched[0]!.processId!) const identity = JSON.parse(supervised[0]!.processId!) - expect(dispatch).toMatchObject({ kind: 'native-shell-dispatch', nonce: identity.nonce }) + expect(dispatch).toMatchObject({ kind: 'native-shell-dispatch', executionId: identity.executionId, dispatchId: identity.dispatchId }) expect(processes.find(process => process.phase === 'exited')?.exitCode).toBe(0) expect(JSON.stringify(events)).not.toMatch(/synthetic-secret|example-allowed|printf/) }, 15_000) + it('recovers the native SRT command domain when final service persistence is lost', async () => { + const identities: string[] = [] + const receiptDirectory = join(directory, 'receipts') + const { result } = await execute('printf committed > recovered-output', { + onSettlement(record) { + if (record.settlement === 'pending') + identities.push(record.processId) + else + throw new Error('final persistence disconnected') + }, + }, { receiptDirectory }) + expect(result.ok).toBe(false) + expect(await readFile(join(workspace, 'recovered-output'), 'utf8')).toBe('committed') + expect(identities).toHaveLength(2) + for (const identity of identities) { + await expect(verifyOwnedProcessRelease(identity, { receiptDirectory })) + .resolves + .toEqual({ status: 'released', reason: 'supervisor_receipt' }) + } + }) + it('enforces read-only writes in the kernel while retaining private scratch space', async () => { await writeFile(join(workspace, 'preserved'), 'workspace-preserved') const { result, output } = await execute('set -e; ! rm preserved; ! touch new-file; printf scratch > "$TMPDIR/scratch"; test "$(cat "$TMPDIR/scratch")" = scratch; if test -w /tmp; then scratch=$(mktemp /tmp/standard-XXXXXX); printf private > "$scratch"; test "$(cat "$scratch")" = private; rm "$scratch"; fi', {}, { readOnly: true }) diff --git a/apps/buddy/service/src/sandbox/backends/srt/runSrtSandbox.ts b/apps/buddy/service/src/sandbox/backends/srt/runSrtSandbox.ts index 66c783998..3f1269dcc 100644 --- a/apps/buddy/service/src/sandbox/backends/srt/runSrtSandbox.ts +++ b/apps/buddy/service/src/sandbox/backends/srt/runSrtSandbox.ts @@ -64,9 +64,13 @@ export async function runSrtSandbox(input: SrtSandboxInput, options: SandboxExec }, }, { signal: options.signal, + executionId: input.executionId, + dispatchId: input.dispatchId, + receiptDirectory: input.receiptDirectory, onData: options.onData, onProcess: options.onProcess, - onSettlement(next) { + async onSettlement(next) { + await options.onSettlement?.(next) settlements.set(next.processId, next) const pid = JSON.parse(next.processId).pid as number | undefined if (next.settlement === 'pending') diff --git a/apps/buddy/service/src/sandbox/backends/windows/runWindowsSandbox.ts b/apps/buddy/service/src/sandbox/backends/windows/runWindowsSandbox.ts index b0119de0e..96c7081c6 100644 --- a/apps/buddy/service/src/sandbox/backends/windows/runWindowsSandbox.ts +++ b/apps/buddy/service/src/sandbox/backends/windows/runWindowsSandbox.ts @@ -49,7 +49,7 @@ export async function runWindowsSandbox(input: SandboxBackendInput Promise onData: (data: Buffer) => void onStarted: () => void + onSettlement?: (settlement: OwnedProcessSettlement) => void | Promise onProcess?: (process: SandboxProcessState) => void } diff --git a/apps/buddy/service/src/sandbox/sandboxProcess.ts b/apps/buddy/service/src/sandbox/sandboxProcess.ts index b45c74cd1..9a6230fb0 100644 --- a/apps/buddy/service/src/sandbox/sandboxProcess.ts +++ b/apps/buddy/service/src/sandbox/sandboxProcess.ts @@ -1,6 +1,6 @@ import process from 'node:process' import { z } from 'zod' -import { sandboxProcessInputSchema } from '../../../shared/permissions/shellSandbox' +import { sandboxProcessInputSchema, sandboxSettlementAckSchema } from '../../../shared/permissions/shellSandbox' import { BuddyServiceRpcServer } from '../rpc/BuddyServiceRpcServer' import { runSandboxCommand } from './runSandboxCommand' @@ -21,6 +21,9 @@ peer.onRequest('sandbox.exec', async (params) => { const input = sandboxProcessInputSchema.parse(params) return runSandboxCommand(input, { signal: controller.signal, + async onSettlement(settlement) { + sandboxSettlementAckSchema.parse(await peer.request('sandbox.settlement', { ...settlement, requestId: input.requestId }, 60_000)) + }, onLifecycle: event => peer.notify('sandbox.lifecycle', { ...event, requestId: input.requestId }), onData(data) { for (let offset = 0; offset < data.length; offset += 64 * 1024) diff --git a/apps/buddy/service/src/sandbox/shellCapability.ts b/apps/buddy/service/src/sandbox/shellCapability.ts index bedab3396..6f8e8d131 100644 --- a/apps/buddy/service/src/sandbox/shellCapability.ts +++ b/apps/buddy/service/src/sandbox/shellCapability.ts @@ -27,6 +27,7 @@ const HOST_TOOLS = toPiToolGroup({ name: 'lexora_host', description: 'Run author export function createShellCapability(options: { cwd: string + receiptDirectory?: string execution: ShellExecution getExecution?: () => ShellExecution getGrants: () => readonly DirectoryGrant[] @@ -82,9 +83,12 @@ export function createShellCapability(options: { operations: { exec: async (command: string, cwd: string, execOptions: Parameters[2]) => { const invocation = options.mutations?.getCurrentInvocation() + if (options.mutations && !invocation) + throw new Error('SHELL_INVOCATION_REQUIRED') const shell = shellName === 'powershell' ? getPowerShellConfig() : getShellConfig() return runSupervisedShellProcess({ ...shell, command, cwd, environment: execOptions.env ?? process.env }, { ...execOptions, + receiptDirectory: options.receiptDirectory, onSettlement(settlement) { if (invocation && !options.mutations!.recordOwnedProcessSettlement({ ...settlement, invocationId: invocation.invocationId })) throw new Error('SHELL_INVOCATION_EXPIRED') @@ -156,6 +160,8 @@ export function createShellCapability(options: { }, { onSettlement: (() => { const invocation = options.mutations?.getCurrentInvocation() + if (options.mutations && !invocation) + throw new Error('SHELL_INVOCATION_REQUIRED') return (settlement: OwnedProcessSettlement) => { if (invocation && !options.mutations!.recordOwnedProcessSettlement({ ...settlement, invocationId: invocation.invocationId })) throw new Error('SHELL_INVOCATION_EXPIRED') diff --git a/apps/buddy/service/src/storage/database.ts b/apps/buddy/service/src/storage/database.ts index 6ed399cb3..a6e35f7cd 100644 --- a/apps/buddy/service/src/storage/database.ts +++ b/apps/buddy/service/src/storage/database.ts @@ -26,6 +26,8 @@ const BUDDY_CURRENT_SCHEMA_COLUMNS = { run_file_change_captures: ['invocation_id'], run_mutation_leases: ['lifetime', 'invocation_id', 'run_id', 'segment_id', 'claims_json', 'status', 'reason', 'created_at', 'updated_at'], run_mutation_processes: ['invocation_id', 'process_id', 'settlement', 'updated_at'], + run_mutation_evidence: ['sequence', 'invocation_id', 'kind', 'outcome', 'reason', 'actor', 'process_id', 'related_run_id', 'created_at'], + run_mutation_executions: ['execution_id', 'invocation_id', 'dispatch_id', 'launch_state', 'created_at', 'updated_at'], conversation_public_heads: ['conversation_id', 'sequence'], conversation_publications: ['source_id', 'conversation_id', 'branch_id', 'sequence', 'kind', 'message_id', 'run_id', 'source_sequence', 'author_actor_id'], run_public_snapshots: ['run_id', 'public_sequence'], diff --git a/apps/buddy/service/src/storage/migrations/v31MutationRecovery.ts b/apps/buddy/service/src/storage/migrations/v31MutationRecovery.ts new file mode 100644 index 000000000..49f3a3616 --- /dev/null +++ b/apps/buddy/service/src/storage/migrations/v31MutationRecovery.ts @@ -0,0 +1,27 @@ +export const BUDDY_V31_MUTATION_RECOVERY_SCHEMA_SQL = ` +CREATE TABLE run_mutation_evidence ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + invocation_id TEXT NOT NULL REFERENCES run_mutation_leases(invocation_id) ON DELETE CASCADE, + kind TEXT NOT NULL CHECK (kind IN ('constructive', 'quiesced', 'observed', 'waived')), + outcome TEXT NOT NULL CHECK (outcome IN ('active', 'unknown', 'requested', 'applied')), + reason TEXT NOT NULL, + actor TEXT NOT NULL CHECK (actor IN ('system', 'user')), + process_id TEXT, + related_run_id TEXT, + created_at TEXT NOT NULL, + CHECK ((kind = 'waived' AND actor = 'user') OR (kind <> 'waived' AND actor = 'system')) +); +CREATE INDEX mutation_evidence_invocation ON run_mutation_evidence(invocation_id, sequence); +CREATE UNIQUE INDEX mutation_evidence_blocker ON run_mutation_evidence(invocation_id, related_run_id) + WHERE related_run_id IS NOT NULL; + +CREATE TABLE run_mutation_executions ( + execution_id TEXT PRIMARY KEY, + invocation_id TEXT NOT NULL REFERENCES run_mutation_leases(invocation_id) ON DELETE CASCADE, + dispatch_id TEXT NOT NULL, + launch_state TEXT NOT NULL CHECK (launch_state IN ('prepared', 'authorized', 'settled')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX mutation_executions_invocation ON run_mutation_executions(invocation_id); +` diff --git a/apps/buddy/service/src/storage/schema.ts b/apps/buddy/service/src/storage/schema.ts index 2071eca84..089ebb9d3 100644 --- a/apps/buddy/service/src/storage/schema.ts +++ b/apps/buddy/service/src/storage/schema.ts @@ -29,13 +29,15 @@ import { BUDDY_V28_AVATARS_SCHEMA_SQL } from './migrations/v28BuddyAvatars' import { BUDDY_V29_SESSION_REFERENCES_SCHEMA_SQL } from './migrations/v29SessionReferences' import { BUDDY_V30_TASK_MODELS_SCHEMA_SQL } from './migrations/v30TaskModels' +import { BUDDY_V31_MUTATION_RECOVERY_SCHEMA_SQL } from './migrations/v31MutationRecovery' + export interface BuddySchemaMigration { foreignKeys?: 'off' sql: string version: number } -export const BUDDY_SCHEMA_VERSION = 30 as const +export const BUDDY_SCHEMA_VERSION = 31 as const export const BUDDY_SCHEMA_MIGRATIONS: readonly BuddySchemaMigration[] = [ { sql: BUDDY_V1_INITIAL_SCHEMA_SQL, version: 1 }, @@ -68,4 +70,5 @@ export const BUDDY_SCHEMA_MIGRATIONS: readonly BuddySchemaMigration[] = [ { sql: BUDDY_V28_AVATARS_SCHEMA_SQL, version: 28 }, { sql: BUDDY_V29_SESSION_REFERENCES_SCHEMA_SQL, version: 29 }, { sql: BUDDY_V30_TASK_MODELS_SCHEMA_SQL, version: 30 }, + { sql: BUDDY_V31_MUTATION_RECOVERY_SCHEMA_SQL, version: 31 }, ] diff --git a/apps/buddy/shared/changes/__tests__/changeRecoveryApi.spec.ts b/apps/buddy/shared/changes/__tests__/changeRecoveryApi.spec.ts new file mode 100644 index 000000000..f68bd2aa0 --- /dev/null +++ b/apps/buddy/shared/changes/__tests__/changeRecoveryApi.spec.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { changeRecoveryItemSchema, changesRecoveryRpc } from '../changeApi' + +describe('change recovery user decisions', () => { + const selection = { conversationId: 'task', items: [{ invocationId: 'invocation', revision: 'revision-1' }], decision: 'waived' } + + it('requires a fixed, versioned, unique selection and rejects broad or unrecognized decisions', () => { + expect(changesRecoveryRpc.resolve.input.parse(selection)).toEqual(selection) + for (const invalid of [ + { ...selection, all: true }, + { ...selection, items: [] }, + { ...selection, items: [{ invocationId: 'invocation' }] }, + { ...selection, items: [{ invocationId: 'invocation', revision: '' }] }, + { ...selection, items: [selection.items[0], selection.items[0]] }, + { ...selection, decision: 'released' }, + ]) + expect(changesRecoveryRpc.resolve.input.safeParse(invalid).success).toBe(false) + }) + + it('rechecks only an explicit nonempty set and does not require user decisions to inspect', () => { + expect(changesRecoveryRpc.list.input.parse({ conversationId: 'task' })).toEqual({ conversationId: 'task' }) + expect(changesRecoveryRpc.recheck.input.parse({ conversationId: 'task', invocationIds: ['first', 'second'] })).toEqual({ conversationId: 'task', invocationIds: ['first', 'second'] }) + for (const invocationIds of [[], ['first', 'first']]) + expect(changesRecoveryRpc.recheck.input.safeParse({ conversationId: 'task', invocationIds }).success).toBe(false) + }) + + it('preserves deleted-task provenance and rejects internal process details from the public projection', () => { + const item = { + invocationId: 'old-operation', + runId: 'old-run', + conversationId: null, + taskTitle: '', + occurredAt: '2026-10-15T00:00:00.000Z', + affectedResources: ['/workspace/example'], + reason: 'MUTATION_INTERRUPTED', + status: 'unknown', + revision: 'revision-1', + } + expect(changeRecoveryItemSchema.parse(item)).toEqual(item) + expect(changeRecoveryItemSchema.safeParse({ ...item, pid: 1234 }).success).toBe(false) + }) +}) diff --git a/apps/buddy/shared/changes/changeApi.ts b/apps/buddy/shared/changes/changeApi.ts index 78295919d..67b44b923 100644 --- a/apps/buddy/shared/changes/changeApi.ts +++ b/apps/buddy/shared/changes/changeApi.ts @@ -72,6 +72,45 @@ export const changesRpc = { get: { method: 'changes.get', input: changesRequestSchemas.changeSet, response: changesResponseSchemas.changeSet }, } as const satisfies Record +export const changeRecoveryItemSchema = z.object({ + invocationId: idSchema, + runId: idSchema, + conversationId: idSchema.nullable(), + taskTitle: z.string().max(512), + occurredAt: timestampSchema, + affectedResources: z.array(z.string().min(1).max(4096)), + reason: z.string().min(1).max(128), + status: z.enum(['active', 'unknown']), + revision: z.string().min(1).max(256), +}).strict() + +export const changeRecoveryListRequestSchema = z.object({ + conversationId: idSchema, +}).strict() + +export const changeRecoveryRecheckRequestSchema = changeRecoveryListRequestSchema.extend({ + invocationIds: z.array(idSchema).min(1).max(100).refine(ids => new Set(ids).size === ids.length), +}).strict() + +export const changeRecoveryResolveRequestSchema = changeRecoveryListRequestSchema.extend({ + items: z.array(z.object({ + invocationId: idSchema, + revision: z.string().min(1).max(256), + }).strict()).min(1).max(100).refine(items => new Set(items.map(item => item.invocationId)).size === items.length), + decision: z.literal('waived'), +}).strict() + +export type LocalChangeRecoveryItem = DeepReadonly> +export type ChangeRecoveryListRequest = z.infer +export type ChangeRecoveryRecheckRequest = z.infer +export type ChangeRecoveryResolveRequest = z.infer + +export const changesRecoveryRpc = { + list: { method: 'changes.recovery.list', input: changeRecoveryListRequestSchema, response: z.array(changeRecoveryItemSchema) }, + recheck: { method: 'changes.recovery.recheck', input: changeRecoveryRecheckRequestSchema, response: z.array(changeRecoveryItemSchema) }, + resolve: { method: 'changes.recovery.resolve', input: changeRecoveryResolveRequestSchema, response: z.array(changeRecoveryItemSchema) }, +} as const satisfies Record + export const changeSetChangeNoticeSchema = z.object({ sourceId: z.uuid(), revision: z.number().int().positive(), diff --git a/apps/buddy/shared/permissions/shellSandbox.ts b/apps/buddy/shared/permissions/shellSandbox.ts index 8cf0cb1b9..85d705203 100644 --- a/apps/buddy/shared/permissions/shellSandbox.ts +++ b/apps/buddy/shared/permissions/shellSandbox.ts @@ -48,6 +48,8 @@ export const sandboxCommandSchema = z.object({ resourceReadRoots: z.array(pathSchema).max(512).default([]), additionalDirectories: z.array(sandboxDirectoryGrantSchema).max(128).default([]), requestId: z.uuid(), + executionId: z.uuid().optional(), + dispatchId: z.uuid().optional(), timeout: z.number().positive().max(86_400).optional(), }).strict() @@ -60,6 +62,7 @@ export const sandboxPreparationSchema = z.object({ export const sandboxProcessInputSchema = sandboxCommandSchema.extend({ home: pathSchema, privateRoot: pathSchema, + receiptDirectory: pathSchema.optional(), protectedRoots: z.array(pathSchema).max(32), sensitiveRoots: z.array(pathSchema).max(128), searchDirectory: pathSchema, @@ -99,6 +102,17 @@ export const sandboxResultSchema = z.discriminatedUnion('ok', [ ]) export type SandboxResult = z.infer +export const sandboxSettlementSchema = z.object({ + requestId: z.uuid(), + processId: z.string().min(1).max(8192), + settlement: z.enum(['pending', 'released', 'unknown']), + executionId: z.uuid(), + dispatchId: z.uuid(), + launchState: z.enum(['prepared', 'authorized', 'settled']), +}).strict() + +export const sandboxSettlementAckSchema = z.object({ accepted: z.literal(true) }).strict() + export const SANDBOX_RPC_TIMEOUT_MS = 25 * 60 * 60 * 1_000 export class ShellSandboxError extends Error { diff --git a/apps/buddy/shared/runtime/runtimeProtocol.ts b/apps/buddy/shared/runtime/runtimeProtocol.ts index 12d2c863f..22cbd6df3 100644 --- a/apps/buddy/shared/runtime/runtimeProtocol.ts +++ b/apps/buddy/shared/runtime/runtimeProtocol.ts @@ -1,6 +1,6 @@ import { z } from 'zod' -export const BUDDY_SERVICE_PROTOCOL_VERSION = 18 as const +export const BUDDY_SERVICE_PROTOCOL_VERSION = 19 as const export const BUDDY_SERVICE_FAILURE_CODES = [ 'EVENT_LOG_CORRUPTED', diff --git a/apps/buddy/src/i18n/locales/en-US/chat.ts b/apps/buddy/src/i18n/locales/en-US/chat.ts index 4e4a5d028..e8d2934af 100644 --- a/apps/buddy/src/i18n/locales/en-US/chat.ts +++ b/apps/buddy/src/i18n/locales/en-US/chat.ts @@ -533,6 +533,24 @@ export default { 'desktop.chat.progressApproval': 'Approval pending', 'desktop.chat.progressPreparing': 'Preparing', 'desktop.chat.progressToolExecuting': 'Running', + 'desktop.chat.changeRecovery.title': 'Some locations are protected from changes', + 'desktop.chat.changeRecovery.description': 'An earlier operation still needs verification. These locations are protected to avoid concurrent changes. You can still view tasks and files.', + 'desktop.chat.changeRecovery.active': 'The earlier operation is still running. Changes cannot be resumed yet.', + 'desktop.chat.changeRecovery.unknown': 'We cannot yet confirm whether the earlier operation has stopped.', + 'desktop.chat.changeRecovery.unknownLocation': 'The affected locations are not yet known', + 'desktop.chat.changeRecovery.untitledTask': 'Untitled task', + 'desktop.chat.changeRecovery.deletedTask': 'Deleted task', + 'desktop.chat.changeRecovery.recheck': 'Check again', + 'desktop.chat.changeRecovery.more': 'More options', + 'desktop.chat.changeRecovery.acceptRisk': 'Accept risk and resume changes', + 'desktop.chat.changeRecovery.confirmTitle': 'Accept risk and resume changes?', + 'desktop.chat.changeRecovery.confirmDescription': 'Resume changes for only the following {count} items. Earlier operations may still be running, so further changes could conflict or overwrite files. Continue only if you accept these risks.', + 'desktop.chat.changeRecovery.noReplay': 'This will not stop earlier operations or rerun old commands. Accepting the risk does not confirm that those operations have ended.', + 'desktop.chat.changeRecovery.waived': 'You accepted the risk and resumed changes for the selected locations. Earlier operations have not been confirmed stopped. Old commands will not rerun automatically.', + 'desktop.chat.changeRecovery.changed': 'These items have changed. Close this confirmation and check again before choosing.', + 'desktop.chat.changeRecovery.loadFailed': 'Could not load protected items. Please check again.', + 'desktop.chat.changeRecovery.recheckFailed': 'The check did not complete. Protection remains in place. Please check again later.', + 'desktop.chat.changeRecovery.resolveFailed': 'Could not resume changes for all selected items. Check again before choosing. New items will not be accepted automatically.', 'desktop.chat.draftRestoration.restoring': 'Restoring unsent input. Your input will be saved once restoration finishes.', 'desktop.chat.draftRestoration.failed': 'Unsent input could not be restored. Current input is not saved yet. Retry before sending or quitting.', 'desktop.chat.draftRestoration.conflict': 'Saved content conflicts with input entered during restoration. New input cannot be saved until this is resolved.', diff --git a/apps/buddy/src/i18n/locales/zh-CN/chat.ts b/apps/buddy/src/i18n/locales/zh-CN/chat.ts index ac3e457c8..f0bcf1aa3 100644 --- a/apps/buddy/src/i18n/locales/zh-CN/chat.ts +++ b/apps/buddy/src/i18n/locales/zh-CN/chat.ts @@ -531,6 +531,24 @@ export default { 'desktop.chat.progressApproval': '有操作待批准', 'desktop.chat.progressPreparing': '准备中', 'desktop.chat.progressToolExecuting': '正在执行', + 'desktop.chat.changeRecovery.title': '部分位置暂时无法修改', + 'desktop.chat.changeRecovery.description': '此前操作的状态仍需核验。为避免同时修改,下面的位置暂时受到保护;你仍可查看任务和文件。', + 'desktop.chat.changeRecovery.active': '此前的操作仍在运行,暂不允许恢复修改。', + 'desktop.chat.changeRecovery.unknown': '暂时无法确认此前的操作是否已停止。', + 'desktop.chat.changeRecovery.unknownLocation': '影响范围暂时无法确定', + 'desktop.chat.changeRecovery.untitledTask': '未命名任务', + 'desktop.chat.changeRecovery.deletedTask': '已删除的任务', + 'desktop.chat.changeRecovery.recheck': '重新检查', + 'desktop.chat.changeRecovery.more': '更多选项', + 'desktop.chat.changeRecovery.acceptRisk': '接受风险并恢复修改', + 'desktop.chat.changeRecovery.confirmTitle': '接受风险并恢复修改?', + 'desktop.chat.changeRecovery.confirmDescription': '仅对以下 {count} 项恢复修改。此前的操作可能仍在运行,继续修改可能产生冲突或覆盖文件。请仅在愿意承担这些风险时继续。', + 'desktop.chat.changeRecovery.noReplay': '此操作不会结束此前的操作,也不会重新运行旧命令。接受风险不代表已确认此前的操作结束。', + 'desktop.chat.changeRecovery.waived': '已按你的选择接受风险并恢复所选位置的修改;此前操作仍未确认结束,旧命令不会自动重跑。', + 'desktop.chat.changeRecovery.changed': '待处理事项已变化。请关闭此确认,重新检查后再选择。', + 'desktop.chat.changeRecovery.loadFailed': '暂时无法读取受保护事项。请重新检查。', + 'desktop.chat.changeRecovery.recheckFailed': '检查未完成,已保留保护状态。请稍后重新检查。', + 'desktop.chat.changeRecovery.resolveFailed': '未能恢复全部所选事项的修改。请重新检查后再选择;不会自动放行新增事项。', 'desktop.chat.draftRestoration.restoring': '正在恢复未发送内容,完成后将自动保存当前输入。', 'desktop.chat.draftRestoration.failed': '未发送内容尚未恢复,当前输入还未保存。请重试恢复后再发送或退出。', 'desktop.chat.draftRestoration.conflict': '已保存内容与恢复期间的输入有冲突。处理前,新输入还无法保存。', diff --git a/apps/buddy/src/modules/tasks/contracts.ts b/apps/buddy/src/modules/tasks/contracts.ts index 95ff3f1ec..63d8b782c 100644 --- a/apps/buddy/src/modules/tasks/contracts.ts +++ b/apps/buddy/src/modules/tasks/contracts.ts @@ -1,6 +1,6 @@ import type { DesktopChatWelcomeItem, DesktopChatWelcomePreference, DesktopTaskPinnedItem, DesktopTaskSidebarSection } from '@buddy-electron/shared/desktopApi' import type { LocalChatApi } from '@buddy-electron/shared/localChatApi' -import type { LocalChangeSetSummary } from '@buddy-shared/changes/changeApi' +import type { ChangeRecoveryResolveRequest, LocalChangeRecoveryItem, LocalChangeSetSummary } from '@buddy-shared/changes/changeApi' import type { LocalChatQueueItem } from '@buddy-shared/conversation/chatQueueApi' import type { BuddyComposerDraftScope } from '@buddy-shared/conversation/composerDraft' import type { BuddyComposerSource } from '@buddy-shared/conversation/composerResource' @@ -227,7 +227,19 @@ export interface TaskExecutionTree { api: LocalChatApi['executions'] } +export interface TaskChangeRecovery { + conversationId: State + items: State + busy: State + error: State<'loadFailed' | 'recheckFailed' | 'resolveFailed' | null> + waived: State + recheck: () => Promise + resolve: (input: ChangeRecoveryResolveRequest) => Promise + dismissWaived: () => void +} + export interface TaskChatWorkspace { + changeRecovery: TaskChangeRecovery executionTree: TaskExecutionTree marks: TaskMarks tree: TaskConversationTree diff --git a/apps/buddy/src/modules/tasks/state/runs/__tests__/useChangeRecovery.spec.ts b/apps/buddy/src/modules/tasks/state/runs/__tests__/useChangeRecovery.spec.ts new file mode 100644 index 000000000..3f0b276c0 --- /dev/null +++ b/apps/buddy/src/modules/tasks/state/runs/__tests__/useChangeRecovery.spec.ts @@ -0,0 +1,99 @@ +import type { LocalChatApi } from '@buddy-electron/shared/localChatApi' +import type { ChangeRecoveryResolveRequest, LocalChangeRecoveryItem } from '@buddy-shared/changes/changeApi' +import { deferred } from '@buddy-tests/deferred' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { effectScope, shallowRef } from 'vue' +import { useChangeRecovery } from '../useChangeRecovery' + +const disposals: (() => void)[] = [] +afterEach(() => disposals.splice(0).forEach(dispose => dispose())) + +function item(invocationId = 'old-operation'): LocalChangeRecoveryItem { + return { invocationId, runId: 'old-run', conversationId: 'old-task', taskTitle: 'Earlier task', occurredAt: '2026-10-15T00:00:00.000Z', affectedResources: ['/workspace/example'], reason: 'MUTATION_INTERRUPTED', status: 'unknown', revision: 'revision-1' } +} + +function setup() { + const conversationId = shallowRef('new-task') + const ready = shallowRef(false) + const recovery: LocalChatApi['changes']['recovery'] = { + list: vi.fn(async () => [item()]), + recheck: vi.fn(async () => [item()]), + resolve: vi.fn(async () => []), + } + const scope = effectScope() + disposals.push(() => scope.stop()) + const state = scope.run(() => useChangeRecovery({ + conversationId, + ready, + api: { + changes: { recovery, onChanged: () => () => {} }, + chat: { onRunEvent: () => () => {} }, + }, + }))! + return { conversationId, ready, recovery, state } +} + +describe('task change recovery', () => { + it('loads actual older blockers for the requesting task without transforming their ownership', async () => { + const f = setup() + f.ready.value = true + await vi.waitFor(() => expect(f.state.items.value).toEqual([item()])) + await f.state.recheck() + expect(f.state.items.value[0]?.conversationId).toBe('old-task') + expect(vi.mocked(f.recovery.recheck).mock.calls[0]?.[0]).toEqual({ conversationId: 'new-task', invocationIds: ['old-operation'] }) + }) + + it('does not apply a late response from the previous task', async () => { + const f = setup() + const late = deferred() + vi.mocked(f.recovery.list).mockImplementation(async input => input.conversationId === 'new-task' ? late.promise : [item('second-task-operation')]) + f.ready.value = true + f.conversationId.value = 'second-task' + await vi.waitFor(() => expect(f.state.items.value[0]?.invocationId).toBe('second-task-operation')) + late.resolve([item()]) + await late.promise + expect(f.state.items.value[0]?.invocationId).toBe('second-task-operation') + }) + + it('applies the server list without clearing new or active blockers outside the fixed decision', async () => { + const f = setup() + f.ready.value = true + await vi.waitFor(() => expect(f.state.items.value).toHaveLength(1)) + const additional = { ...item('additional-operation'), status: 'active' as const } + vi.mocked(f.recovery.resolve).mockResolvedValue([additional]) + const decision: ChangeRecoveryResolveRequest = { + conversationId: 'new-task', + items: [{ invocationId: 'old-operation', revision: 'revision-1' }], + decision: 'waived', + } + expect(await f.state.resolve(decision)).toBe(true) + expect(f.state.items.value).toEqual([additional]) + expect(f.state.waived.value).toBe(true) + expect(vi.mocked(f.recovery.resolve).mock.calls[0]?.[0]).toEqual(decision) + }) + + it('retains protection and reports failure when the selected revision cannot be resolved', async () => { + const f = setup() + f.ready.value = true + await vi.waitFor(() => expect(f.state.items.value).toHaveLength(1)) + vi.mocked(f.recovery.resolve).mockRejectedValue(new Error('stale revision')) + expect(await f.state.resolve({ conversationId: 'new-task', items: [{ invocationId: 'old-operation', revision: 'revision-1' }], decision: 'waived' })).toBe(false) + expect(f.state.items.value).toEqual([item()]) + expect(f.state.waived.value).toBe(false) + expect(f.state.error.value).toBe('resolveFailed') + }) + + it('does not mark another task as waived when a pending decision completes after navigation', async () => { + const f = setup() + const pending = deferred() + f.ready.value = true + await vi.waitFor(() => expect(f.state.items.value).toHaveLength(1)) + vi.mocked(f.recovery.resolve).mockReturnValue(pending.promise) + const resolution = f.state.resolve({ conversationId: 'new-task', items: [{ invocationId: 'old-operation', revision: 'revision-1' }], decision: 'waived' }) + f.conversationId.value = 'another-task' + pending.resolve([]) + expect(await resolution).toBe(false) + expect(f.state.waived.value).toBe(false) + expect(f.state.busy.value).toBe(false) + }) +}) diff --git a/apps/buddy/src/modules/tasks/state/runs/useChangeRecovery.ts b/apps/buddy/src/modules/tasks/state/runs/useChangeRecovery.ts new file mode 100644 index 000000000..6c3f75908 --- /dev/null +++ b/apps/buddy/src/modules/tasks/state/runs/useChangeRecovery.ts @@ -0,0 +1,164 @@ +import type { LocalChatApi } from '@buddy-electron/shared/localChatApi' +import type { ChangeRecoveryResolveRequest, LocalChangeRecoveryItem } from '@buddy-shared/changes/changeApi' +import type { Ref } from 'vue' +import type { TaskChangeRecovery } from '../../contracts' +import { onScopeDispose, readonly, shallowRef, watch } from 'vue' + +export function useChangeRecovery(options: { + conversationId: Readonly> + ready: Readonly> + api: { + changes: Pick + chat: Pick + } +}): TaskChangeRecovery { + const items = shallowRef([]) + const busy = shallowRef(false) + const error = shallowRef(null) + const waived = shallowRef(false) + let generation = 0 + let requestId = 0 + let disposed = false + let timer: ReturnType | undefined + + function scope() { + const conversationId = options.conversationId.value + if (disposed || !options.ready.value || !conversationId) + return null + const currentGeneration = generation + const request = ++requestId + return { + conversationId, + current: () => !disposed && currentGeneration === generation && request === requestId, + } + } + + async function refresh(): Promise { + if (busy.value) + return + const target = scope() + if (!target) + return + try { + const result = await options.api.changes.recovery.list({ conversationId: target.conversationId }) + if (target.current()) { + items.value = result + if (error.value === 'loadFailed') + error.value = null + } + } + catch { + if (target.current()) + error.value = 'loadFailed' + } + } + + async function recheck(): Promise { + if (busy.value) + return + if (!items.value.length) { + await refresh() + return + } + const target = scope() + if (!target) + return + const invocationIds = items.value.map(item => item.invocationId) + busy.value = true + error.value = null + waived.value = false + try { + for (let offset = 0; offset < invocationIds.length; offset += 100) { + const result = await options.api.changes.recovery.recheck({ + conversationId: target.conversationId, + invocationIds: invocationIds.slice(offset, offset + 100), + }) + if (!target.current()) + return + items.value = result + } + } + catch { + if (target.current()) + error.value = 'recheckFailed' + } + finally { + if (target.current()) + busy.value = false + } + } + + async function resolve(input: ChangeRecoveryResolveRequest): Promise { + if (busy.value || input.conversationId !== options.conversationId.value) + return false + const target = scope() + if (!target) + return false + busy.value = true + error.value = null + waived.value = false + try { + const result = await options.api.changes.recovery.resolve(input) + if (!target.current()) + return false + items.value = result + const unresolved = result.some(item => input.items.some(selected => selected.invocationId === item.invocationId)) + waived.value = !unresolved + if (unresolved) + error.value = 'resolveFailed' + return !unresolved + } + catch { + if (target.current()) + error.value = 'resolveFailed' + return false + } + finally { + if (target.current()) { + busy.value = false + scheduleRefresh() + } + } + } + + function scheduleRefresh() { + if (timer || disposed) + return + timer = setTimeout(() => { + timer = undefined + void refresh() + }, 80) + } + + const stopChanges = options.api.changes.onChanged(scheduleRefresh) + const stopEvents = options.api.chat.onRunEvent((event) => { + if (['tool.completed', 'tool.failed', 'run.completed', 'run.failed', 'run.cancelled'].includes(event.type)) + scheduleRefresh() + }) + watch([options.conversationId, options.ready], () => { + generation++ + items.value = [] + busy.value = false + error.value = null + waived.value = false + void refresh() + }, { immediate: true, flush: 'sync' }) + onScopeDispose(() => { + disposed = true + generation++ + clearTimeout(timer) + stopChanges() + stopEvents() + }) + + return { + conversationId: options.conversationId, + items: readonly(items), + busy: readonly(busy), + error: readonly(error), + waived: readonly(waived), + recheck, + resolve, + dismissWaived: () => { waived.value = false }, + } +} diff --git a/apps/buddy/src/modules/tasks/state/useTaskCapability.ts b/apps/buddy/src/modules/tasks/state/useTaskCapability.ts index d47778bf5..c122fed37 100644 --- a/apps/buddy/src/modules/tasks/state/useTaskCapability.ts +++ b/apps/buddy/src/modules/tasks/state/useTaskCapability.ts @@ -26,6 +26,7 @@ import { useActorInputTarget } from '@/modules/tasks/state/composer/useActorInpu import { useChatComposerInteractions } from '@/modules/tasks/state/composer/useChatComposerInteractions' import { useComposerContextOptions } from '@/modules/tasks/state/composer/useComposerContextOptions' import { useComposerResources } from '@/modules/tasks/state/composer/useComposerResources' +import { useChangeRecovery } from '@/modules/tasks/state/runs/useChangeRecovery' import { useChatApprovals } from '@/modules/tasks/state/runs/useChatApprovals' import { useChatContextUsage } from '@/modules/tasks/state/runs/useChatContextUsage' import { useChatExecution } from '@/modules/tasks/state/runs/useChatExecution' @@ -121,6 +122,11 @@ export function useTaskCapability(options: UseTaskCapabilityOptions): TaskCapabi timelineItems, } = runSync + const changeRecovery = useChangeRecovery({ + conversationId: activeConversationId, + ready: computed(() => runtimeSupervisor.runtimeState.value.status === 'ready'), + api: api.localChat, + }) const executionTree = useChatExecutionTree({ conversationId: activeConversationId, branchId: activeBranchId, @@ -574,6 +580,7 @@ export function useTaskCapability(options: UseTaskCapabilityOptions): TaskCapabi runs, }) const workspace = { + changeRecovery, executionTree, marks: taskMarks, tree, diff --git a/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryItems.vue b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryItems.vue new file mode 100644 index 000000000..61e5b60fa --- /dev/null +++ b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryItems.vue @@ -0,0 +1,37 @@ + + + diff --git a/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryNotice.vue b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryNotice.vue new file mode 100644 index 000000000..22226ebb1 --- /dev/null +++ b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChangeRecoveryNotice.vue @@ -0,0 +1,111 @@ + + + diff --git a/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChatWorkspace.vue b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChatWorkspace.vue index 6b3dbc210..049bd8207 100644 --- a/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChatWorkspace.vue +++ b/apps/buddy/src/modules/tasks/widgets/workspace/DesktopChatWorkspace.vue @@ -253,6 +253,7 @@ function openDetailChanges(id: string) {