From a65c861cb2374ec596b336e3300421f9186ae955 Mon Sep 17 00:00:00 2001 From: shanyuhai123 <864299347@qq.com> Date: Tue, 20 Oct 2026 20:05:08 +0800 Subject: [PATCH] =?UTF-8?q?fix(buddy):=20=E9=81=BF=E5=85=8D=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E5=BC=82=E5=B8=B8=E9=98=BB=E5=A1=9E=E6=96=B0=E6=93=8D?= =?UTF-8?q?=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../__tests__/mutation-recovery.e2e.mjs | 173 +++++- .../native/host/__tests__/file_engine.rs | 40 +- .../host/__tests__/file_engine_interrupt.rs | 247 +++++++++ .../host/__tests__/file_engine_recovery.rs | 170 ++++-- .../__tests__/file_engine_windows_protocol.rs | 79 +++ .../host/__tests__/file_writer_windows.rs | 33 ++ .../native/host/src/file_engine/journal.rs | 16 - apps/buddy/native/host/src/file_engine/mod.rs | 4 + .../native/host/src/file_engine/operations.rs | 9 +- .../native/host/src/file_engine/server.rs | 22 +- apps/buddy/native/host/src/file_writer.rs | 4 + .../buddy/native/host/src/file_writer/unix.rs | 4 +- .../native/host/src/file_writer/windows.rs | 42 +- apps/buddy/service/src/BuddyService.ts | 36 +- .../buddyService.integration.spec.ts | 145 ++++- .../mutationConcurrency.integration.spec.ts | 348 ++++++++++++ .../__tests__/mutationConcurrencyFixture.ts | 400 ++++++++++++++ .../mutationContinuation.integration.spec.ts | 72 +++ .../agent/execution/ExecutionCoordinator.ts | 123 +++-- .../execution/ExecutionDelegationService.ts | 7 +- .../execution/ExecutionPresentationService.ts | 10 +- .../__tests__/ExecutionCoordinator.spec.ts | 200 ++++++- .../__tests__/mutationToolExecution.spec.ts | 120 ++++- .../extensions/changeCaptureExtension.ts | 5 + .../sessions/tree/BuddyConversationTree.ts | 9 +- .../__tests__/BuddyConversationTree.spec.ts | 35 +- .../src/changes/MutationCoordinator.ts | 97 ++-- .../src/changes/MutationRecoveryService.ts | 256 +++++++-- .../__tests__/MutationCoordinator.spec.ts | 122 ++++- .../__tests__/MutationRecoveryService.spec.ts | 8 +- .../mutationRecoveryDecisions.spec.ts | 88 ++- .../mutationRecoveryScheduling.spec.ts | 509 ++++++++++++++++++ .../src/changes/mutationEvidenceRepository.ts | 10 +- .../conversations/registerConversationRpc.ts | 6 +- apps/buddy/service/src/events/RunEventLog.ts | 24 +- .../service/src/events/RunEventQueries.ts | 6 +- .../buddy/service/src/events/RunEventStore.ts | 38 +- .../src/events/__tests__/RunEventLog.spec.ts | 13 +- .../__tests__/RunEventQuarantine.spec.ts | 73 +-- .../events/__tests__/RunEventRecovery.spec.ts | 12 +- .../service/src/files/NativeFileEngine.ts | 374 ++++++++++--- .../__tests__/FileResourceCoordinator.spec.ts | 11 +- .../files/__tests__/NativeFileEngine.spec.ts | 505 +++++++++++++++++ .../service/src/runs/RunRecoveryService.ts | 33 +- apps/buddy/service/src/runs/registerRunRpc.ts | 8 +- .../__tests__/shellProcessOwnership.spec.ts | 6 +- .../__tests__/SpaceFileRecovery.spec.ts | 22 +- .../src/storage/__tests__/schema.spec.ts | 5 +- .../service/src/storage/runRepository.ts | 15 + apps/buddy/src/i18n/locales/en-US/chat.ts | 24 +- .../src/i18n/locales/en-US/diagnostics.ts | 4 +- apps/buddy/src/i18n/locales/zh-CN/chat.ts | 24 +- .../src/i18n/locales/zh-CN/diagnostics.ts | 4 +- .../modules/files/explorer/explorerLabels.ts | 8 +- .../executions/BuddyExecutionGroup.vue | 2 +- .../widgets/executions/BuddyExecutionNode.vue | 2 +- .../executions/DesktopExecutionDetail.vue | 2 +- .../widgets/executions/executionLabels.ts | 21 +- .../desktopChangeRecoveryNotice.spec.ts | 16 +- 59 files changed, 4218 insertions(+), 483 deletions(-) create mode 100644 apps/buddy/native/host/__tests__/file_engine_interrupt.rs create mode 100644 apps/buddy/native/host/__tests__/file_writer_windows.rs create mode 100644 apps/buddy/service/src/__tests__/mutationConcurrency.integration.spec.ts create mode 100644 apps/buddy/service/src/__tests__/mutationConcurrencyFixture.ts create mode 100644 apps/buddy/service/src/__tests__/mutationContinuation.integration.spec.ts create mode 100644 apps/buddy/service/src/changes/__tests__/mutationRecoveryScheduling.spec.ts create mode 100644 apps/buddy/service/src/files/__tests__/NativeFileEngine.spec.ts diff --git a/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs b/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs index 114e96d36..406c2bfb0 100644 --- a/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs +++ b/.playwright/scripts/__tests__/mutation-recovery.e2e.mjs @@ -1,12 +1,16 @@ +import { Buffer } from 'node:buffer' +import { once } from 'node:events' +import fs from 'node:fs/promises' +import { createServer } from 'node:http' import path from 'node:path' import { DatabaseSync } from 'node:sqlite' -import { expect, test } from '../fixtures/electron.mjs' +import { expect, test, useSyntheticCredentialStorage } from '../fixtures/electron.mjs' const oldOperation = '11111111-1111-4111-8111-111111111111' const deletedOperation = '22222222-2222-4222-8222-222222222222' const unrelatedOperation = '33333333-3333-4333-8333-333333333333' -function seedRecovery(databasePath) { +function seedRecovery(databasePath, { linkHistory = true, claim } = {}) { const database = new DatabaseSync(databasePath) const now = '2026-10-15T00:00:00.000Z' try { @@ -23,10 +27,10 @@ function seedRecovery(databasePath) { [unrelatedOperation, 'run-unrelated', '/example/shared/report.md'], ]) { database.prepare(`INSERT INTO run_mutation_leases (invocation_id, run_id, segment_id, claims_json, status, reason, created_at, updated_at, lifetime) - VALUES (?, ?, ?, ?, 'hazard', 'MUTATION_INTERRUPTED', ?, ?, 'external')`).run(id, runId, `segment-${id}`, JSON.stringify([{ kind: 'file', canonicalPath: resource }]), now, now) + VALUES (?, ?, ?, ?, 'hazard', 'MUTATION_INTERRUPTED', ?, ?, 'external')`).run(id, runId, `segment-${id}`, JSON.stringify([id === oldOperation && claim ? claim : { kind: 'file', canonicalPath: resource }]), now, now) database.prepare('INSERT INTO run_mutation_processes (invocation_id, process_id, settlement, updated_at) VALUES (?, ?, \'unknown\', ?)') .run(id, JSON.stringify({ kind: 'legacy-process', pid: 123456, job: 'not-user-facing', checkpoint: 'not-user-facing' }), now) - if (id !== unrelatedOperation) { + if (linkHistory && id !== unrelatedOperation) { database.prepare(`INSERT INTO run_mutation_evidence (invocation_id, kind, outcome, reason, actor, related_run_id, created_at) VALUES (?, 'observed', 'unknown', 'WRITE_BLOCKED', 'system', 'run-blocked', ?)`).run(id, now) } @@ -41,13 +45,15 @@ function readRecovery(databasePath) { return { leases: database.prepare('SELECT invocation_id, status, reason FROM run_mutation_leases ORDER BY invocation_id').all(), decisions: database.prepare('SELECT invocation_id, kind, outcome, actor FROM run_mutation_evidence WHERE kind = \'waived\' ORDER BY sequence').all(), - runs: database.prepare('SELECT id FROM runs ORDER BY id').all(), + runs: database.prepare('SELECT id, conversation_id, status FROM runs ORDER BY id').all(), + associations: database.prepare('SELECT invocation_id, related_run_id, reason FROM run_mutation_evidence WHERE related_run_id IS NOT NULL ORDER BY sequence').all(), + processes: database.prepare('SELECT invocation_id, settlement FROM run_mutation_processes ORDER BY invocation_id').all(), } } finally { database.close() } } -test('task recovery discovers older blockers and only waives the explicitly confirmed set', async ({ buddy }, testInfo) => { +test('task review preserves historical warnings and only acknowledges the explicitly confirmed set', async ({ buddy }, testInfo) => { test.setTimeout(120_000) const instance = await buddy.createInstance('mutation-recovery') const databasePath = path.join(instance.home, 'buddy/buddy.sqlite3') @@ -65,13 +71,14 @@ test('task recovery discovers older blockers and only waives the explicitly conf }) await page.locator('[data-task-id="blocked"] .desktop-task-sidebar__task').click() const card = () => page.locator('.desktop-change-recovery:visible') + await expect(card()).toContainText('这些历史记录不会阻止新操作') await expect(card()).toContainText('此前的整理任务') await expect(card()).toContainText('已删除的任务') await expect(card()).toContainText('/example/shared/report.md') await expect(card()).not.toContainText('其他任务') await expect(card()).not.toContainText(/123456|not-user-facing|PID|checkpoint/i) await expect(card().getByRole('button', { name: /停止|结束/ })).toHaveCount(0) - await expect(card().getByRole('button', { name: '接受风险并恢复修改', exact: true })).toHaveCount(0) + await expect(card().getByRole('button', { name: '标记已核对并接受风险', exact: true })).toHaveCount(0) const initial = await page.evaluate(() => window.lexoraDesktop.localChat.changes.recovery.list({ conversationId: 'blocked' })) expect(initial.map(item => item.invocationId).sort()).toEqual([oldOperation, deletedOperation]) @@ -94,20 +101,20 @@ test('task recovery discovers older blockers and only waives the explicitly conf async function review() { await card().getByRole('button', { name: '更多选项', exact: true }).click() - await page.locator('.n-dropdown-menu:visible').getByText('接受风险并恢复修改', { exact: true }).click() + await page.locator('.n-dropdown-menu:visible').getByText('标记已核对并接受风险', { exact: true }).click() } await review() - const confirmation = page.locator('.n-dialog').filter({ hasText: '接受风险并恢复修改？' }) - await expect(confirmation).toContainText('仅对以下 2 项恢复修改') - await expect(confirmation).toContainText('不会重新运行旧命令') + const confirmation = page.locator('.n-dialog').filter({ hasText: '标记已核对并接受风险？' }) + await expect(confirmation).toContainText('仅标记以下 2 项为已核对') + await expect(confirmation).toContainText('不会重新运行旧操作') await page.screenshot({ path: path.join(instance.artifactDirectory, 'recovery-confirmation.png'), animations: 'disabled' }) await confirmation.getByRole('button', { name: '取消', exact: true }).click() expect(readRecovery(databasePath).decisions).toEqual([]) await review() - await confirmation.getByRole('button', { name: '接受风险并恢复修改', exact: true }).click() + await confirmation.getByRole('button', { name: '标记已核对并接受风险', exact: true }).click() await expect(confirmation).toBeHidden() await expect(card()).toHaveCount(0) - await expect(page.getByText('已按你的选择接受风险并恢复所选位置的修改；此前操作仍未确认结束，旧命令不会自动重跑。', { exact: true })).toBeVisible() + await expect(page.getByText('已记录你对所选事项的核对和风险确认；此前操作仍未确认结束，历史记录会保留，旧操作不会自动重跑。', { exact: true })).toBeVisible() await page.screenshot({ path: path.join(instance.artifactDirectory, 'recovery-waived.png'), animations: 'disabled' }) const resolved = readRecovery(databasePath) expect(resolved.leases).toEqual([ @@ -127,3 +134,143 @@ test('task recovery discovers older blockers and only waives the explicitly conf expect(diagnostics.console.filter(item => item.type === 'pageerror')).toEqual([]) await testInfo.attach('recovery-audit', { body: JSON.stringify(resolved, null, 2), contentType: 'application/json' }) }) + +for (const claimKind of ['file', 'host']) { + test(`a successful ${claimKind} overlap shows historical review without blocking another write or replaying after restart`, async ({ buddy }, testInfo) => { + test.setTimeout(180_000) + const requests = [] + const serverErrors = [] + const server = createServer((request, response) => { + void (async () => { + if (request.method !== 'POST' || request.url !== '/v1/chat/completions') { + response.writeHead(404).end() + return + } + const chunks = [] + for await (const chunk of request) + chunks.push(chunk) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) + requests.push(body) + const current = body.messages.slice(body.messages.findLastIndex(message => message.role === 'user') + 1) + const completedWrite = current.some(message => message.tool_calls?.some(call => call.function.name === 'write')) + const shouldWrite = body.tools?.some(tool => tool.function.name === 'write') && !completedWrite + const content = JSON.stringify(body.messages.findLast(message => message.role === 'user')).includes('second-write') + ? 'second-write\n' + : 'first-write\n' + const delta = shouldWrite + ? { tool_calls: [{ index: 0, id: `write-${requests.length}`, type: 'function', function: { name: 'write', arguments: JSON.stringify({ path: 'report.md', content }) } }] } + : { content: '本次写入已完成。' } + const common = { id: `recovery-${requests.length}`, model: body.model, object: 'chat.completion.chunk', created: 1 } + response.writeHead(200, { 'content-type': 'text/event-stream' }) + response.write(`data: ${JSON.stringify({ ...common, choices: [{ index: 0, delta: { role: 'assistant', ...delta }, finish_reason: null }] })}\n\n`) + response.write(`data: ${JSON.stringify({ ...common, choices: [{ index: 0, delta: {}, finish_reason: shouldWrite ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 32, completion_tokens: 12, total_tokens: 44 } })}\n\n`) + response.end('data: [DONE]\n\n') + })().catch((error) => { + serverErrors.push(String(error)) + response.writeHead(500).end('Local Provider scenario failed') + }) + }) + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const instance = await buddy.createInstance(`mutation-overlap-${claimKind}`) + const databasePath = path.join(instance.home, 'buddy/buddy.sqlite3') + const workspace = path.join(instance.home, 'buddy/conversations/blocked/workspace') + const target = path.join(workspace, 'report.md') + try { + await instance.launch() + await instance.stop() + await fs.mkdir(workspace, { recursive: true }) + seedRecovery(databasePath, { + linkHistory: false, + claim: claimKind === 'host' ? { kind: 'host' } : { kind: 'file', canonicalPath: target }, + }) + let desktop = await instance.launch() + await useSyntheticCredentialStorage(desktop) + let page = desktop.page + await page.evaluate(async (baseUrl) => { + const api = window.lexoraDesktop.localChat + const providers = api.providers + await providers.upsertCustom({ id: 'recovery-fixture', displayName: 'Recovery fixture', api: 'openai-completions', baseUrl, enabled: true, models: [{ id: 'recovery-fixture', name: 'Recovery fixture', input: ['text'], reasoning: false, contextWindow: 128000, maxTokens: 1024 }] }) + const stop = providers.onAuthChallenge((challenge) => { + if (challenge.providerId === 'recovery-fixture' && challenge.type === 'secret') + void providers.respondToAuth(challenge.challengeId, 'offline-fixture-key') + }) + try { + await providers.login('recovery-fixture', 'api_key') + } + finally { stop() } + await providers.setDefaultModel({ providerId: 'recovery-fixture', modelId: 'recovery-fixture', reasoning: null }) + const { defaultBuddyId } = await api.buddies.list() + await api.conversations.addMember('blocked', defaultBuddyId) + await api.conversations.setBuddy('blocked', defaultBuddyId) + }, `http://127.0.0.1:${server.address().port}/v1`) + await page.locator('[data-task-id="blocked"] .desktop-task-sidebar__task').click() + const card = () => page.locator('.desktop-change-recovery:visible') + const list = () => page.evaluate(() => window.lexoraDesktop.localChat.changes.recovery.list({ conversationId: 'blocked' })) + expect(await list()).toEqual([]) + expect(readRecovery(databasePath).associations).toEqual([]) + await expect(card()).toHaveCount(0) + await page.screenshot({ path: path.join(instance.artifactDirectory, 'overlap-before.png'), animations: 'disabled' }) + + async function send(content) { + await page.locator('.desktop-chat-composer__prosemirror:visible').fill(`请将 report.md 写为 ${content}。`) + const button = page.getByRole('button', { name: '发送消息', exact: true }) + await expect(button).toBeEnabled() + await button.click() + } + async function settled(count, content) { + await expect.poll(() => readRecovery(databasePath).runs.filter(run => run.conversation_id === 'blocked' && run.id !== 'run-blocked').map(run => run.status)).toEqual(Array.from({ length: count }).fill('completed')) + expect(await fs.readFile(target, 'utf8')).toBe(content) + } + await send('first-write') + await settled(1, 'first-write\n') + await expect(card()).toContainText('此前操作有待核对') + await expect(card()).toContainText('这些历史记录不会阻止新操作') + await expect(card()).toContainText('此前的整理任务') + await expect(card()).toContainText('暂时无法确认此前的操作是否已停止') + await expect(card()).not.toContainText('已删除的任务') + await expect(page.locator('.n-dialog:visible')).toHaveCount(0) + expect((await list()).map(item => [item.invocationId, item.status])).toEqual([[oldOperation, 'unknown']]) + await page.locator('.desktop-chat-composer__prosemirror:visible').fill('请将 report.md 写为 second-write。') + await expect(page.getByRole('button', { name: '发送消息', exact: true })).toBeEnabled() + for (const theme of ['light', 'dark']) { + await page.evaluate(theme => window.lexoraDesktop.themes.request({ action: 'preference', preference: { id: `lexora.themes.classic-${theme}` } }), theme) + await expect(page.locator('html')).toHaveAttribute('data-buddy-theme', theme) + await page.screenshot({ path: path.join(instance.artifactDirectory, `overlap-${theme}.png`), animations: 'disabled' }) + } + await send('second-write') + await settled(2, 'second-write\n') + await expect(card()).toBeVisible() + const completed = readRecovery(databasePath) + expect(completed.decisions).toEqual([]) + expect(completed.leases.find(lease => lease.invocation_id === oldOperation)).toMatchObject({ status: 'hazard', reason: 'MUTATION_INTERRUPTED' }) + expect(completed.processes.find(process => process.invocation_id === oldOperation).settlement).toBe('unknown') + expect(completed.associations).toHaveLength(2) + expect(completed.associations.every(item => item.invocation_id === oldOperation && item.reason === 'RESOURCE_OVERLAP')).toBe(true) + expect(new Set(completed.associations.map(item => item.related_run_id)).size).toBe(2) + expect(desktop.diagnostics.console.filter(item => item.type === 'pageerror')).toEqual([]) + const requestCount = requests.length + await instance.stop() + desktop = await instance.launch() + await useSyntheticCredentialStorage(desktop) + page = desktop.page + await page.locator('[data-task-id="blocked"] .desktop-task-sidebar__task').click() + await expect(card()).toContainText('此前的整理任务') + expect(readRecovery(databasePath).associations).toEqual(completed.associations) + expect(requests).toHaveLength(requestCount) + expect(await fs.readFile(target, 'utf8')).toBe('second-write\n') + expect(serverErrors).toEqual([]) + expect(desktop.diagnostics.console.filter(item => item.type === 'pageerror')).toEqual([]) + await page.screenshot({ path: path.join(instance.artifactDirectory, 'overlap-restarted.png'), animations: 'disabled' }) + const toolMessages = [...new Map(requests.flatMap(body => body.messages.filter(message => message.role === 'tool')).map(message => [message.tool_call_id, message])).values()] + const auditPath = testInfo.outputPath('overlap-audit.json') + await fs.writeFile(auditPath, JSON.stringify({ claimKind, completed, recovery: await list(), notice: await card().textContent(), modelToolResults: toolMessages }, null, 2)) + await testInfo.attach('overlap-audit', { path: auditPath, contentType: 'application/json' }) + } + finally { + await instance.stop() + server.closeAllConnections() + await new Promise(resolve => server.close(resolve)) + } + }) +} diff --git a/apps/buddy/native/host/__tests__/file_engine.rs b/apps/buddy/native/host/__tests__/file_engine.rs index 8c98999a0..f54f7165d 100644 --- a/apps/buddy/native/host/__tests__/file_engine.rs +++ b/apps/buddy/native/host/__tests__/file_engine.rs @@ -227,7 +227,7 @@ fn shared_roots_and_hardlinks_share_the_same_resource_lease() { } #[test] -fn restart_preserves_unknown_outcomes_and_blocks_replay_until_reviewed() { +fn restart_preserves_unknown_outcomes_without_blocking_new_operations() { let directory = tempfile::tempdir().unwrap(); let root = directory.path().join("root"); std::fs::create_dir(&root).unwrap(); @@ -263,25 +263,32 @@ fn restart_preserves_unknown_outcomes_and_blocks_replay_until_reviewed() { "root", "a", json!({"kind":"rename","destination":"b"}) - )["error"], - "FILE_RECOVERY_REQUIRED" + )["result"]["status"], + "applied" ); - assert!(root.join("a").exists()); - assert!(!root.join("b").exists()); + assert!(!root.join("a").exists()); + assert!(root.join("b").exists()); assert_eq!( - engine.call(json!({"type":"acknowledge","root":"root","operationId":operation}))["result"] - ["status"], - "acknowledged" + engine.mutate(&operation, "root", "a", json!({"kind":"createFile"}))["result"]["status"], + "needs-recovery" ); + assert!(!root.join("a").exists()); assert_eq!( engine.mutate( - &id(), + &operation, "root", "a", - json!({"kind":"rename","destination":"b"}) - )["result"]["status"], - "applied" + json!({"kind":"write","text":"replay"}) + )["error"], + "FILE_OPERATION_ID_CONFLICT" ); + assert_eq!( + engine.call(json!({"type":"acknowledge","root":"root","operationId":operation}))["result"] + ["status"], + "acknowledged" + ); + assert!(!root.join("a").exists()); + assert!(root.join("b").exists()); } #[cfg(unix)] @@ -428,6 +435,15 @@ fn queued_mutation_can_be_cancelled_without_modifying_the_file() { assert_eq!(engine.response(queued)["error"], "FILE_OPERATION_CANCELLED"); assert_eq!(engine.response(cancel)["result"], Value::Null); assert_eq!(std::fs::read(root.join("a")).unwrap(), b"retained"); + assert_eq!( + engine.call(json!({"type":"release","leaseId":lease,"hazard":false}))["result"], + Value::Null + ); + assert_eq!( + engine.call(json!({"type":"query","operationId":operation}))["result"], + Value::Null + ); + assert_eq!(std::fs::read(root.join("a")).unwrap(), b"retained"); } #[path = "file_engine_recovery.rs"] diff --git a/apps/buddy/native/host/__tests__/file_engine_interrupt.rs b/apps/buddy/native/host/__tests__/file_engine_interrupt.rs new file mode 100644 index 000000000..6173393d9 --- /dev/null +++ b/apps/buddy/native/host/__tests__/file_engine_interrupt.rs @@ -0,0 +1,247 @@ +use serde_json::{Value, json}; +use std::{ + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Child, ChildStdin, Command, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver}, + }, + time::{Duration, Instant}, +}; + +const HELPER: &str = "file_engine::interruption::helper"; +const DIRECTORY: &str = "LEXORA_FILE_TEST_DIRECTORY"; +const CHECKPOINT: &str = "LEXORA_FILE_TEST_CHECKPOINT"; +static REACHED: AtomicBool = AtomicBool::new(false); + +pub(super) fn checkpoint(stage: &str) { + if std::env::var(CHECKPOINT).as_deref() != Ok(stage) || REACHED.swap(true, Ordering::SeqCst) { + return; + } + let directory = PathBuf::from(std::env::var_os(DIRECTORY).unwrap()); + std::fs::write(directory.join("checkpoint"), stage).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !directory.join("continue").exists() { + assert!(Instant::now() < deadline, "Checkpoint was not resumed"); + std::thread::sleep(Duration::from_millis(5)); + } +} + +#[test] +#[ignore = "subprocess entry, invoked by interruption tests"] +fn helper() { + let directory = PathBuf::from(std::env::var_os(DIRECTORY).unwrap()); + super::serve(&directory.join("journal")).unwrap(); +} + +struct Engine { + child: Child, + input: ChildStdin, + output: Receiver, + sequence: u64, +} + +impl Engine { + fn new(directory: &Path, stage: Option<&str>) -> Self { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", HELPER, "--ignored", "--nocapture"]) + .env(DIRECTORY, directory) + .env_remove(CHECKPOINT) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + if let Some(stage) = stage { + command.env(CHECKPOINT, stage); + } + let mut child = command.spawn().unwrap(); + let input = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, output) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + let Ok(value) = serde_json::from_str(&line) else { + continue; + }; + if sender.send(value).is_err() { + break; + } + } + }); + let mut engine = Self { + child, + input, + output, + sequence: 0, + }; + assert_eq!( + engine.output.recv_timeout(Duration::from_secs(10)).unwrap(), + json!({"event":"ready","version":2}) + ); + assert!( + engine + .call(json!({"type":"register","key":"root","path":directory.join("root")})) + .get("result") + .is_some() + ); + engine + } + + fn send(&mut self, command: Value) -> u64 { + self.sequence += 1; + writeln!( + self.input, + "{}", + json!({"id":self.sequence,"command":command}) + ) + .unwrap(); + self.sequence + } + + fn response(&self, id: u64) -> Value { + loop { + let value = self.output.recv_timeout(Duration::from_secs(10)).unwrap(); + if value["id"] == id { + return value; + } + assert!(value.get("event").is_some(), "Unexpected response: {value}"); + } + } + + fn call(&mut self, command: Value) -> Value { + let id = self.send(command); + self.response(id) + } + + fn kill(&mut self) { + self.child.kill().unwrap(); + assert!(!self.child.wait().unwrap().success()); + } +} + +impl Drop for Engine { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn mutation(operation: &str, text: &str) -> Value { + json!({"type":"mutate","request":{"operationId":operation,"root":"root","path":"note","action":{"kind":"write","text":text}}}) +} + +fn wait_for_checkpoint(directory: &Path, stage: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if std::fs::read_to_string(directory.join("checkpoint")).is_ok_and(|value| value == stage) { + return; + } + assert!(Instant::now() < deadline, "Helper did not reach {stage}"); + std::thread::sleep(Duration::from_millis(5)); + } +} + +fn interrupted_write(stage: &str, expected: &str) { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("root"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("note"), "original").unwrap(); + std::fs::write(root.join("note.backup"), "retained backup").unwrap(); + let operation = uuid::Uuid::new_v4().to_string(); + let request = mutation(&operation, "committed"); + let mut engine = Engine::new(directory.path(), Some(stage)); + engine.send(request.clone()); + wait_for_checkpoint(directory.path(), stage); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + expected + ); + engine.kill(); + drop(engine); + let db = + rusqlite::Connection::open(directory.path().join("journal/operations.sqlite")).unwrap(); + let record: String = db + .query_row( + "SELECT record FROM operations WHERE id=?", + [&operation], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + serde_json::from_str::(&record).unwrap()["status"], + "executing" + ); + drop(db); + let mut engine = Engine::new(directory.path(), None); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + expected + ); + let unknown = engine.call(json!({"type":"query","operationId":operation}))["result"].clone(); + assert_eq!(unknown["status"], "needs-recovery"); + assert_eq!(unknown["error"], "FILE_OPERATION_INTERRUPTED"); + assert_eq!(engine.call(request.clone())["result"], unknown); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + expected + ); + assert_eq!( + engine.call(mutation(&operation, "different"))["error"], + "FILE_OPERATION_ID_CONFLICT" + ); + assert_eq!( + engine.call(mutation(&uuid::Uuid::new_v4().to_string(), "continued"))["result"]["status"], + "applied" + ); + assert_eq!(engine.call(request)["result"], unknown); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "continued" + ); + assert_eq!( + std::fs::read_to_string(root.join("note.backup")).unwrap(), + "retained backup" + ); +} + +#[test] +fn killed_helper_before_side_effect_keeps_original_and_allows_a_new_operation() { + interrupted_write("executing", "original"); +} + +#[test] +fn killed_helper_after_commit_keeps_new_content_without_replaying_the_old_operation() { + interrupted_write("completed", "committed"); +} + +#[test] +fn cancellation_does_not_finish_an_inflight_commit_or_start_the_next_write_early() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("root"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("note"), "original").unwrap(); + let mut engine = Engine::new(directory.path(), Some("completed")); + let operation = uuid::Uuid::new_v4().to_string(); + let first = engine.send(mutation(&operation, "committed")); + wait_for_checkpoint(directory.path(), "completed"); + let cancel = engine.send(json!({"type":"cancel","operationId":operation})); + let second = engine.send(mutation(&uuid::Uuid::new_v4().to_string(), "continued")); + assert!(matches!( + engine.output.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "committed" + ); + std::fs::write(directory.path().join("continue"), "").unwrap(); + assert_eq!(engine.response(first)["result"]["status"], "applied"); + assert_eq!(engine.response(cancel)["result"], Value::Null); + assert_eq!(engine.response(second)["result"]["status"], "applied"); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "continued" + ); +} diff --git a/apps/buddy/native/host/__tests__/file_engine_recovery.rs b/apps/buddy/native/host/__tests__/file_engine_recovery.rs index 6bf84f7df..dad4b679c 100644 --- a/apps/buddy/native/host/__tests__/file_engine_recovery.rs +++ b/apps/buddy/native/host/__tests__/file_engine_recovery.rs @@ -18,7 +18,7 @@ fn interrupt(journal: &std::path::Path, records: &mut [Value]) { } #[test] -fn protection_overlays_interrupted_operations_without_granting_write_and_survives_restart() { +fn legacy_unknown_records_and_protection_remain_diagnostic_after_restart() { let directory = tempfile::tempdir().unwrap(); let root = directory.path().join("root"); let journal = directory.path().join("journal"); @@ -62,7 +62,6 @@ fn protection_overlays_interrupted_operations_without_granting_write_and_survive json!({"type":"protect", "leaseId":lease, "claims":[{"path":source,"subtree":false}]}); let mut engine = Engine::new(&journal); engine.register("root", &root); - assert_eq!(engine.call(json!({"type":"acquire", "leaseId":lease, "external":true, "claims":[{"root":"root", "path":"note", "subtree":false}]}))["error"], "FILE_RECOVERY_REQUIRED"); assert_eq!(engine.call(protection.clone())["result"], Value::Null); assert_eq!(engine.call(protection.clone())["result"], Value::Null); assert_eq!( @@ -81,42 +80,121 @@ fn protection_overlays_interrupted_operations_without_granting_write_and_survive let mut engine = Engine::new(&journal); engine.register("root", &root); assert_eq!(engine.call(protection)["result"], Value::Null); - let acknowledged = - engine.call(json!({"type":"acknowledge", "root":"root", "operationId":operation})); - assert_eq!(acknowledged["result"]["status"], "acknowledged"); - assert_eq!( - engine.call(json!({"type":"acknowledge", "root":"root", "operationId":operation}))["result"], - acknowledged["result"] - ); + let unknown = engine.call(json!({"type":"query", "operationId":operation}))["result"].clone(); + assert_eq!(unknown["status"], "needs-recovery"); assert_eq!(engine.call(json!({"type":"mutate", "request":{"operationId":id(), "leaseId":lease, "root":"root", "path":"note", "action":{"kind":"write","text":"unsafe"}}}))["error"], "FILE_LEASE_INVALID"); assert_eq!( engine.mutate( &id(), "root", "note", - json!({"kind":"write","text":"unsafe"}) - )["error"], - "FILE_RECOVERY_REQUIRED" + json!({"kind":"write","text":"allowed"}) + )["result"]["status"], + "applied" ); assert_eq!( std::fs::read_to_string(root.join("note")).unwrap(), - "retained" - ); - assert_eq!( - engine.call(json!({"type":"release", "leaseId":lease, "hazard":false}))["result"], - Value::Null + "allowed" ); assert_eq!( engine.mutate( - &id(), + &operation, "root", "note", - json!({"kind":"write","text":"allowed"}) - )["result"]["status"], - "applied" + json!({"kind":"write","text":"retained"}) + )["result"], + unknown + ); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "allowed" + ); + assert_eq!( + engine.call(json!({"type":"leases"}))["result"], + json!([lease]) ); } +#[test] +fn protection_does_not_release_or_expand_an_active_lease() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("root"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("note"), "original").unwrap(); + let mut engine = Engine::new(&directory.path().join("journal")); + engine.register("root", &root); + let lease = id(); + assert_eq!(engine.call(json!({"type":"acquire","leaseId":lease,"external":true,"claims":[{"root":"root","path":"note","subtree":false}]}))["result"], Value::Null); + assert_eq!( + engine.call( + json!({"type":"protect","leaseId":lease,"claims":[{"path":root,"subtree":true}]}) + )["result"], + Value::Null + ); + assert_eq!(engine.call(json!({"type":"mutate","request":{"operationId":id(),"leaseId":lease,"root":"root","path":"other","action":{"kind":"write","text":"invalid"}}}))["error"], "FILE_LEASE_INVALID"); + let operation = id(); + let queued = engine.send(json!({"type":"mutate","request":{"operationId":operation,"root":"root","path":"note","action":{"kind":"write","text":"continued"}}})); + let query = engine.send(json!({"type":"query","operationId":operation})); + assert_eq!(engine.response(query)["result"], Value::Null); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "original" + ); + let release = engine.send(json!({"type":"release","leaseId":lease,"hazard":true})); + assert_eq!(engine.response(release)["result"], Value::Null); + assert_eq!(engine.response(queued)["result"]["status"], "applied"); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "continued" + ); + assert_eq!( + engine.call(json!({"type":"leases"}))["result"], + json!([lease]) + ); + assert_eq!(engine.call(json!({"type":"mutate","request":{"operationId":id(),"leaseId":lease,"root":"root","path":"note","action":{"kind":"write","text":"invalid"}}}))["error"], "FILE_LEASE_INVALID"); + assert!(!root.join("other").exists()); +} + +#[test] +fn restart_retains_external_lease_diagnostics_without_restoring_ownership() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("root"); + let journal = directory.path().join("journal"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("note"), "original").unwrap(); + let old = id(); + let mut engine = Engine::new(&journal); + engine.register("root", &root); + assert_eq!(engine.call(json!({"type":"acquire","leaseId":old,"external":true,"claims":[{"root":"root","path":"note","subtree":false}]}))["result"], Value::Null); + drop(engine); + let mut engine = Engine::new(&journal); + engine.register("root", &root); + assert_eq!( + engine.call(json!({"type":"leases"}))["result"], + json!([old]) + ); + assert_eq!(engine.call(json!({"type":"mutate","request":{"operationId":id(),"leaseId":old,"root":"root","path":"note","action":{"kind":"write","text":"invalid"}}}))["error"], "FILE_LEASE_INVALID"); + let fresh = id(); + assert_eq!(engine.call(json!({"type":"acquire","leaseId":fresh,"external":true,"claims":[{"root":"root","path":"note","subtree":false}]}))["result"], Value::Null); + assert_eq!(engine.call(json!({"type":"mutate","request":{"operationId":id(),"leaseId":fresh,"root":"root","path":"note","action":{"kind":"write","text":"continued"}}}))["result"]["status"], "applied"); + assert_eq!( + std::fs::read_to_string(root.join("note")).unwrap(), + "continued" + ); + assert_eq!( + engine.call(json!({"type":"release","leaseId":fresh,"hazard":false}))["result"], + Value::Null + ); + drop(engine); + let db = rusqlite::Connection::open(journal.join("operations.sqlite")).unwrap(); + let hazard: bool = db + .query_row("SELECT hazard FROM leases WHERE id=?", [&old], |row| { + row.get(0) + }) + .unwrap(); + assert!(hazard); +} + #[test] fn protection_preserves_prior_claims_and_does_not_require_paths_to_exist() { let directory = tempfile::tempdir().unwrap(); @@ -159,7 +237,7 @@ fn protection_preserves_prior_claims_and_does_not_require_paths_to_exist() { #[cfg(unix)] #[test] -fn first_acknowledgement_rejects_replaced_history_root_and_keeps_the_hazard() { +fn acknowledgement_rejects_replaced_root_without_blocking_a_new_capability() { let directory = tempfile::tempdir().unwrap(); let root = directory.path().join("root"); let journal = directory.path().join("journal"); @@ -200,13 +278,13 @@ fn first_acknowledgement_rejects_replaced_history_root_and_keeps_the_hazard() { &id(), "replacement-probe", "note", - json!({"kind":"write","text":"unsafe"}) - )["error"], - "FILE_RECOVERY_REQUIRED" + json!({"kind":"write","text":"allowed"}) + )["result"]["status"], + "applied" ); assert_eq!( std::fs::read_to_string(root.join("note")).unwrap(), - "replacement" + "allowed" ); assert_eq!( std::fs::read_to_string(retained.join("note")).unwrap(), @@ -256,13 +334,13 @@ fn acknowledgement_requires_source_destination_and_staging_in_the_bound_root() { &id(), "root", "note", - json!({"kind":"write","text":"unsafe"}) - )["error"], - "FILE_RECOVERY_REQUIRED" + json!({"kind":"write","text":"allowed"}) + )["result"]["status"], + "applied" ); assert_eq!( std::fs::read_to_string(root.join("note")).unwrap(), - "retained" + "allowed" ); assert_eq!(std::fs::read_to_string(outside).unwrap(), "outside"); } @@ -292,6 +370,38 @@ fn rejected_creation_has_no_recovery_hazard_or_temporary_file() { ); } +#[test] +fn rejected_hardlink_save_preserves_both_files_without_poisoning_later_writes() { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path().join("root"); + std::fs::create_dir(&root).unwrap(); + let path = root.join("note.txt"); + let alias = root.join("alias.txt"); + std::fs::write(&path, "original").unwrap(); + std::fs::hard_link(&path, &alias).unwrap(); + let mut engine = Engine::new(&directory.path().join("journal")); + engine.register("root", &root); + let rejected = engine.mutate( + &id(), + "root", + "note.txt", + json!({"kind":"write","text":"rejected"}), + ); + assert_eq!(rejected["result"]["status"], "not-applied", "{rejected}"); + assert_eq!(rejected["result"]["error"], "FILE_IO_FAILED", "{rejected}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "original"); + assert_eq!(std::fs::read_to_string(&alias).unwrap(), "original"); + std::fs::remove_file(alias).unwrap(); + let continued = engine.mutate( + &id(), + "root", + "note.txt", + json!({"kind":"write","text":"continued"}), + ); + assert_eq!(continued["result"]["status"], "applied", "{continued}"); + assert_eq!(std::fs::read_to_string(path).unwrap(), "continued"); +} + #[test] fn occupied_staging_file_is_preserved_and_does_not_create_a_hazard() { let directory = tempfile::tempdir().unwrap(); diff --git a/apps/buddy/native/host/__tests__/file_engine_windows_protocol.rs b/apps/buddy/native/host/__tests__/file_engine_windows_protocol.rs index f1d876125..f6c88e245 100644 --- a/apps/buddy/native/host/__tests__/file_engine_windows_protocol.rs +++ b/apps/buddy/native/host/__tests__/file_engine_windows_protocol.rs @@ -2,6 +2,85 @@ use super::{Engine, id}; use serde_json::json; use std::fs; +#[test] +fn shared_read_handle_does_not_leave_a_recovery_hazard_after_release() { + rejected_existing_write_can_continue(false); +} + +#[test] +fn readonly_attribute_does_not_leave_a_recovery_hazard_after_restoration() { + rejected_existing_write_can_continue(true); +} + +fn rejected_existing_write_can_continue(readonly: bool) { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("root"); + fs::create_dir(&root).unwrap(); + let path = root.join("note.txt"); + fs::write(&path, "original").unwrap(); + let permissions = fs::metadata(&path).unwrap().permissions(); + let occupied = if readonly { + let mut restricted = permissions.clone(); + restricted.set_readonly(true); + fs::set_permissions(&path, restricted).unwrap(); + None + } else { + Some( + fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ) + .open(&path) + .unwrap(), + ) + }; + let journal = temp.path().join("journal"); + let mut engine = Engine::new(&journal); + engine.register("space", &root); + let operation = id(); + let action = json!({"kind":"write","text":"must-not-be-replayed"}); + let rejected = engine.mutate(&operation, "space", "note.txt", action.clone()); + drop(occupied); + fs::set_permissions(&path, permissions).unwrap(); + assert_eq!(rejected["result"]["status"], "not-applied", "{rejected}"); + assert_eq!(rejected["result"]["error"], "FILE_IO_FAILED", "{rejected}"); + assert_eq!(fs::read_to_string(&path).unwrap(), "original"); + assert_eq!(fs::read_dir(&root).unwrap().count(), 1); + let duplicate = engine.mutate(&operation, "space", "note.txt", action); + assert_eq!(duplicate["result"], rejected["result"]); + assert_eq!(fs::read_to_string(&path).unwrap(), "original"); + let continued = engine.mutate( + &id(), + "space", + "note.txt", + json!({"kind":"write","text":"after-release"}), + ); + assert_eq!(continued["result"]["status"], "applied", "{continued}"); + assert_eq!(fs::read_to_string(&path).unwrap(), "after-release"); + drop(engine); + let mut engine = Engine::new(&journal); + engine.register("space", &root); + let history = engine.call(json!({"type":"history","root":"space"})); + assert!( + history["result"] + .as_array() + .unwrap() + .iter() + .all(|record| record["status"] != "needs-recovery") + ); + let continued = engine.mutate( + &id(), + "space", + "note.txt", + json!({"kind":"write","text":"after-restart"}), + ); + assert_eq!(continued["result"]["status"], "applied", "{continued}"); + assert_eq!(fs::read_to_string(&path).unwrap(), "after-restart"); + assert_eq!(fs::read_dir(&root).unwrap().count(), 1); +} + #[test] fn rejects_invalid_names_before_creating_an_operation() { let temp = tempfile::tempdir().unwrap(); diff --git a/apps/buddy/native/host/__tests__/file_writer_windows.rs b/apps/buddy/native/host/__tests__/file_writer_windows.rs new file mode 100644 index 000000000..b0d2203bb --- /dev/null +++ b/apps/buddy/native/host/__tests__/file_writer_windows.rs @@ -0,0 +1,33 @@ +use super::{WriteError, replacement_failure}; +use windows_sys::Win32::Foundation::{ + ERROR_ACCESS_DENIED, ERROR_LOCK_VIOLATION, ERROR_SHARING_VIOLATION, + ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + ERROR_UNABLE_TO_REMOVE_REPLACED, ERROR_WRITE_PROTECT, +}; + +#[test] +fn replacement_rejections_are_distinct_from_unconfirmed_commits() { + for code in [ + ERROR_ACCESS_DENIED, + ERROR_LOCK_VIOLATION, + ERROR_SHARING_VIOLATION, + ERROR_WRITE_PROTECT, + ] { + assert!( + matches!(replacement_failure(code), WriteError::Failed), + "{code}" + ); + } + for code in [ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, + ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + ERROR_UNABLE_TO_REMOVE_REPLACED, + 0, + u32::MAX, + ] { + assert!( + matches!(replacement_failure(code), WriteError::OutcomeUnknown), + "{code}" + ); + } +} diff --git a/apps/buddy/native/host/src/file_engine/journal.rs b/apps/buddy/native/host/src/file_engine/journal.rs index b45252bc6..8fca491b4 100644 --- a/apps/buddy/native/host/src/file_engine/journal.rs +++ b/apps/buddy/native/host/src/file_engine/journal.rs @@ -104,22 +104,6 @@ impl Journal { rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect() } - pub fn hazards(&self) -> Result> { - let mut query = self.0.prepare("SELECT record, claims FROM operations WHERE status IN ('\"executing\"','\"needs-recovery\"')")?; - let rows = query.query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - })?; - let mut hazards = Vec::new(); - for row in rows { - let (record, claims) = row?; - let record: Operation = serde_json::from_str(&record)?; - if matches!(record.status, Status::Executing | Status::NeedsRecovery) { - hazards.extend(serde_json::from_str::>(&claims)?); - } - } - Ok(hazards) - } - pub fn leases(&self) -> Result, bool)>> { let mut query = self.0.prepare("SELECT id, claims, hazard FROM leases")?; let rows = query.query_map([], |row| { diff --git a/apps/buddy/native/host/src/file_engine/mod.rs b/apps/buddy/native/host/src/file_engine/mod.rs index e40392b1e..8f4118999 100644 --- a/apps/buddy/native/host/src/file_engine/mod.rs +++ b/apps/buddy/native/host/src/file_engine/mod.rs @@ -11,3 +11,7 @@ mod windows; mod windows_shell; pub use server::serve; + +#[cfg(test)] +#[path = "../../__tests__/file_engine_interrupt.rs"] +mod interruption; diff --git a/apps/buddy/native/host/src/file_engine/operations.rs b/apps/buddy/native/host/src/file_engine/operations.rs index 2316de514..6f2ccf693 100644 --- a/apps/buddy/native/host/src/file_engine/operations.rs +++ b/apps/buddy/native/host/src/file_engine/operations.rs @@ -205,6 +205,8 @@ pub fn execute( } record.status = Status::Executing; journal.update(&mut record)?; + #[cfg(test)] + super::interruption::checkpoint("executing"); let mut may_have_changes = false; let result: Result = (|| { if let Some(metadata) = &existing @@ -227,7 +229,10 @@ pub fn execute( }) .map_err(|error| match error { crate::file_writer::WriteError::Conflict => Error::Conflict, - crate::file_writer::WriteError::Failed => Error::Recovery, + crate::file_writer::WriteError::Failed => { + Error::NotApplied(Box::new(Error::Io(std::io::Error::other(error)))) + } + crate::file_writer::WriteError::OutcomeUnknown => Error::Recovery, })?; } else { let relative = relative(&request.path)?; @@ -383,6 +388,8 @@ pub fn execute( } } })(); + #[cfg(test)] + super::interruption::checkpoint("completed"); match result { Ok(receipt) => { record.status = Status::Applied; diff --git a/apps/buddy/native/host/src/file_engine/server.rs b/apps/buddy/native/host/src/file_engine/server.rs index 69b5de51b..d0c9e154f 100644 --- a/apps/buddy/native/host/src/file_engine/server.rs +++ b/apps/buddy/native/host/src/file_engine/server.rs @@ -111,6 +111,7 @@ pub fn serve(directory: &Path) -> Result<()> { subscriptions: HashSet::new(), watched: HashMap::new(), pending: VecDeque::new(), + active_leases: HashMap::new(), }; send( &engine.output, @@ -169,6 +170,7 @@ struct Engine { subscriptions: HashSet, watched: HashMap, pending: VecDeque, + active_leases: HashMap>, } impl Engine { @@ -238,17 +240,10 @@ impl Engine { } fn available(&self, claims: &[Claim], own: Option<&str>) -> Result { - if overlap(claims, &self.journal.hazards()?) { - return Err(Error::Recovery); - } - let leases = self.journal.leases()?; if let Some(id) = own { - let lease = leases - .iter() - .find(|(key, _, hazard)| key == id && !hazard) - .ok_or(Error::Lease)?; + let lease = self.active_leases.get(id).ok_or(Error::Lease)?; if !claims.iter().all(|claim| { - lease.1.iter().any(|held| { + lease.iter().any(|held| { held.path == "*" || held.path == claim.path || (held.subtree && contains(&held.path, &claim.path)) @@ -257,11 +252,8 @@ impl Engine { return Err(Error::Lease); } } - for (id, held, hazard) in leases { - if Some(id.as_str()) != own && overlap(claims, &held) { - if hazard { - return Err(Error::Recovery); - } + for (id, held) in &self.active_leases { + if Some(id.as_str()) != own && overlap(claims, held) { return Ok(false); } } @@ -323,6 +315,7 @@ impl Engine { return Ok(None); } self.journal.acquire(lease_id, &claims, *external)?; + self.active_leases.insert(lease_id.clone(), claims); Value::Null } Command::Protect { lease_id, claims } => { @@ -366,6 +359,7 @@ impl Engine { } } self.pending = retained; + self.active_leases.remove(lease_id); self.journal.release(lease_id, *hazard)?; Value::Null } diff --git a/apps/buddy/native/host/src/file_writer.rs b/apps/buddy/native/host/src/file_writer.rs index 2178122b7..a94552c88 100644 --- a/apps/buddy/native/host/src/file_writer.rs +++ b/apps/buddy/native/host/src/file_writer.rs @@ -12,10 +12,14 @@ const MAX_ENGINE_TEXT_BYTES: usize = 16 * 1024 * 1024; #[derive(Debug, thiserror::Error)] pub enum WriteError { + /// The destination was not changed by this attempt. #[error("BOUNDED_FILE_WRITE_FAILED")] Failed, #[error("BOUNDED_FILE_CONFLICT")] Conflict, + /// Replacement may have happened, including a failure to sync its directory. + #[error("BOUNDED_FILE_WRITE_FAILED")] + OutcomeUnknown, } #[derive(Deserialize)] diff --git a/apps/buddy/native/host/src/file_writer/unix.rs b/apps/buddy/native/host/src/file_writer/unix.rs index 4ffc2596b..5a2d805c0 100644 --- a/apps/buddy/native/host/src/file_writer/unix.rs +++ b/apps/buddy/native/host/src/file_writer/unix.rs @@ -73,8 +73,8 @@ pub fn write(request: &WriteRequest) -> Result<(), WriteError> { verify(&root, root_path)?; verify(&parent, path.parent().ok_or(WriteError::Failed)?)?; renameat(&parent, temporary.as_str(), &parent, Path::new(name)) - .map_err(|_| WriteError::Failed)?; - parent.sync_all().map_err(|_| WriteError::Failed) + .map_err(|_| WriteError::OutcomeUnknown)?; + parent.sync_all().map_err(|_| WriteError::OutcomeUnknown) })(); let _ = unlinkat(&parent, temporary.as_str(), AtFlags::empty()); result diff --git a/apps/buddy/native/host/src/file_writer/windows.rs b/apps/buddy/native/host/src/file_writer/windows.rs index 9eefa237f..e7a279a40 100644 --- a/apps/buddy/native/host/src/file_writer/windows.rs +++ b/apps/buddy/native/host/src/file_writer/windows.rs @@ -7,8 +7,14 @@ use std::{ path::Path, time::{SystemTime, UNIX_EPOCH}, }; -use windows_sys::Win32::Storage::FileSystem::{ - FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, MoveFileExW, ReplaceFileW, +use windows_sys::Win32::{ + Foundation::{ + ERROR_ACCESS_DENIED, ERROR_LOCK_VIOLATION, ERROR_SHARING_VIOLATION, ERROR_WRITE_PROTECT, + GetLastError, + }, + Storage::FileSystem::{ + FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, MoveFileExW, ReplaceFileW, + }, }; pub fn write(request: &WriteRequest) -> Result<(), WriteError> { @@ -54,12 +60,12 @@ pub fn write(request: &WriteRequest) -> Result<(), WriteError> { ".lexora-save-{}-{nonce}.backup", std::process::id() )); + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|_| WriteError::Failed)?; let result = (|| { - let mut output = OpenOptions::new() - .write(true) - .create_new(true) - .open(&temporary) - .map_err(|_| WriteError::Failed)?; output .write_all(request.content.as_bytes()) .map_err(|_| WriteError::Failed)?; @@ -92,11 +98,17 @@ pub fn write(request: &WriteRequest) -> Result<(), WriteError> { ) }; if replaced == 0 { + // SAFETY: GetLastError has no preconditions and immediately follows ReplaceFileW. + let error = unsafe { GetLastError() }; + let outcome = replacement_failure(error); + if matches!(outcome, WriteError::Failed) { + return Err(outcome); + } if !path.exists() && backup.exists() { // SAFETY: Both buffers are terminated; zero flags preserve any newly occupied destination. unsafe { MoveFileExW(previous.as_ptr(), destination.as_ptr(), 0) }; } - return Err(WriteError::Failed); + return Err(outcome); } let _ = fs::remove_file(&backup); Ok(()) @@ -104,3 +116,17 @@ pub fn write(request: &WriteRequest) -> Result<(), WriteError> { let _ = fs::remove_file(&temporary); result } + +fn replacement_failure(error: u32) -> WriteError { + match error { + ERROR_ACCESS_DENIED + | ERROR_SHARING_VIOLATION + | ERROR_LOCK_VIOLATION + | ERROR_WRITE_PROTECT => WriteError::Failed, + _ => WriteError::OutcomeUnknown, + } +} + +#[cfg(test)] +#[path = "../../__tests__/file_writer_windows.rs"] +mod tests; diff --git a/apps/buddy/service/src/BuddyService.ts b/apps/buddy/service/src/BuddyService.ts index b2bc3deb9..c607fb78c 100644 --- a/apps/buddy/service/src/BuddyService.ts +++ b/apps/buddy/service/src/BuddyService.ts @@ -441,10 +441,15 @@ export async function startBuddyService( await service.initialize() return service }, ['runtime.file_engine']) - const mutationRecovery = new MutationRecoveryService({ mutations, record, evidence: mutationEvidence, receiptDirectory: join(options.buddyHome, 'process-receipts'), context: { conversations, runs }, onChange: notifyRecovery }) - await host.step('runtime.mutation_recovery', async () => { - await mutationRecovery.recheck({ signal: AbortSignal.timeout(5_000) }) - }) + const mutationRecovery = await host.start('runtime.mutation_recovery', async ({ defer }) => { + const service = new MutationRecoveryService({ mutations, record, evidence: mutationEvidence, receiptDirectory: join(options.buddyHome, 'process-receipts'), context: { conversations, runs }, onChange: notifyRecovery }) + defer(async () => { + mutations.stopAdmission() + await service.dispose() + }) + await service.start() + return service + }, ['runtime.mutations']) const changeCaptureService = await host.start('runtime.change_capture', ({ defer }) => { const service = new ChangeCaptureService({ paths, repository: createChangeSetRepository(options.database), onListenerError: () => record({ event: 'observer.failed', component: 'runtime.change_capture', level: 'warn' }) }) service.onDidChange(event => record({ event: `changes.${event.kind.replaceAll('-', '_')}`, level: event.errorCode ? 'warn' : 'info', producerInstanceId: event.sourceId, operationId: event.operationId, conversationId: event.conversationId, runId: event.runId, toolCallId: event.toolCallId, revision: event.revision, count: event.count, errorCode: event.errorCode })) @@ -961,6 +966,7 @@ export async function startBuddyService( repository: treeRepository, threads, runs, + isRunHistoryAvailable: runId => !options.eventLog.getRecoveryFailures().some(failure => failure.runId === runId), recovery: sessionRecovery, onObserverError: () => record({ event: 'tree.observer_failed', level: 'warn' }), }) @@ -1001,6 +1007,7 @@ export async function startBuddyService( lifecycle: runLifecycleService, mutations, listIncomplete: () => runs.listIncomplete(), + recoverQuarantinedRun: runId => runRecoveryService.recoverQuarantinedRun(runId), getFileAccessCeiling: runId => getRunPermissionCeiling(runId) === 'read_only' ? 'read_only' : undefined, onObserverError: () => record({ event: 'execution.tree_observer_failed', level: 'error' }), readResult: runId => readExecutionResult(options.eventLog, runId), @@ -1088,8 +1095,17 @@ export async function startBuddyService( sessions, }) const quarantine = options.eventLog.onDidQuarantine(async (failure) => { - if (failure.conversationId) - await service.cancelAndWaitForConversation(failure.conversationId) + const run = runs.findById(failure.runId) + if (!run) + return + await sessions.invalidateMatching(identity => identity.conversationId === run.conversationId + && identity.branchId === run.branchId && identity.threadId === run.threadId) + const root = runs.findById(run.rootRunId) + if (root?.status === 'queued' || root?.status === 'running') { + chatQueueService?.pause(root) + await service.cancel(root.id, 'EVENT_LOG_CORRUPTED') + } + runRecoveryService.recoverQuarantinedRun(run.id) }) defer(() => quarantine.dispose()) const settledDiagnostics = service.onDidSettle(settled => record({ @@ -1432,9 +1448,11 @@ export async function startBuddyService( isBusy: () => { if (runner.hasActiveExecutions) return true - const quarantined = new Set(options.eventLog.getRecoveryFailures().map(failure => failure.conversationId)) - return runs.listIncomplete().some(run => !quarantined.has(run.conversationId)) - || approvalsRepository.listPending().some(approval => !quarantined.has(runs.findById(approval.runId)?.conversationId ?? null)) + return runs.hasIncomplete() + || approvalsRepository.listPending().some((approval) => { + const run = runs.findById(approval.runId) + return run?.status === 'queued' || run?.status === 'running' + }) }, getCacheWarmingStatus: (conversationId) => { const conversation = conversations.findById(conversationId) diff --git a/apps/buddy/service/src/__tests__/buddyService.integration.spec.ts b/apps/buddy/service/src/__tests__/buddyService.integration.spec.ts index d0de500bd..2f288a416 100644 --- a/apps/buddy/service/src/__tests__/buddyService.integration.spec.ts +++ b/apps/buddy/service/src/__tests__/buddyService.integration.spec.ts @@ -16,7 +16,7 @@ import type { ApplicationDiagnostic } from '../../../shared/diagnostics/applicat import type { ReusableBuddySession } from '../agent/sessions/ReusableBuddySession' import type { BuddyServiceRpcServer } from '../rpc/BuddyServiceRpcServer' import { Buffer } from 'node:buffer' -import { mkdir, mkdtemp, readFile, realpath, rm, unlink, writeFile } from 'node:fs/promises' +import { appendFile, mkdir, mkdtemp, readFile, realpath, rm, unlink, writeFile } from 'node:fs/promises' import { createServer } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,11 +44,14 @@ import { prepareTestCommandRequest, prepareTestTurnRequest, } from '../storage/__tests__/composerDraftTestFixture' +import { createExecutionStorageFixture, executionNow } from '../storage/__tests__/executionFixture' +import { createApprovalRepository } from '../storage/approvalRepository' import { createArtifactRepository } from '../storage/artifactRepository' import { createAttachmentRepository } from '../storage/attachmentRepository' import { createConversationDeletionRepository } from '../storage/conversationDeletionRepository' import { createConversationRepository } from '../storage/conversationRepository' import { openBuddyDatabase } from '../storage/database' +import { createExecutionRepository } from '../storage/executionRepository' import { createRunRepository } from '../storage/runRepository' import { createSpaceRepository } from '../storage/spaceRepository' import { createUsageRepository } from '../storage/usageRepository' @@ -66,6 +69,124 @@ afterEach(async () => { }) describe('buddy runtime cross-subsystem contract', () => { + it.each(['root', 'child'] as const)('recovers a corrupt %s across database restarts and executes new turns in the same conversation', async (damagedId) => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'lexora-buddy-run-recovery-'))) + directories.push(directory) + const buddyHome = join(directory, 'buddy') + const databasePath = join(buddyHome, 'buddy.sqlite3') + const conversationsDirectory = join(buddyHome, 'conversations') + const scope = { conversationId: 'conversation', branchId: 'branch' } + const seed = createExecutionStorageFixture(databasePath) + const seedLog = createRunEventLog({ database: seed.database, conversationsDirectory }) + seed.conversations.createMessage({ id: 'goal', ...scope, runId: null, role: 'user', content: { text: 'Old interrupted goal' }, createdAt: executionNow }) + seed.executions.admitChild(seed.child('child')) + for (const runId of ['root', 'child']) { + await seedLog.append({ runId, type: 'run.started', payload: {} }) + await seedLog.append({ runId, type: 'tool.started', payload: { toolCallId: `old-tool-${runId}`, toolName: 'read', args: { path: 'evidence.txt' } } }) + } + await seedLog.append({ runId: damagedId, type: 'approval.requested', payload: { + id: 'old-approval', + runId: damagedId, + toolCallId: 'old-read', + kind: 'read', + status: 'pending', + summary: 'Old read', + createdAt: executionNow, + resolvedAt: null, + payload: { card: 'paths', access: 'read', grant: null, toolName: 'read', targets: [{ path: '/fixture.txt', zone: 'workspace' }] }, + } }) + const originalProjection = seed.database.prepare('SELECT * FROM run_events WHERE run_id = ?').all(damagedId) + await seedLog.close() + seed.database.close() + const damagedFile = join(conversationsDirectory, scope.conversationId, 'events', `${damagedId}.jsonl`) + await appendFile(damagedFile, '{"torn":') + const damagedBytes = await readFile(damagedFile) + let response = Promise.withResolvers() + const provider = await startLocalModelProvider(async () => { + const current = response + await current.promise + return { text: 'NEW_TURN_EXECUTED' } + }) + let priorCompleted: string | undefined + let newlyDamagedFile: string | undefined + let newlyDamagedBytes: Buffer | undefined + let queuedId: string | undefined + for (let boot = 0; boot < 3; boot++) { + const database = openBuddyDatabase({ databasePath }) + const eventLog = createRunEventLog({ database, conversationsDirectory }) + const harness = createRuntimeRpcHarness({ 'offline-provider': { type: 'api_key', key: 'test-api-key' } }) + let runtime: Awaited> | undefined + const beforeRequests = provider.requests.length + try { + expect(await eventLog.recoverAll()).toMatchObject({ failed: boot === 2 ? 2 : 1 }) + runtime = await startBuddyService({ buddyHome, database, eventLog, rpc: harness.rpc, builtinSkillsDirectories: [] }) + const runs = createRunRepository(database) + const executions = createExecutionRepository(database) + expect(runs.listIncomplete()).toEqual([]) + expect(runs.findById(damagedId)).toMatchObject({ status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED', piSessionFile: null }) + expect(executions.find(damagedId)).toMatchObject({ phase: 'closed', cleanupState: 'degraded', cleanupErrorCode: 'EVENT_LOG_CORRUPTED' }) + expect(executions.find('root')).toMatchObject({ phase: 'closed', cleanupState: 'degraded' }) + expect(createApprovalRepository(database).findById('old-approval')?.status).toBe('cancelled') + expect(database.prepare('SELECT * FROM run_events WHERE run_id = ?').all(damagedId)).toEqual(originalProjection) + expect(await readFile(damagedFile)).toEqual(damagedBytes) + expect(provider.requests).toHaveLength(beforeRequests) + const healthyId = damagedId === 'root' ? 'child' : 'root' + expect(await eventLog.read(healthyId)).toEqual(expect.arrayContaining([expect.objectContaining({ type: 'run.failed' })])) + const timeline = await harness.invoke('conversations.listTimeline', { ...scope, limit: 100 }) as { runs: Array<{ id: string, errorCode: string | null }>, runEvents: Array<{ runId: string }> } + expect(timeline.runEvents.some(event => event.runId === damagedId)).toBe(false) + if (damagedId === 'root') + expect(timeline.runs.find(run => run.id === 'root')?.errorCode).toBe('EVENT_LOG_CORRUPTED') + expect(await harness.invoke('executions.list', scope)).toEqual(expect.arrayContaining([ + expect.objectContaining({ runId: damagedId, phase: 'closed', cleanup: 'degraded', errorCode: 'EVENT_LOG_CORRUPTED', toolCount: 1 }), + ])) + if (queuedId) { + expect(await harness.invoke('chat.queue.list', scope)).toEqual([expect.objectContaining({ id: queuedId, state: 'paused' })]) + expect(await harness.invoke('chat.queue.cancel', { ...scope, id: queuedId })).toBe(true) + } + expect(await harness.invoke('runs.isBusy', {})).toBe(false) + await harness.invoke('providers.upsertCustom', offlineProviderConfig(provider.baseUrl)) + response = Promise.withResolvers() + const turn = await harness.invoke('chat.startTurn', await turnRequest(harness, { ...scope, content: `New explicit turn ${boot}`, requestId: `new-turn-${boot}` })) as LocalTurnStart + await expect.poll(() => provider.requests.length).toBe(beforeRequests + 1) + expect(await harness.invoke('runs.isBusy', {})).toBe(true) + expect(runs.findById(turn.runId)?.status).toBe('running') + if (boot === 1 && priorCompleted) { + newlyDamagedFile = join(conversationsDirectory, scope.conversationId, 'events', `${priorCompleted}.jsonl`) + await appendFile(newlyDamagedFile, '{broken}\n') + newlyDamagedBytes = await readFile(newlyDamagedFile) + await expect(eventLog.read(priorCompleted)).rejects.toMatchObject({ code: 'EVENT_LOG_CORRUPTED' }) + expect(runs.findById(turn.runId)?.status).toBe('running') + expect(await harness.invoke('runs.isBusy', {})).toBe(true) + } + response.resolve() + expect(await waitForTerminalRun(harness, turn.runId)).toMatchObject({ status: 'completed' }) + expect(JSON.stringify(await eventLog.read(turn.runId))).toContain('NEW_TURN_EXECUTED') + priorCompleted = turn.runId + if (boot === 1) { + response = Promise.withResolvers() + const cancelled = await harness.invoke('chat.startTurn', await turnRequest(harness, { ...scope, content: 'Explicitly cancel this new turn', requestId: 'cancel-new-turn' })) as LocalTurnStart + await expect.poll(() => provider.requests.length).toBe(beforeRequests + 2) + const queued = await harness.invoke('chat.queue.enqueue', await turnRequest(harness, { ...scope, content: 'Do not replay this queued input', requestId: 'retained-queue' })) as { id: string } + queuedId = queued.id + expect(await harness.invoke('chat.cancel', { runId: cancelled.runId })).toMatchObject({ status: 'cancelled' }) + response.resolve() + expect(await harness.invoke('chat.queue.list', scope)).toEqual([expect.objectContaining({ id: queuedId, state: 'paused' })]) + } + await expect.poll(() => harness.invoke('runs.isBusy', {})).toBe(false) + expect(await readFile(damagedFile)).toEqual(damagedBytes) + if (newlyDamagedFile) + expect(await readFile(newlyDamagedFile)).toEqual(newlyDamagedBytes) + } + finally { + response.resolve() + await runtime?.dispose() + await eventLog.close() + database.close() + } + } + expect(provider.requests).toHaveLength(4) + }, 30_000) + it.each(['empty legacy directory', 'legacy outputs', 'pending deletion'] as const)('starts past a quarantined conversation with %s and continues healthy work', async (maintenance) => { const root = await realpath(await mkdtemp(join(tmpdir(), 'lexora-buddy-startup-quarantine-'))) directories.push(root) @@ -123,14 +244,14 @@ describe('buddy runtime cross-subsystem contract', () => { runtime = await startBuddyService({ buddyHome, database, eventLog, rpc: harness.rpc, builtinSkillsDirectories: [] }) const runs = createRunRepository(database) expect(eventLog.state).toBe('open') - expect(runs.findById('run-damaged')?.status).toBe('queued') + expect(runs.findById('run-damaged')).toMatchObject({ status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' }) expect(database.prepare('SELECT * FROM run_events WHERE run_id = ?').all('run-damaged')).toEqual(projection) expect(runs.findById('run-healthy')).toMatchObject(maintenance === 'pending deletion' ? { status: 'cancelled', errorCode: 'RUN_CANCELLED' } : { status: 'failed', errorCode: 'RUNTIME_RESTARTED' }) if (maintenance === 'pending deletion') { expect(createConversationRepository(database).isDeleted('damaged')).toBe(true) - expect(createConversationDeletionRepository(database).pending()).toEqual(['damaged']) + expect(createConversationDeletionRepository(database).pending()).toEqual([]) } if (maintenance === 'legacy outputs') { const artifacts = createArtifactRepository(database) @@ -144,7 +265,8 @@ describe('buddy runtime cross-subsystem contract', () => { expect(JSON.stringify(await eventLog.read(healthy.runId))).toContain('HEALTHY_AFTER_STARTUP') expect(provider.requests).toHaveLength(1) expect(await readFile(damagedFile, 'utf8')).toBe(damaged) - expect(() => eventLog.assertConversationWritable('damaged')).toThrow(RunEventCorruptionError) + expect(() => eventLog.assertConversationWritable('damaged')).not.toThrow() + expect(() => eventLog.assertRunWritable('run-damaged')).toThrow(RunEventCorruptionError) } finally { await runtime?.dispose() @@ -164,6 +286,8 @@ describe('buddy runtime cross-subsystem contract', () => { const requested = { damaged: Promise.withResolvers(), healthy: Promise.withResolvers() } const responses = { damaged: Promise.withResolvers(), healthy: Promise.withResolvers() } const provider = await startLocalModelProvider(async (payload) => { + if (JSON.stringify(payload.messages).includes('AFTER_LIVE_QUARANTINE')) + return { text: 'NEW_AFTER_LIVE_QUARANTINE' } const key = JSON.stringify(payload.messages).includes('DAMAGED_TASK') ? 'damaged' : 'healthy' requested[key].resolve() await responses[key].promise @@ -194,6 +318,17 @@ describe('buddy runtime cross-subsystem contract', () => { expect(await waitForTerminalRun(harness, healthy.runId)).toMatchObject({ status: 'completed' }) expect(await readFile(file, 'utf8')).toBe(corrupted) expect(JSON.stringify(await eventLog.read(healthy.runId))).toContain('HEALTHY_RESULT') + await expect.poll(() => runs.findById(damaged.runId)?.status).toBe('failed') + const next = await harness.invoke('chat.startTurn', await turnRequest(harness, { + conversationId: damaged.conversationId, + branchId: damaged.branchId, + content: 'AFTER_LIVE_QUARANTINE', + requestId: 'after-live-quarantine', + })) as LocalTurnStart + const continued = await waitForTerminalRun(harness, next.runId) + expect(continued, JSON.stringify(await eventLog.read(next.runId))).toMatchObject({ status: 'completed', errorCode: null }) + expect(JSON.stringify(await eventLog.read(next.runId))).toContain('NEW_AFTER_LIVE_QUARANTINE') + expect(await readFile(file, 'utf8')).toBe(corrupted) } finally { responses.damaged.resolve() @@ -1768,6 +1903,8 @@ function createRuntimeRpcHarness( }) return Promise.resolve({ ok: true, url: input.url, status: 200, headers: { 'content-type': 'text/plain; charset=utf-8' } }) } + if (method === 'host.images.preferences') + return Promise.resolve({}) if (method === 'host.runtimePreferences.get') return Promise.resolve({ cacheWarming: 'off', codemode: false, modelRetryLimit: 3 }) if (method === 'host.credentials.list') { diff --git a/apps/buddy/service/src/__tests__/mutationConcurrency.integration.spec.ts b/apps/buddy/service/src/__tests__/mutationConcurrency.integration.spec.ts new file mode 100644 index 000000000..95488ef08 --- /dev/null +++ b/apps/buddy/service/src/__tests__/mutationConcurrency.integration.spec.ts @@ -0,0 +1,348 @@ +import type { ServerResponse } from 'node:http' +import type { ConcurrencyRuntime, Role } from './mutationConcurrencyFixture' +import { readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { join } from 'node:path' +import process from 'node:process' +import { describe, expect, it, vi } from 'vitest' +import { resolveBuddyPlatform } from '../../../shared/platform' +import { MutationCoordinator } from '../changes/MutationCoordinator' +import { + assertCaptureChain, + assertDurableTree, + captures, + createConcurrencyRuntime, + deferred, + hash, + mutationTiming, + roles, + tool, + treeByRole, +} from './mutationConcurrencyFixture' + +function eventPayload(event: ConcurrencyRuntime['events'][number]) { + return event.payload as { toolCallId?: string, toolName?: string, isError?: boolean } +} + +async function assertToolOverlap(f: ConcurrencyRuntime, ids: string[]) { + await expect.poll(() => f.events.filter(event => event.type === 'tool.started' && ids.includes(eventPayload(event).toolCallId ?? '')).length, { timeout: 15000 }).toBe(ids.length) + expect(f.events.filter(event => event.type === 'tool.completed' && ids.includes(eventPayload(event).toolCallId ?? ''))).toEqual([]) + expect(new Set(f.events.filter(event => event.type === 'tool.started' && ids.includes(eventPayload(event).toolCallId ?? '')).map(event => event.runId)).size).toBe(ids.length) +} + +function shellCommand(script: string, ...args: string[]) { + const quote = (value: string) => process.platform === 'win32' ? `'${value.replaceAll('\'', '\'\'')}'` : `'${value.replaceAll('\'', '\'\\\'\'')}'` + return `${process.platform === 'win32' ? '& ' : ''}${[process.execPath, script, ...args].map(quote).join(' ')}` +} + +function toolResults(f: ConcurrencyRuntime, ids: string[]) { + return f.events.filter(event => event.type === 'tool.completed' && ids.includes(eventPayload(event).toolCallId ?? '')) +} + +describe('real Runtime multi-agent mutation concurrency', () => { + it('preserves root/actor/repeated-NPC ownership for concurrent files and serializes non-overlapping edits', async () => { + let workspace = '' + const f = await createConcurrencyRuntime({ + steps: role => [ + tool(`${role}-write`, 'write', { path: join(workspace, `${role}.txt`), content: `owned:${role}\n` }), + tool(`${role}-edit`, 'edit', { path: join(workspace, 'shared.txt'), edits: [{ oldText: `${role}=before`, newText: `${role}=after` }] }), + ], + }) + const timing = mutationTiming() + try { + workspace = f.workspace + const before = roles.map(role => `${role}=before\n`).join('') + const after = roles.map(role => `${role}=after\n`).join('') + const shared = join(workspace, 'shared.txt') + await writeFile(shared, before) + const differentFiles = timing.gate(input => input.claims.some(claim => claim.kind === 'file' && claim.canonicalPath !== shared)) + const sharedFile = timing.gate(input => input.claims.some(claim => claim.kind === 'file' && claim.canonicalPath === shared)) + f.start() + await differentFiles.waitFor(4, 0) + const live = treeByRole(f) + expect(Object.values(live).every(run => run.status === 'running')).toBe(true) + await assertToolOverlap(f, roles.map(role => `${role}-write`)) + differentFiles.release() + await sharedFile.waitFor(1, 3) + await assertToolOverlap(f, roles.map(role => `${role}-edit`)) + expect(await readFile(shared, 'utf8')).toBe(before) + sharedFile.release() + await f.complete() + const tree = await assertDurableTree(f) + expect(await readFile(shared, 'utf8')).toBe(after) + const ledger = captures(f) + expect(ledger).toHaveLength(8) + for (const role of roles) { + expect(await readFile(join(workspace, `${role}.txt`), 'utf8')).toBe(`owned:${role}\n`) + const owned = ledger.filter(capture => capture.invocation?.runId === tree[role].id) + expect(owned).toHaveLength(2) + expect(owned.map(capture => capture.toolCallId).sort()).toEqual([`${role}-edit`, `${role}-write`]) + expect(owned.every(capture => capture.actorId === tree[role].buddyId && capture.toolReportedError === false)).toBe(true) + } + const chain = await assertCaptureChain(f, shared, before, after, 4) + expect(new Set(chain.map(capture => capture.invocation!.segmentId)).size).toBe(4) + expect(toolResults(f, roles.flatMap(role => [`${role}-write`, `${role}-edit`])).map(event => eventPayload(event).isError)).toEqual(Array.from({ length: 8 }).fill(false)) + } + finally { + timing.dispose() + await f.dispose() + } + }, 60000) + + it('rejects one stale exact edit without overwriting the winning sibling', async () => { + let path = '' + const f = await createConcurrencyRuntime({ + steps: role => role.startsWith('npc-') + ? [ + tool(`${role}-read`, 'read', { path }), + tool(`${role}-race`, 'edit', { path, edits: [{ oldText: 'value=original', newText: `value=${role}` }] }), + ] + : [], + }) + const timing = mutationTiming() + try { + path = join(f.workspace, 'conflict.txt') + const original = 'value=original\n' + await writeFile(path, original) + const race = timing.gate(input => input.claims.some(claim => claim.kind === 'file' && claim.canonicalPath === path)) + f.start() + await race.waitFor(1, 1) + await assertToolOverlap(f, ['npc-a-race', 'npc-b-race']) + expect(toolResults(f, ['npc-a-read', 'npc-b-read'])).toHaveLength(2) + for (const role of ['npc-a', 'npc-b'] as const) { + const continuation = f.requests.find(request => request.role === role && JSON.stringify(request.payload.messages).includes('value=original')) + expect(continuation, `${role} must actually read the original before racing`).toBeDefined() + } + race.release() + await f.complete() + const tree = await assertDurableTree(f) + const results = toolResults(f, ['npc-a-race', 'npc-b-race']) + expect(results).toHaveLength(2) + expect(results.map(event => eventPayload(event).isError).sort()).toEqual([false, true]) + const winner = results.find(event => !eventPayload(event).isError)! + const winnerRole = winner.runId === tree['npc-a'].id ? 'npc-a' : 'npc-b' + const loserRole = winnerRole === 'npc-a' ? 'npc-b' : 'npc-a' + const conflict = f.requests.filter(request => request.role === loserRole).at(-1)!.payload.messages.find(message => message.tool_call_id === `${loserRole}-race`) + expect(conflict?.content).toContain('The old text must match exactly including all whitespace and newlines.') + const final = `value=${winnerRole}\n` + expect(await readFile(path, 'utf8')).toBe(final) + const chain = await assertCaptureChain(f, path, original, final, 2) + expect(chain[0]).toMatchObject({ toolReportedError: false, invocation: { runId: winner.runId } }) + expect(chain[1]).toMatchObject({ toolReportedError: true }) + expect(chain[1]!.before.hash).toBe(chain[1]!.after?.hash) + } + finally { + timing.dispose() + await f.dispose() + } + }, 60000) + + it('serializes two full writes as whole replacements, never as an automatic merge', async () => { + let path = '' + const versions: Partial> = { + 'npc-a': `A-BEGIN\n${'alpha-only line\n'.repeat(2048)}A-END\n`, + 'npc-b': `B-BEGIN\n${'bravo-only line\n'.repeat(2048)}B-END\n`, + } + const f = await createConcurrencyRuntime({ + steps: role => role.startsWith('npc-') ? [tool(`${role}-replace`, 'write', { path, content: versions[role]! })] : [], + }) + const timing = mutationTiming() + try { + path = join(f.workspace, 'replace.txt') + const before = 'initial whole version\n' + await writeFile(path, before) + const race = timing.gate(input => input.claims.some(claim => claim.kind === 'file' && claim.canonicalPath === path)) + f.start() + await race.waitFor(1, 1) + await assertToolOverlap(f, ['npc-a-replace', 'npc-b-replace']) + race.release() + await f.complete() + await assertDurableTree(f) + const final = await readFile(path, 'utf8') + expect(Object.values(versions)).toContain(final) + const chain = await assertCaptureChain(f, path, before, final, 2) + expect(chain.every(capture => !capture.toolReportedError)).toBe(true) + expect(new Set(chain.map(capture => capture.after?.hash))).toEqual(new Set(Object.values(versions).map(hash))) + expect(chain[1]!.after?.hash).toBe(hash(final)) + expect(toolResults(f, ['npc-a-replace', 'npc-b-replace']).map(event => eventPayload(event).isError)).toEqual([false, false]) + } + finally { + timing.dispose() + await f.dispose() + } + }, 60000) + + it('propagates an ancestor read_only ceiling through actor and repeated NPC descendants', async () => { + let path = '' + const f = await createConcurrencyRuntime({ + readOnly: true, + steps: role => [tool(`${role}-forbidden`, 'write', { path, content: `forbidden:${role}` })], + }) + try { + path = join(f.workspace, 'protected.txt') + await writeFile(path, 'untouched\n') + f.start() + await f.complete() + const tree = await assertDurableTree(f) + expect(await readFile(path, 'utf8')).toBe('untouched\n') + for (const role of roles) { + expect(tree[role].executionProfile).toBe(role === 'root' ? 'workspace_write' : 'read_only') + expect(f.database.prepare('SELECT file_access_ceiling FROM execution_state WHERE run_id = ?').get(tree[role].id)?.file_access_ceiling).toBe('read_only') + if (role !== 'root') + expect(tree[role].actorSnapshot?.settings.limits.fileAccess).toBe('inherit') + const payloads = f.requests.filter(request => request.role === role).map(request => request.payload) + expect(payloads.length).toBeGreaterThanOrEqual(2) + expect(payloads.every(payload => !payload.tools?.some(tool => tool.function.name === 'write'))).toBe(true) + expect(JSON.stringify(payloads.at(-1)?.messages)).toContain('Tool write not found') + } + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_mutation_leases').get()?.count).toBe(0) + expect(captures(f)).toEqual([]) + } + finally { + await f.dispose() + } + }, 60000) + + it.skipIf(!resolveBuddyPlatform(process.platform).hostShellSupervision)('recovers a lost native settlement from real receipts without replaying the command before a child writes', async () => { + let command = '' + let path = '' + let lostInvocation: string | undefined + const original = MutationCoordinator.prototype.recordOwnedProcessSettlement + const failure = vi.spyOn(MutationCoordinator.prototype, 'recordOwnedProcessSettlement').mockImplementation(function (this: MutationCoordinator, input) { + if (!lostInvocation && input.launchState === 'settled') { + lostInvocation = input.invocationId + throw new Error('INJECTED_RELEASE_NOTIFICATION_LOST') + } + return original.call(this, input) + }) + let f: ConcurrencyRuntime | undefined + try { + f = await createConcurrencyRuntime({ + fullAccess: true, + steps: (role) => { + if (role === 'root') + return [tool('lost-settlement-shell', process.platform === 'win32' ? 'powershell' : 'bash', { command, timeout: 40 })] + if (role === 'npc-a') + return [tool('write-after-recovery', 'write', { path, content: 'native write after recovery\n' })] + return [] + }, + }) + const runtime = f + path = join(f.workspace, 'after-recovery.txt') + const marker = join(f.workspace, 'effects.log') + const script = join(f.workspace, 'single-effect.cjs') + await writeFile(script, 'require(\'node:fs\').appendFileSync(process.argv[2], \'old-effect\\n\');\n') + command = shellCommand(script, marker) + const writer = f.holdReply('write-after-recovery') + f.start() + await expect.poll(() => toolResults(runtime, ['lost-settlement-shell']).length, { timeout: 15000 }).toBe(1) + expect(toolResults(f, ['lost-settlement-shell'])[0]).toMatchObject({ payload: { isError: true } }) + expect(lostInvocation).toBeDefined() + expect(await readFile(marker, 'utf8')).toBe('old-effect\n') + writer.resolve() + await f.complete() + const tree = await assertDurableTree(f) + expect(await readFile(path, 'utf8')).toBe('native write after recovery\n') + expect(await readFile(marker, 'utf8')).toBe('old-effect\n') + const evidence = f.database.prepare('SELECT kind, outcome, reason, actor FROM run_mutation_evidence WHERE invocation_id = ?').all(lostInvocation!) + expect(evidence).toEqual(expect.arrayContaining([expect.objectContaining({ kind: 'quiesced', outcome: 'applied', reason: 'supervisor_receipt', actor: 'system' })])) + expect(f.database.prepare('SELECT status FROM run_mutation_leases WHERE invocation_id = ?').get(lostInvocation!)?.status).toBe('released') + const processes = f.database.prepare('SELECT settlement FROM run_mutation_processes WHERE invocation_id = ?').all(lostInvocation!) + expect(processes.length).toBeGreaterThan(0) + expect(processes.every(process => process.settlement === 'released')).toBe(true) + expect(f.events.filter(event => event.type === 'tool.started' && eventPayload(event).toolCallId === 'lost-settlement-shell')).toHaveLength(1) + expect(toolResults(f, ['write-after-recovery'])).toMatchObject([{ runId: tree['npc-a'].id, payload: { isError: false } }]) + expect(f.database.prepare('SELECT tool_call_id, status FROM run_change_invocations WHERE run_id = ?').all(tree['npc-a'].id)).toEqual([expect.objectContaining({ tool_call_id: 'write-after-recovery', status: 'completed' })]) + } + finally { + failure.mockRestore() + await f?.dispose() + } + }, 60000) + + it.skipIf(!resolveBuddyPlatform(process.platform).hostShellSupervision)('cancels a queued child behind a real supervised full_access shell and admits a fresh write after release', async () => { + const shellReached = deferred() + let shellResponse: ServerResponse | undefined + const server = createServer((_request, response) => { + shellResponse = response + shellReached.resolve() + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const address = server.address() + if (!address || typeof address === 'string') + throw new Error('Shell barrier did not bind TCP') + let command = '' + let cancelledPath = '' + let freshPath = '' + const timing = mutationTiming() + let f: ConcurrencyRuntime | undefined + try { + f = await createConcurrencyRuntime({ + fullAccess: true, + steps: (role) => { + if (role === 'root') + return [tool('root-supervised-shell', process.platform === 'win32' ? 'powershell' : 'bash', { command, timeout: 40 })] + if (role === 'npc-a') + return [tool('cancelled-write', 'write', { path: cancelledPath, content: 'MUST NOT WRITE\n' })] + if (role === 'npc-b') + return [tool('fresh-write', 'write', { path: freshPath, content: 'after supervised release\n' })] + return [] + }, + }) + const runtime = f + cancelledPath = join(f.workspace, 'cancelled.txt') + freshPath = join(f.workspace, 'fresh.txt') + await writeFile(cancelledPath, 'preserved\n') + const script = join(f.workspace, 'hold-shell.cjs') + await writeFile(script, 'require(\'node:http\').get(process.argv[2], response => { response.resume(); response.on(\'end\', () => process.stdout.write(\'SHELL_RELEASED\')); }).on(\'error\', error => { console.error(error); process.exitCode = 1; });\n') + command = shellCommand(script, `http://127.0.0.1:${address.port}/hold`) + const cancelledReply = f.holdReply('cancelled-write') + const freshReply = f.holdReply('fresh-write') + const queued = timing.gate(input => input.claims.some(claim => claim.kind === 'file' && claim.canonicalPath === cancelledPath)) + f.start() + await expect.poll(() => Boolean(shellResponse), { timeout: 15000 }).toBe(true) + await shellReached.promise + const tree = treeByRole(f) + const active = f.database.prepare('SELECT invocation_id, claims_json FROM run_mutation_leases WHERE run_id = ? AND status = \'active\'').get(tree.root.id)! + expect(JSON.parse(String(active.claims_json))).toEqual([{ kind: 'host' }]) + const processes = f.database.prepare('SELECT process_id, settlement FROM run_mutation_processes WHERE invocation_id = ? ORDER BY process_id').all(active.invocation_id!) + expect(processes.length).toBeGreaterThan(0) + expect(processes.every(process => process.settlement === 'pending')).toBe(true) + cancelledReply.resolve() + await queued.waitFor(0, 1) + await assertToolOverlap(f, ['cancelled-write']) + expect(await readFile(cancelledPath, 'utf8')).toBe('preserved\n') + expect(await f.invoke('executions.cancel', { + conversationId: f.turn.conversationId, + branchId: f.turn.branchId, + rootRunId: f.turn.runId, + runId: tree['npc-a'].id, + })).toEqual({ cancelled: true }) + await expect.poll(() => runtime.runs.findById(tree['npc-a'].id)?.status, { timeout: 15000 }).toBe('cancelled') + expect(queued.admissions[0]!.coordinator.getRunState(tree['npc-a'].id)).toMatchObject({ pending: 0, active: 0, hazards: [] }) + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_mutation_leases WHERE run_id = ?').get(tree['npc-a'].id)?.count).toBe(0) + expect(await readFile(cancelledPath, 'utf8')).toBe('preserved\n') + shellResponse!.end('release') + await expect.poll(() => runtime.database.prepare('SELECT status FROM run_mutation_leases WHERE invocation_id = ?').get(active.invocation_id!)?.status, { timeout: 15000 }).toBe('released') + expect(f.database.prepare('SELECT process_id, settlement FROM run_mutation_processes WHERE invocation_id = ? ORDER BY process_id').all(active.invocation_id!)).toEqual(processes.map(process => ({ ...process, settlement: 'released' }))) + freshReply.resolve() + await f.complete() + await assertDurableTree(f, ['npc-a']) + expect(await readFile(freshPath, 'utf8')).toBe('after supervised release\n') + expect(await readFile(cancelledPath, 'utf8')).toBe('preserved\n') + expect(toolResults(f, ['root-supervised-shell', 'fresh-write']).map(event => eventPayload(event).isError)).toEqual([false, false]) + expect(f.database.prepare('SELECT status FROM run_change_invocations WHERE run_id = ?').get(tree.root.id)?.status).toBe('partial') + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_change_invocations WHERE run_id = ?').get(tree['npc-a'].id)?.count).toBe(0) + } + finally { + shellResponse?.end('cleanup') + timing.dispose() + await f?.dispose() + server.closeAllConnections() + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) + } + }, 60000) +}) diff --git a/apps/buddy/service/src/__tests__/mutationConcurrencyFixture.ts b/apps/buddy/service/src/__tests__/mutationConcurrencyFixture.ts new file mode 100644 index 000000000..940c561ae --- /dev/null +++ b/apps/buddy/service/src/__tests__/mutationConcurrencyFixture.ts @@ -0,0 +1,400 @@ +import type { DatabaseSync } from 'node:sqlite' +import type { LocalBuddy } from '../../../shared/buddies/buddyApi' +import type { LocalTurnStart } from '../../../shared/conversation/chatApi' +import type { BuddyServiceRpcServer } from '../rpc/BuddyServiceRpcServer' +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { SessionManager } from '@earendil-works/pi-coding-agent' +import { expect, vi } from 'vitest' +import { DEFAULT_BUDDY_SETTINGS } from '../../../shared/buddies/buddySettings' +import { createBuddyUserContent } from '../../../shared/conversation/buddyUserContent' +import { readCheckpoint } from '../agent/sessions/tree/BuddyTreeStore' +import { startBuddyService } from '../BuddyService' +import { createChangeSetRepository } from '../changes/changeSetRepository' +import { MutationCoordinator } from '../changes/MutationCoordinator' +import { createRunEventLog } from '../events/createRunEventLog' +import { openBuddyDatabase } from '../storage/database' +import { createRunRepository } from '../storage/runRepository' + +export const roles = ['root', 'actor', 'npc-a', 'npc-b'] as const +export type Role = typeof roles[number] +interface Payload { + model: string + messages: { role: string, content?: unknown, tool_call_id?: string }[] + tools?: { function: { name: string } }[] +} +export interface ToolCall { + id: string + name: string + arguments: Record +} +type Reply = { text: string } | { toolCall: ToolCall } + +export function tool(id: string, name: string, args: Record): ToolCall { + return { id, name, arguments: args } +} + +export function deferred() { + return Promise.withResolvers() +} + +export function hash(text: string) { + return createHash('sha256').update(text).digest('hex') +} + +function requestRole(payload: Payload): Role { + const system = JSON.stringify(payload.messages.filter(message => message.role === 'system' || message.role === 'developer')) + const identities = ['ROOT', 'ACTOR', 'NPC'].filter(role => system.includes(`MUTATION_ROLE_${role}`)) + if (identities.length !== 1) + throw new Error(`Expected one private role marker, got ${identities.join(',')}`) + if (identities[0] === 'ROOT') + return 'root' + if (identities[0] === 'ACTOR') + return 'actor' + const firstUser = JSON.stringify(payload.messages.find(message => message.role === 'user')?.content) + const task = /MUTATION_TASK:(npc-a|npc-b)\b/.exec(firstUser ?? '')?.[1] + if (task !== 'npc-a' && task !== 'npc-b') + throw new Error('NPC initial user task has no unique identity') + return task +} + +function rpcHarness() { + const handlers = new Map[1]>() + const rpc = { + close() {}, + notify() {}, + onNotification() { return () => {} }, + onRequest(method: string, handler: Parameters[1]) { + handlers.set(method, handler) + return () => handlers.delete(method) + }, + async request(method: string) { + switch (method) { + case 'extensions.agent.list': return [] + case 'host.images.preferences': return {} + case 'host.runtimePreferences.get': return { cacheWarming: 'off', codemode: false, modelRetryLimit: 0 } + case 'host.credentials.list': return { ok: true, providers: [{ providerId: 'mutation-local', type: 'api_key' }] } + case 'host.credentials.read': return { ok: true, value: { type: 'api_key', key: 'local-test-only' } } + default: throw new Error(`Unexpected host RPC: ${method}`) + } + }, + } as unknown as BuddyServiceRpcServer + return { + rpc, + async invoke(method: string, params: unknown): Promise { + const handler = handlers.get(method) + if (!handler) + throw new Error(`Missing Runtime RPC: ${method}`) + return handler(params) + }, + } +} + +export async function createConcurrencyRuntime(options: { + steps: (role: Role) => ToolCall[] + readOnly?: boolean + fullAccess?: boolean + prepare?: (database: DatabaseSync, buddyHome: string) => Promise +}) { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'lexora-mutation-tree-'))) + const buddyHome = join(directory, 'buddy') + const skills = join(directory, 'skills') + await mkdir(skills) + const database = openBuddyDatabase({ databasePath: join(buddyHome, 'buddy.sqlite3') }) + const eventLog = createRunEventLog({ conversationsDirectory: join(buddyHome, 'conversations'), database }) + const events: Awaited> = [] + const subscription = eventLog.onDidCommit(event => events.push(event)) + const harness = rpcHarness() + const ready = deferred() + const begin = deferred() + const requests: { role: Role, payload: Payload }[] = [] + const errors: unknown[] = [] + const cursors = new Map() + const modelGates: ReturnType[] = [] + const holds = new Map>() + let npcId = '' + let actorId = '' + let runtime: Awaited> | undefined + + const server = createServer((request, response) => { + void (async () => { + if (request.method !== 'POST' || request.url !== '/v1/chat/completions') + throw new Error(`Unexpected local Provider request: ${request.method} ${request.url}`) + const chunks: Buffer[] = [] + for await (const chunk of request) + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + const payload = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Payload + const role = requestRole(payload) + requests.push({ role, payload }) + const cursor = cursors.get(role) ?? 0 + cursors.set(role, cursor + 1) + if (role === 'root' && cursor === 0) { + ready.resolve() + await begin.promise + } + const delegation = role === 'root' + ? [ + tool('root-delegate-actor', 'lexora_agents', { operation: 'delegate', targetId: actorId, text: 'MUTATION_TASK:actor' }), + tool('root-delegate-npc', 'lexora_agents', { operation: 'delegate', targetId: npcId, text: 'MUTATION_TASK:npc-a' }), + ] + : role === 'actor' + ? [tool('actor-delegate-npc', 'lexora_agents', { operation: 'delegate', targetId: npcId, text: 'MUTATION_TASK:npc-b' })] + : [] + const call = [...delegation, ...options.steps(role)][cursor] + if (call) + await holds.get(call.id) + const reply: Reply = call ? { toolCall: call } : { text: `MUTATION_DONE:${role}` } + const common = { created: 1_786_759_963, id: `completion-${requests.length}`, model: payload.model, object: 'chat.completion.chunk' } + response.writeHead(200, { 'cache-control': 'no-cache', 'content-type': 'text/event-stream' }) + response.write(`data: ${JSON.stringify({ + ...common, + choices: [{ + delta: { role: 'assistant', ...('text' in reply + ? { content: reply.text } + : { tool_calls: [{ index: 0, id: reply.toolCall.id, type: 'function', function: { name: reply.toolCall.name, arguments: JSON.stringify(reply.toolCall.arguments) } }] }) }, + finish_reason: null, + index: 0, + }], + })}\n\n`) + response.write(`data: ${JSON.stringify({ ...common, choices: [{ delta: {}, finish_reason: call ? 'tool_calls' : 'stop', index: 0 }] })}\n\n`) + response.write(`data: ${JSON.stringify({ ...common, choices: [], usage: { completion_tokens: 4, prompt_tokens: 10, total_tokens: 14 } })}\n\n`) + response.end('data: [DONE]\n\n') + })().catch((error) => { + errors.push(error) + response.writeHead(500).end('Local Provider scenario failed') + }) + }) + + async function dispose() { + begin.resolve() + for (const gate of modelGates) + gate.resolve() + try { + await runtime?.dispose() + } + finally { + subscription.dispose() + await eventLog.close() + server.closeAllConnections() + if (server.listening) + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) + database.close() + await rm(directory, { force: true, recursive: true }) + } + } + + try { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const address = server.address() + if (!address || typeof address === 'string') + throw new Error('Local Provider did not bind TCP') + await options.prepare?.(database, buddyHome) + runtime = await startBuddyService({ buddyHome, database, eventLog, rpc: harness.rpc, builtinSkillsDirectories: [skills] }) + await harness.invoke('providers.upsertCustom', { + api: 'openai-completions', + baseUrl: `http://127.0.0.1:${address.port}/v1`, + displayName: 'Local mutation scenario', + enabled: true, + id: 'mutation-local', + models: [{ contextWindow: 131072, id: 'mutation-model', input: ['text'], maxTokens: 4096, name: 'Local mutation model', reasoning: false }], + }) + const npc = await harness.invoke('buddies.create', { kind: 'npc', name: 'Repeated mutation worker' }) as LocalBuddy + const actor = await harness.invoke('buddies.create', { kind: 'actor', name: 'Mutation actor' }) as LocalBuddy + const root = await harness.invoke('buddies.get', { buddyId: 'default' }) as LocalBuddy + npcId = npc.id + actorId = actor.id + for (const [buddy, role] of [[npc, 'NPC'], [actor, 'ACTOR'], [root, 'ROOT']] as const) { + await harness.invoke('buddies.update', { + buddyId: buddy.id, + expectedRevision: buddy.revision, + settings: { + ...DEFAULT_BUDDY_SETTINGS, + instructions: `MUTATION_ROLE_${role}`, + model: role === 'ACTOR' ? { modelId: 'mutation-model', providerId: 'mutation-local', reasoning: null, serviceTier: null } : null, + skills: { mode: 'none' }, + tools: { mode: 'selected', ids: ['read', 'write', 'edit', ...(options.fullAccess ? [process.platform === 'win32' ? 'powershell' : 'bash'] : [])].map(name => JSON.stringify(['builtin', 'core', name])) }, + limits: { fileAccess: options.readOnly && role === 'ROOT' ? 'read_only' : 'inherit' }, + npcIds: role === 'NPC' ? [] : [npcId], + }, + }) + } + const content = createBuddyUserContent('MUTATION_TASK:root') + const executionConfig = { approvalPolicy: 'policy', executionProfile: options.fullAccess ? 'full_access' : 'workspace_write' } + const modelSelection = { modelId: 'mutation-model', providerId: 'mutation-local', reasoning: null, serviceTier: null } + const opened = await harness.invoke('composerDrafts.open', { + draftId: 'mutation-draft', + initialContent: content, + initialExecutionConfig: executionConfig, + initialModelSelection: modelSelection, + scope: { kind: 'global' }, + }) as { draftId: string, revision: number } + const saved = await harness.invoke('composerDrafts.save', { + content, + draftId: opened.draftId, + executionConfig, + expectedRevision: opened.revision, + modelSelection, + }) as { draftId: string, revision: number } + const turn = await harness.invoke('chat.startTurn', { + draftId: saved.draftId, + expectedRevision: saved.revision, + requestId: 'mutation-turn', + }) as LocalTurnStart + await expect.poll(() => requests.some(request => request.role === 'root'), { timeout: 15000 }).toBe(true) + await ready.promise + await harness.invoke('conversations.addMember', { conversationId: turn.conversationId, buddyId: actorId }) + const workspace = join(buddyHome, 'conversations', turn.conversationId, 'workspace') + const runs = createRunRepository(database) + return { + directory, + buddyHome, + database, + eventLog, + events, + requests, + errors, + workspace, + turn, + runs, + npcId, + actorId, + invoke: harness.invoke, + start: () => begin.resolve(), + holdReply(toolCallId: string) { + const gate = deferred() + modelGates.push(gate) + holds.set(toolCallId, gate.promise) + return gate + }, + async complete() { + await expect.poll(() => runs.findById(turn.runId)?.status, { timeout: 25000 }).toBe('completed') + expect(errors).toEqual([]) + expect(events.filter(event => event.type === 'approval.requested')).toEqual([]) + }, + dispose, + } + } + catch (error) { + await dispose() + throw error + } +} + +export type ConcurrencyRuntime = Awaited> + +// Only delay delivery of a real lease; admission, native locking and cancellation stay real. +export function mutationTiming() { + const original = MutationCoordinator.prototype.acquire + type Admission = Parameters[0] + const gates: { match: (input: Admission) => boolean, release: ReturnType, admissions: { input: Admission, coordinator: MutationCoordinator }[] }[] = [] + const spy = vi.spyOn(MutationCoordinator.prototype, 'acquire').mockImplementation(async function (this: MutationCoordinator, input) { + const gate = gates.find(gate => gate.match(input)) + const acquiring = original.call(this, input) + gate?.admissions.push({ input, coordinator: this }) + const lease = await acquiring + if (gate) + await gate.release.promise + return lease + }) + return { + gate(match: (input: Admission) => boolean) { + const gate = { match, release: deferred(), admissions: [] as { input: Admission, coordinator: MutationCoordinator }[] } + gates.push(gate) + return { + release: () => gate.release.resolve(), + admissions: gate.admissions, + async waitFor(active: number, pending: number) { + await expect.poll(() => { + const states = [...new Map(gate.admissions.map(({ input, coordinator }) => [input.runId, coordinator.getRunState(input.runId)])).values()] + return { + admissions: gate.admissions.length, + active: states.reduce((sum, state) => sum + state.active, 0), + pending: states.reduce((sum, state) => sum + state.pending, 0), + } + }, { timeout: 15000 }).toEqual({ admissions: active + pending, active, pending }) + }, + } + }, + dispose() { + for (const gate of gates) + gate.release.resolve() + spy.mockRestore() + }, + } +} + +export function treeByRole(f: ConcurrencyRuntime) { + const tree = f.runs.listTree(f.turn.runId) + expect(tree).toHaveLength(4) + const root = tree.find(run => run.id === f.turn.runId)! + const actor = tree.find(run => run.executionKind === 'actor_child')! + const npcA = tree.find(run => run.executionKind === 'npc_child' && run.parentRunId === root.id)! + const npcB = tree.find(run => run.executionKind === 'npc_child' && run.parentRunId === actor.id)! + expect(actor).toMatchObject({ parentRunId: root.id, buddyId: f.actorId }) + expect([npcA.buddyId, npcB.buddyId]).toEqual([f.npcId, f.npcId]) + expect(new Set(tree.map(run => run.threadId)).size).toBe(4) + expect(new Set(tree.map(run => run.piSessionFile)).size).toBe(4) + for (const run of tree) { + expect(run.rootRunId).toBe(root.id) + expect(run.actorSnapshot?.settings.npcConcurrency).toEqual({ mode: 'limited', limit: 5 }) + } + return { root, actor, 'npc-a': npcA, 'npc-b': npcB } +} + +export async function assertDurableTree(f: ConcurrencyRuntime, cancelled: Role[] = []) { + const tree = treeByRole(f) + for (const role of roles) { + const run = tree[role] + const status = cancelled.includes(role) ? 'cancelled' : 'completed' + expect(run.status).toBe(status) + const persisted = await f.eventLog.read(run.id) + expect(persisted.filter(event => event.type === `run.${status}`)).toHaveLength(1) + const journal = SessionManager.open(run.piSessionFile!) + expect(JSON.stringify(journal.getEntries())).toContain(`MUTATION_TASK:${role}`) + if (status === 'completed') { + const after = journal.getEntries().filter(entry => readCheckpoint(entry)?.runId === run.id && readCheckpoint(entry)?.position === 'after') + expect(after).toHaveLength(1) + expect(JSON.stringify(journal.getBranch(after[0]!.id))).toContain(`MUTATION_DONE:${role}`) + } + } + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_mutation_leases WHERE run_id IN (?, ?, ?, ?) AND status <> \'released\'').get(...Object.values(tree).map(run => run.id))?.count).toBe(0) + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_change_invocations WHERE status = \'pending\' OR (tool_name IN (\'write\', \'edit\') AND status <> \'completed\')').get()?.count).toBe(0) + return tree +} + +export function captures(f: ConcurrencyRuntime) { + const repository = createChangeSetRepository(f.database) + const sets = repository.listSetsForRunTrees([f.turn.runId]) + for (const set of sets) + expect(set).toMatchObject({ status: 'completed', coverage: 'complete' }) + return sets.flatMap(set => repository.listCaptures(set.id)).sort((a, b) => a.invocation!.sequence - b.invocation!.sequence) +} + +export async function assertCaptureChain(f: ConcurrencyRuntime, path: string, before: string, after: string, count: number) { + const chain = captures(f).filter(capture => capture.canonicalPath === path) + expect(chain).toHaveLength(count) + expect(chain[0]!.before.hash).toBe(hash(before)) + expect(chain.at(-1)!.after?.hash).toBe(hash(after)) + for (const [index, capture] of chain.entries()) { + expect(capture.status).toBe('completed') + expect(capture.invocation).toMatchObject({ status: 'completed', toolCallId: capture.toolCallId }) + expect(capture.actorId).toBe(f.runs.findById(capture.invocation!.runId)?.buddyId) + const lease = f.database.prepare('SELECT run_id, segment_id, status FROM run_mutation_leases WHERE invocation_id = ?').get(capture.invocationId!) + expect(lease).toMatchObject({ run_id: capture.invocation!.runId, segment_id: capture.invocation!.segmentId, status: 'released' }) + if (index) + expect(capture.before.hash).toBe(chain[index - 1]!.after?.hash) + for (const state of [capture.before, capture.after!]) { + expect(state.kind).toBe('text') + expect(hash(await readFile(state.snapshotPath!, 'utf8'))).toBe(state.hash) + } + } + return chain +} diff --git a/apps/buddy/service/src/__tests__/mutationContinuation.integration.spec.ts b/apps/buddy/service/src/__tests__/mutationContinuation.integration.spec.ts new file mode 100644 index 000000000..dca193495 --- /dev/null +++ b/apps/buddy/service/src/__tests__/mutationContinuation.integration.spec.ts @@ -0,0 +1,72 @@ +import type { LocalChangeRecoveryItem } from '../../../shared/changes/changeApi' +import { randomUUID } from 'node:crypto' +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import process from 'node:process' +import { expect, it } from 'vitest' +import { resolveBuddyPlatform } from '../../../shared/platform' +import { createMutationRepository } from '../changes/mutationRepository' +import { createConversationRepository } from '../storage/conversationRepository' +import { createRunRepository } from '../storage/runRepository' +import { assertDurableTree, createConcurrencyRuntime, roles, tool } from './mutationConcurrencyFixture' + +it.skipIf(!resolveBuddyPlatform(process.platform).hostShellSupervision)('starts a real execution tree and supervised command despite persisted unknown shell and MCP outcomes', async () => { + const invocationIds = [randomUUID(), randomUUID()] + let workspace = '' + let command = '' + const f = await createConcurrencyRuntime({ + fullAccess: true, + async prepare(database) { + const now = new Date().toISOString() + const conversationId = randomUUID() + const messageId = randomUUID() + const runId = randomUUID() + const conversations = createConversationRepository(database) + conversations.create({ id: conversationId, branchId: conversationId, createdAt: now, approvalPolicy: 'policy', executionProfile: 'full_access', spaceId: null, title: 'Legacy interrupted fixture' }) + conversations.createMessage({ id: messageId, conversationId, branchId: conversationId, role: 'user', runId: null, createdAt: now, content: { text: 'Legacy fixture; no external request is issued' } }) + createRunRepository(database).create({ id: runId, conversationId, branchId: conversationId, triggeringMessageId: messageId, provider: 'fixture', model: 'fixture', purpose: 'chat', status: 'failed', startedAt: now, completedAt: now, errorCode: 'RUNTIME_RESTARTED', approvalPolicy: 'policy', executionProfile: 'full_access', piSessionFile: null }) + const repository = createMutationRepository(database) + for (const [index, kind] of ['legacy-shell', 'mcp'].entries()) { + const invocationId = invocationIds[index]! + repository.createLease({ invocationId, runId, segmentId: randomUUID(), lifetime: 'external', claims: [{ kind: 'host' }] }) + repository.recordProcess(invocationId, JSON.stringify({ kind, nonce: 'unverifiable-legacy-fixture' }), 'unknown') + repository.settle(invocationId, 'hazard', 'OWNED_PROCESS_CLEANUP_UNCONFIRMED') + } + }, + steps: role => [ + tool(`continue-${role}`, 'write', { path: join(workspace, `${role}.txt`), content: `continued:${role}\n` }), + ...(role === 'root' ? [tool('continue-shell', process.platform === 'win32' ? 'powershell' : 'bash', { command, timeout: 40 })] : []), + ], + }) + try { + workspace = f.workspace + const script = join(workspace, 'continue-once.cjs') + const marker = join(workspace, 'continued.log') + await writeFile(script, 'require(\'node:fs\').appendFileSync(process.argv[2], \'new-effect\\n\');\n') + const quote = (value: string) => process.platform === 'win32' ? `'${value.replaceAll('\'', '\'\'')}'` : `'${value.replaceAll('\'', '\'\\\'\'')}'` + command = `${process.platform === 'win32' ? '& ' : ''}${[process.execPath, script, marker].map(quote).join(' ')}` + expect(await f.invoke('changes.recovery.list', { conversationId: f.turn.conversationId })).toEqual([]) + f.start() + await f.complete() + await assertDurableTree(f) + for (const role of roles) + expect(await readFile(join(workspace, `${role}.txt`), 'utf8')).toBe(`continued:${role}\n`) + expect(await readFile(marker, 'utf8')).toBe('new-effect\n') + const operations = f.events.filter(event => event.type === 'tool.completed' && String((event.payload as { toolCallId?: string }).toolCallId).startsWith('continue-')) + expect(operations).toHaveLength(5) + expect(operations.every(event => (event.payload as { isError?: boolean }).isError === false)).toBe(true) + const notices = await f.invoke('changes.recovery.list', { conversationId: f.turn.conversationId }) as LocalChangeRecoveryItem[] + expect(notices.map(item => item.invocationId).sort()).toEqual([...invocationIds].sort()) + expect(notices.every(item => item.status === 'unknown' && item.taskTitle === 'Legacy interrupted fixture')).toBe(true) + const affectedRunIds = f.runs.listTree(f.turn.runId).map(run => run.id).sort() + for (const invocationId of invocationIds) { + const overlaps = f.database.prepare('SELECT related_run_id, reason FROM run_mutation_evidence WHERE invocation_id = ? AND related_run_id IS NOT NULL').all(invocationId) + expect(overlaps.map(item => item.related_run_id).sort()).toEqual(affectedRunIds) + expect(overlaps.every(item => item.reason === 'RESOURCE_OVERLAP')).toBe(true) + expect(f.database.prepare('SELECT status, reason FROM run_mutation_leases WHERE invocation_id = ?').get(invocationId)).toMatchObject({ status: 'hazard', reason: 'OWNED_PROCESS_CLEANUP_UNCONFIRMED' }) + expect(f.database.prepare('SELECT settlement FROM run_mutation_processes WHERE invocation_id = ?').get(invocationId)).toMatchObject({ settlement: 'unknown' }) + expect(f.database.prepare('SELECT COUNT(*) AS count FROM run_mutation_evidence WHERE invocation_id = ? AND (kind = \'waived\' OR outcome = \'applied\')').get(invocationId)?.count).toBe(0) + } + } + finally { await f.dispose() } +}, 60000) diff --git a/apps/buddy/service/src/agent/execution/ExecutionCoordinator.ts b/apps/buddy/service/src/agent/execution/ExecutionCoordinator.ts index 3bf8d52a9..d7a55e7bc 100644 --- a/apps/buddy/service/src/agent/execution/ExecutionCoordinator.ts +++ b/apps/buddy/service/src/agent/execution/ExecutionCoordinator.ts @@ -11,7 +11,6 @@ import type { RunExecutionOutcome } from './RunExecutionBackend' import type { BuddyTurnHandle } from './turnTypes' import { Emitter } from '../../../../shared/events/Emitter' import { isExecutionProgress } from '../../../../shared/executions/executionApi' -import { MutationCoordinatorError } from '../../changes/MutationCoordinator' import { RunEventCorruptionError, RunEventLogUnavailableError } from '../../events/RunEventFailure' import { readStableRunErrorCode } from '../../runs/runError' import { ExecutionAdmissionError, executionFence } from '../../storage/executionRepository' @@ -37,6 +36,7 @@ export interface ExecutionCoordinatorOptions { getFileAccessCeiling?: (runId: string) => 'read_only' | undefined onObserverError?: (error: unknown) => void hostShellSupervision?: boolean + recoverQuarantinedRun?: (runId: string) => boolean } class ShellSupervisionUnavailableError extends Error { @@ -139,9 +139,7 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { } async admitChild(input: AdmitChildExecutionInput): Promise { - const effects = await this.#options.mutations.waitForRun(input.callerId) - if (effects.hazards.length) - throw new MutationCoordinatorError('MUTATION_RESOURCE_UNSAFE') + await this.#options.mutations.waitForRun(input.callerId) const rootRunId = this.#require(input.callerId).run.rootRunId return this.#serialize(rootRunId, async () => { if (this.#stopped) @@ -149,8 +147,6 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { if (this.#uncontainedShellRoots.has(rootRunId)) throw new ShellSupervisionUnavailableError() const currentEffects = this.#options.mutations.getRunState(input.callerId) - if (currentEffects.hazards.length) - throw new MutationCoordinatorError('MUTATION_RESOURCE_UNSAFE') if (currentEffects.active || currentEffects.pending) throw new ExecutionAdmissionError('busy') this.#options.availability?.assertRunExecutable(input.callerId) @@ -232,10 +228,11 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { }) } - async wait(runId: string, targetRunIds: readonly string[], timeoutMs: number, signal: AbortSignal, deliveredMessageIds: readonly string[] = []): Promise { - await this.#options.mutations.waitForRun(runId) + async wait(runId: string, targetRunIds: readonly string[], timeoutMs: number, signal: AbortSignal, deliveredMessageIds: readonly string[] = []): Promise<'changed' | 'timedout'> { const deadline = AbortSignal.timeout(timeoutMs) + const waitingSignal = AbortSignal.any([signal, deadline]) try { + await this.#options.mutations.waitForRun(runId, waitingSignal) for (;;) { signal.throwIfAborted() const caller = this.#require(runId) @@ -246,18 +243,21 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { && !isExecutionProgress(message) && (!targetRunIds.length || message.kind !== 'result' || targetRunIds.includes(message.sourceRunId)), ) - if (pending || targetRunIds.some(id => this.#require(id).phase === 'closed') || deadline.aborted) - return + if (pending || targetRunIds.some(id => this.#require(id).phase === 'closed')) + return 'changed' + if (deadline.aborted) + return 'timedout' if (caller.phase === 'working') { this.#options.executions.markWaiting(this.#fence(caller)) this.#changed(caller.run.rootRunId) } - await this.#wait(caller.run.rootRunId, revision, AbortSignal.any([signal, deadline])) + await this.#wait(caller.run.rootRunId, revision, waitingSignal) } } catch (error) { if (!deadline.aborted || signal.aborted) throw error + return 'timedout' } finally { if (!signal.aborted) @@ -273,18 +273,16 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { this.#options.availability?.assertRunExecutable(runId) } catch (error) { - if (!(error instanceof RunEventLogUnavailableError)) - throw error - const records = this.#options.executions.listTree(source.run.rootRunId) - for (const record of records) - this.#preparations.get(record.run.id)?.controller.abort(error) - const stopped = await Promise.allSettled(records.map(record => stop(record.run.id))) - const failures = stopped.flatMap(result => result.status === 'rejected' ? [result.reason] : []) - if (!(error instanceof RunEventCorruptionError)) - failures.unshift(error) - if (failures.length) - throw new AggregateError(failures, 'Execution cleanup could not settle an unavailable tree') - return true + if (!(error instanceof RunEventCorruptionError)) { + if (!(error instanceof RunEventLogUnavailableError)) + throw error + const records = this.#options.executions.listTree(source.run.rootRunId) + for (const record of records) + this.#preparations.get(record.run.id)?.controller.abort(error) + const stopped = await Promise.allSettled(records.map(record => stop(record.run.id))) + const failures = stopped.flatMap(result => result.status === 'rejected' ? [result.reason] : []) + throw new AggregateError([error, ...failures], 'Execution cleanup could not settle an unavailable tree') + } } const affected = await this.#serialize(source.run.rootRunId, async () => { const records = this.#options.executions.requestCancel(runId, new Date().toISOString()) @@ -293,23 +291,33 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { this.#changed(source.run.rootRunId) return records.filter(record => record.phase !== 'closed') }) - await Promise.all(affected.map(async (record) => { + const stopped = await Promise.allSettled(affected.map(async (record) => { const preparation = this.#preparations.get(record.run.id) - if (await stop(record.run.id)) - return + const owned = await stop(record.run.id) if (preparation) await preparation.settled - const current = this.#require(record.run.id) - if (current.phase !== 'closed' && current.run.status === 'queued' && !this.#children.has(record.run.id)) - await this.finalize({ runId: record.run.id, cleanup: 'completed', outcome: { status: 'cancelled', errorCode } }) + if (!owned) + await this.#children.get(record.run.id) })) + const failures = stopped.flatMap(result => result.status === 'rejected' ? [result.reason] : []) + if (failures.length) + throw new AggregateError(failures, 'Execution cleanup could not stop all owners') + for (const record of affected.toReversed()) { + if (await this.#recoverQuarantined(record.run.id)) + continue + const current = this.#require(record.run.id) + if (current.phase !== 'closed' && !this.#children.has(record.run.id)) + await this.finalize({ runId: record.run.id, cleanup: current.cleanupState === 'degraded' ? 'degraded' : 'completed', outcome: { status: 'cancelled', errorCode } }) + } return true } async failBeforeStart(runId: string, error: unknown): Promise { - if (error instanceof RunEventLogUnavailableError) + if (error instanceof RunEventLogUnavailableError && (!(error instanceof RunEventCorruptionError) || error.runId !== runId)) throw error - const current = this.#require(runId) + const current = this.#options.executions.find(runId) + if (!current) + throw new ExecutionAdmissionError('stale') if (current.phase === 'closed') return current.run const cancelled = current.cancelRequestedAt || (error instanceof Error && error.name === 'AbortError') @@ -321,6 +329,54 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { } async finalize(input: Parameters[0]): Promise { + try { + return await this.#finalize(input) + } + catch (error) { + if (!(error instanceof RunEventCorruptionError) || error.runId !== input.runId) + throw error + return this.#finalizeQuarantined(input.runId, error) + } + } + + async #finalizeQuarantined(runId: string, error: RunEventCorruptionError): Promise { + const source = this.#options.executions.find(runId) + if (!source || !this.#options.recoverQuarantinedRun) + throw error + if (source.phase === 'closed') + return source.run + const affected = this.#options.executions.requestCancel(runId, new Date().toISOString()) + for (const record of affected) + this.#preparations.get(record.run.id)?.controller.abort(error) + const children = affected.filter(record => record.run.parentRunId === runId && record.phase !== 'closed') + await Promise.all(children.map(child => this.cancel(child.run.id, 'RUN_CANCELLED', id => this.#options.stopExecution(id)))) + const result = await this.#recoverQuarantined(runId) + if (!result) + throw error + return result + } + + async #recoverQuarantined(runId: string): Promise { + try { + this.#options.availability?.assertRunExecutable(runId) + return null + } + catch (error) { + if (!(error instanceof RunEventCorruptionError) || error.runId !== runId || !this.#options.recoverQuarantinedRun) + throw error + await this.#options.mutations.waitForRun(runId) + this.#options.recoverQuarantinedRun(runId) + const current = this.#options.executions.find(runId) + if (!current || current.phase !== 'closed') + throw error + this.#changed(current.run.rootRunId) + if (current.run.parentRunId === null) + this.#uncontainedShellRoots.delete(current.run.rootRunId) + return current.run + } + } + + async #finalize(input: Parameters[0]): Promise { const initial = this.#require(input.runId) if (initial.phase === 'closed') return initial.run @@ -386,7 +442,10 @@ export class ExecutionCoordinator implements ExecutionLifecyclePort { record = this.#require(runId) const mode = record.cancelRequestedAt || outcome.status === 'cancelled' ? 'cancel' : outcome.status === 'failed' ? 'failure' : 'success' try { - const closing = this.#options.executions.beginClosing(this.#fence(record), mode) + const root = this.#options.executions.find(record.run.rootRunId) + if (!root) + throw new ExecutionAdmissionError('stale') + const closing = this.#options.executions.beginClosing(executionFence(record, root), mode) this.#changed(record.run.rootRunId) return { result: { kind: 'sealed' as const, closingGeneration: closing.run.generation }, stopChildren } } diff --git a/apps/buddy/service/src/agent/execution/ExecutionDelegationService.ts b/apps/buddy/service/src/agent/execution/ExecutionDelegationService.ts index a2f5174eb..94c7681e9 100644 --- a/apps/buddy/service/src/agent/execution/ExecutionDelegationService.ts +++ b/apps/buddy/service/src/agent/execution/ExecutionDelegationService.ts @@ -132,8 +132,11 @@ export class ExecutionDelegationService { const delivered = this.#options.mailbox.listUnconsumed(source.run.id).filter(message => !unseen.has(message.messageId)).map(message => message.messageId) const awaitingAnswer = this.#options.mailbox.listQuestions(source.run.id) .some(message => message.kind === 'question' && this.#options.mailbox.findQuestion(message.questionId)?.open) - if (targets.length || pending.length || awaitingAnswer) - await this.#options.coordinator.wait(source.run.id, targets, input.timeoutMs ?? 30000, signal, delivered) + if (targets.length || pending.length || awaitingAnswer) { + const waitStatus = await this.#options.coordinator.wait(source.run.id, targets, input.timeoutMs ?? 30000, signal, delivered) + if (waitStatus === 'timedout') + return { ...this.status(source.run.id, target.run.id, entries), waitStatus } + } } return this.status(source.run.id, target.run.id, entries) } diff --git a/apps/buddy/service/src/agent/execution/ExecutionPresentationService.ts b/apps/buddy/service/src/agent/execution/ExecutionPresentationService.ts index 7c1e59742..a9ea679ae 100644 --- a/apps/buddy/service/src/agent/execution/ExecutionPresentationService.ts +++ b/apps/buddy/service/src/agent/execution/ExecutionPresentationService.ts @@ -3,7 +3,7 @@ import type { BuddiesService } from '../../buddies/BuddiesService' import type { ChangeCaptureService } from '../../changes/ChangeCaptureService' import type { MutationCoordinator } from '../../changes/MutationCoordinator' import type { BuddyRunEvent } from '../../events/BuddyRunEvent' -import type { RunEventObservation, RunEventReader } from '../../events/RunEventPorts' +import type { RunEventAvailability, RunEventObservation, RunEventReader } from '../../events/RunEventPorts' import type { ApprovalRepository } from '../../storage/approvalRepository' import type { ArtifactRepository } from '../../storage/artifactRepository' import type { ConversationRepository } from '../../storage/conversationRepository' @@ -31,7 +31,7 @@ export interface ExecutionPresentationOptions { buddies: Pick coordinator: Pick mutations: Pick - events: Pick & Pick + events: Pick & Pick & Partial> artifacts: Pick changes: Pick usage: Pick @@ -221,6 +221,8 @@ export class ExecutionPresentationService { #present(scope: ExecutionScope, record: ExecutionStateRecord, tree: readonly ExecutionStateRecord[], events: readonly BuddyRunEvent[], toolCount: number): LocalExecution { const { run } = record + const quarantined = this.#options.events.getRecoveryFailures?.().some(failure => failure.runId === run.id + || (run.id === run.rootRunId && tree.some(child => child.run.id === failure.runId))) ?? false const progressEvent = events.findLast(event => event.type === 'run.progress') const parsedProgress = buddyRunProgressSchema.safeParse(progressEvent?.payload) const progress = parsedProgress.success ? parsedProgress.data : null @@ -260,8 +262,8 @@ export class ExecutionPresentationService { && run.branchId === scope.branchId && this.#options.conversations.findById(scope.conversationId)?.activeBranchId === scope.branchId, canSend: mutable && (run.executionKind === 'actor_child' || run.executionKind === 'npc_child'), canAnswer: mutable && questions.some(question => question.open && question.audience === 'user'), - cleanup: mutations.hazards.length ? 'degraded' : record.cleanupState, - errorCode: record.cleanupErrorCode ?? run.errorCode, + cleanup: quarantined || mutations.hazards.length ? 'degraded' : record.cleanupState, + errorCode: quarantined ? 'EVENT_LOG_CORRUPTED' : record.cleanupErrorCode ?? run.errorCode, affectedResources: [...new Map(affectedResources.map(resource => [resource.kind === 'host' ? 'host' : resource.path, resource])).values()], questions, } diff --git a/apps/buddy/service/src/agent/execution/__tests__/ExecutionCoordinator.spec.ts b/apps/buddy/service/src/agent/execution/__tests__/ExecutionCoordinator.spec.ts index e4c78715c..793281d3b 100644 --- a/apps/buddy/service/src/agent/execution/__tests__/ExecutionCoordinator.spec.ts +++ b/apps/buddy/service/src/agent/execution/__tests__/ExecutionCoordinator.spec.ts @@ -1,3 +1,4 @@ +import type { RunEventLogUnavailableError } from '../../../events/RunEventFailure' import type { RunRecord } from '../../../storage/runRecord' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -9,6 +10,7 @@ import { EXECUTION_MAILBOX_ENTRY_TYPE } from '../../../../../shared/executions/e import { BuddyDefinitionResolver } from '../../../buddies/BuddyDefinitionResolver' import { MutationCoordinator } from '../../../changes/MutationCoordinator' import { createRunEventLog } from '../../../events/createRunEventLog' +import { RunEventCorruptionError, RunEventStorageError } from '../../../events/RunEventFailure' import { RunLifecycleService } from '../../../runs/RunLifecycleService' import { createExecutionStorageFixture, executionNow } from '../../../storage/__tests__/executionFixture' import { createConversationMemberRepository } from '../../../storage/conversationMemberRepository' @@ -31,14 +33,32 @@ async function fixture(stopExecution: (runId: string) => Promise = asyn const lifecycle = new RunLifecycleService({ eventLog: events, repository: storage.runs }) const mutations = new MutationCoordinator() const childDone = Promise.withResolvers() + const unavailable = new Map() + const recovered: string[] = [] const coordinator = new ExecutionCoordinator({ hostShellSupervision, + availability: { assertRunExecutable: (id) => { + const error = unavailable.get(id) + if (error) + throw error + } }, + recoverQuarantinedRun: (id) => { + if (!(unavailable.get(id) instanceof RunEventCorruptionError)) + return false + recovered.push(id) + return storage.runs.reconcileQuarantined(id, executionNow) + }, executions: storage.executions, mailbox, events, lifecycle, mutations, - readResult: id => `Result from ${id}`, + readResult: (id) => { + const error = unavailable.get(id) + if (error) + throw error + return `Result from ${id}` + }, launchChild: async runId => ({ runId, completion: childDone.promise }), stopExecution, listIncomplete: () => storage.runs.listIncomplete(), @@ -52,7 +72,7 @@ async function fixture(stopExecution: (runId: string) => Promise = asyn storage.database.close() await rm(directory, { recursive: true, force: true }) }) - return { ...storage, events, mailbox, lifecycle, mutations, coordinator, childDone } + return { ...storage, events, mailbox, lifecycle, mutations, coordinator, childDone, unavailable, recovered } } function delegationService(f: Awaited>) { @@ -64,6 +84,131 @@ function delegationService(f: Awaited>) { } describe('execution coordination', () => { + it('closes a quarantined child only after its owner stops while healthy siblings and the root settle normally', async () => { + const stopped = Promise.withResolvers() + const entered = Promise.withResolvers() + const f = await fixture(async (id) => { + if (id === 'broken') { + entered.resolve() + await stopped.promise + return true + } + if (id === 'root') { + await f.coordinator.finalize({ runId: id, cleanup: 'completed', outcome: { status: 'cancelled', errorCode: 'RUN_CANCELLED' } }) + return true + } + return false + }) + const signal = new AbortController().signal + await f.coordinator.begin('root', signal) + f.executions.admitChild(f.child('broken')) + f.executions.admitChild(f.child('healthy')) + await f.coordinator.begin('broken', signal) + const lease = await f.mutations.acquire({ runId: 'broken', invocationId: 'active', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + f.unavailable.set('broken', new RunEventCorruptionError('broken')) + const cancelling = f.coordinator.cancel('root', 'RUN_CANCELLED', async (id) => { + if (id === 'broken') { + entered.resolve() + await stopped.promise + return true + } + return id === 'root' + }) + try { + await entered.promise + expect(f.recovered).toEqual([]) + expect(f.executions.find('broken')).toMatchObject({ phase: 'working', cleanupState: 'pending' }) + expect(f.executions.find('root')?.phase).not.toBe('closed') + lease.markHazard('FILE_RESULT_UNKNOWN') + stopped.resolve() + await cancelling + expect(f.executions.find('broken')).toMatchObject({ phase: 'closed', cleanupState: 'degraded', run: { status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' } }) + expect(f.executions.find('healthy')).toMatchObject({ phase: 'closed', run: { status: 'cancelled' } }) + expect(f.executions.find('root')).toMatchObject({ phase: 'closed', cleanupState: 'degraded', run: { status: 'cancelled' } }) + expect(await f.events.read('broken')).toEqual([]) + expect(f.mailbox.listUnconsumed('root').some(message => message.sourceRunId === 'broken')).toBe(false) + expect(f.mutations.getRunState('broken').hazards).toHaveLength(1) + f.runs.create({ id: 'new-root', buddyId: 'default', actorSnapshot: f.snapshot('default'), branchId: 'branch', conversationId: 'conversation', triggeringMessageId: 'new-goal', provider: 'fixture', model: 'fixture', purpose: 'chat', status: 'running', piSessionFile: null, startedAt: executionNow, approvalPolicy: 'policy', executionProfile: 'workspace_write' }) + await f.coordinator.begin('new-root', signal) + expect(f.executions.find('new-root')?.phase).toBe('working') + } + finally { + lease.release() + stopped.resolve() + await cancelling + } + }) + + it('does not close a quarantined root before its child and its actual mutation settle', async () => { + const childStopped = Promise.withResolvers() + const entered = Promise.withResolvers() + const f = await fixture(async (id) => { + if (id === 'child') { + entered.resolve() + await childStopped.promise + } + return false + }) + const signal = new AbortController().signal + await f.coordinator.begin('root', signal) + f.executions.admitChild(f.child('child')) + const lease = await f.mutations.acquire({ runId: 'root', invocationId: 'active', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + f.unavailable.set('root', new RunEventCorruptionError('root')) + const finalizing = f.coordinator.finalize({ runId: 'root', cleanup: 'completed', outcome: { status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' } }) + try { + await entered.promise + expect(f.recovered).toEqual([]) + childStopped.resolve() + await vi.waitFor(() => expect(f.executions.find('child')?.phase).toBe('closed')) + expect(f.recovered).toEqual([]) + expect(f.executions.find('root')?.phase).not.toBe('closed') + lease.release() + expect(await finalizing).toMatchObject({ status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' }) + expect(f.executions.find('root')).toMatchObject({ phase: 'closed', cleanupState: 'degraded' }) + expect(await f.events.read('root')).toEqual([]) + expect(f.database.prepare('SELECT count(*) AS count FROM execution_mailbox_receipts WHERE kind = \'consumed\'').get()).toEqual({ count: 0 }) + } + finally { + lease.release() + childStopped.resolve() + await finalizing + } + }) + + it('does not turn a shared storage failure into quarantined cleanup', async () => { + const f = await fixture() + const failure = new RunEventStorageError({ runId: 'root', firstSequence: null, lastSequence: null }, 'read', 'read', 'unknown') + f.unavailable.set('root', failure) + try { + await expect(f.coordinator.finalize({ runId: 'root', cleanup: 'completed', outcome: { status: 'failed', errorCode: 'EVENT_STORAGE_FAILED' } })).rejects.toBe(failure) + await expect(f.coordinator.cancel('root', 'RUN_CANCELLED', async () => false)).rejects.toMatchObject({ errors: [failure] }) + expect(f.recovered).toEqual([]) + expect(f.executions.find('root')?.phase).not.toBe('closed') + } + finally { f.unavailable.clear() } + }) + + it('reports a wait tool deadline while preserving real mutation and execution state', async () => { + const f = await fixture() + const signal = new AbortController().signal + await f.coordinator.begin('root', signal) + f.executions.admitChild(f.child('queued')) + const lease = await f.mutations.acquire({ runId: 'root', invocationId: 'active', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + const service = delegationService(f) + const caller = { run: { runId: 'root', signal, flushProjectedEvents: async () => {}, onToolExecutionAuthorized: async () => {} }, getPermissions: () => ({ approvalPolicy: 'policy' as const, executionProfile: 'workspace_write' as const }) } + try { + expect(await service.execute(caller, 'deadline', { operation: 'wait', timeoutMs: 10 }, signal)).toMatchObject({ + waitStatus: 'timedout', + phase: 'working', + status: 'running', + cleanup: 'pending', + children: [{ runId: 'queued', phase: 'queued' }], + }) + expect(f.mutations.getRunState('root')).toEqual({ active: 1, pending: 0, hazards: [] }) + } + finally { lease.release() } + }) + it('retains progress receipts without waking a parent until an actionable message arrives', async () => { const f = await fixture() const signal = new AbortController().signal @@ -290,6 +435,51 @@ describe('execution coordination', () => { expect(f.executions.find('actor')?.phase).toBe('working') }) + it.each(['timeout', 'cancel'] as const)('ends only the wait on %s while a real mutation and working permit remain owned', async (reason) => { + const f = await fixture() + const controller = new AbortController() + await f.coordinator.begin('root', controller.signal) + const lease = await f.mutations.acquire({ runId: 'root', invocationId: 'active', lifetime: 'runtime', claims: [{ kind: 'host' }], signal: controller.signal }) + let completed = false + const overlapping = f.mutations.acquire({ runId: 'later', invocationId: 'later', lifetime: 'runtime', claims: [{ kind: 'host' }], signal: new AbortController().signal }).then((next) => { + completed = true + next.release() + }) + try { + const waiting = f.coordinator.wait('root', [], reason === 'timeout' ? 10 : 10000, controller.signal) + if (reason === 'cancel') { + const cancelled = expect(waiting).rejects.toMatchObject({ name: 'AbortError' }) + controller.abort() + await cancelled + } + else { + expect(await waiting).toBe('timedout') + } + expect(f.executions.find('root')).toMatchObject({ phase: 'working', cleanupState: 'pending' }) + expect(f.mutations.getRunState('root')).toEqual({ pending: 0, active: 1, hazards: [] }) + expect(f.mutations.getRunState('later')).toMatchObject({ pending: 1, active: 0 }) + expect(completed).toBe(false) + } + finally { + lease.release() + await overlapping + } + expect(completed).toBe(true) + }) + + it('admits a child after a historical unknown without clearing its evidence', async () => { + const f = await fixture() + const signal = new AbortController().signal + await f.coordinator.begin('root', signal) + const lease = await f.mutations.acquire({ runId: 'root', invocationId: 'old', lifetime: 'external', claims: [{ kind: 'host' }], signal }) + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: 'unknown-process', settlement: 'unknown' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + const child = await f.coordinator.admitChild(f.child('after-unknown')) + expect(child.run.id).toBe('after-unknown') + expect(f.mutations.getHazards()).toHaveLength(1) + expect(f.mutations.getOwnedProcesses('old')).toEqual([{ processId: 'unknown-process', settlement: 'unknown' }]) + }) + it('reclaims the working permit when a wait reaches its deadline', async () => { const f = await fixture() const signal = new AbortController().signal @@ -361,7 +551,7 @@ describe('execution coordination', () => { await expect(f.coordinator.begin('child', new AbortController().signal)).rejects.toMatchObject({ code: 'stale' }) }) - it('keeps owned process hazards as degraded cleanup without blocking unrelated resources', async () => { + it('keeps owned process hazards as degraded cleanup without blocking new mutations', async () => { const f = await fixture() const lease = await f.mutations.acquire({ lifetime: 'runtime' as const, runId: 'root', invocationId: 'writer', claims: [{ kind: 'tree', canonicalPath: '/workspace/a' }], signal: new AbortController().signal }) lease.markHazard('PROCESS_CLEANUP_UNKNOWN') @@ -369,7 +559,9 @@ describe('execution coordination', () => { await port.prepareClose({ outcome: { status: 'failed', errorCode: 'TOOL_FAILED' }, consumedReceipts: [] }) await f.coordinator.finalize({ runId: 'root', cleanup: 'completed', outcome: { status: 'failed', errorCode: 'TOOL_FAILED' } }) expect(f.executions.find('root')).toMatchObject({ phase: 'closed', cleanupState: 'degraded' }) - await expect(f.mutations.acquire({ lifetime: 'runtime' as const, runId: 'later', invocationId: 'unsafe', claims: [{ kind: 'file', canonicalPath: '/workspace/a/result' }], signal: new AbortController().signal })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const next = await f.mutations.acquire({ lifetime: 'runtime' as const, runId: 'later', invocationId: 'next', claims: [{ kind: 'file', canonicalPath: '/workspace/a/result' }], signal: new AbortController().signal }) + next.release() + expect(f.mutations.getHazards()).toHaveLength(1) const safe = await f.mutations.acquire({ lifetime: 'runtime' as const, runId: 'later', invocationId: 'safe', claims: [{ kind: 'file', canonicalPath: '/workspace/b/result' }], signal: new AbortController().signal }) safe.release() }) diff --git a/apps/buddy/service/src/agent/extensions/__tests__/mutationToolExecution.spec.ts b/apps/buddy/service/src/agent/extensions/__tests__/mutationToolExecution.spec.ts index 16fe8e4f6..a7023caec 100644 --- a/apps/buddy/service/src/agent/extensions/__tests__/mutationToolExecution.spec.ts +++ b/apps/buddy/service/src/agent/extensions/__tests__/mutationToolExecution.spec.ts @@ -1,8 +1,10 @@ import type { ToolDefinition } from '@earendil-works/pi-coding-agent' +import type { BuddyToolClassification } from '../../../approvals/toolClassification' import type { BuddyExtensionRunContext } from '../BuddyExtensionRunContext' -import { mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises' +import { appendFile, mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import process from 'node:process' import { defineTool } from '@earendil-works/pi-coding-agent' import { Type } from 'typebox' import { afterEach, describe, expect, it } from 'vitest' @@ -10,6 +12,7 @@ import { runSupervisedShellProcess } from '../../../../../platform/process/super import { ChangeCaptureService } from '../../../changes/ChangeCaptureService' import { createChangeSetRepository } from '../../../changes/changeSetRepository' import { MutationCoordinator } from '../../../changes/MutationCoordinator' +import { MutationRecoveryService } from '../../../changes/MutationRecoveryService' import { createMutationRepository } from '../../../changes/mutationRepository' import { ToolAuthorizationService } from '../../../permissions/ToolAuthorizationService' import { ToolExecutionPermissions } from '../../../permissions/ToolExecutionPermissions' @@ -25,6 +28,92 @@ afterEach(async () => { }) describe('authorized mutation execution', () => { + it.skipIf(process.platform !== 'linux')('imports a lost native release receipt in the background without replaying the old command', async () => { + const fixture = await setup() + const recovery = new MutationRecoveryService({ mutations: fixture.mutations, receiptDirectory: join(fixture.root, 'receipts') }) + cleanups.push(async () => { + fixture.mutations.stopAdmission() + await recovery.dispose() + }) + await recovery.start() + const old = await fixture.mutations.acquire({ runId: 'run-1', invocationId: 'lost-release', lifetime: 'external', claims: [{ kind: 'tree', canonicalPath: fixture.root }], signal: new AbortController().signal }) + await expect(fixture.mutations.withInvocation(old, () => runSupervisedShellProcess({ + shell: '/bin/bash', + args: ['--noprofile', '--norc', '-c'], + command: 'printf "old\\n" >> effects.log', + cwd: fixture.root, + environment: { PATH: '/usr/bin:/bin' }, + }, { + receiptDirectory: join(fixture.root, 'receipts'), + onData() {}, + onSettlement(record) { + if (record.launchState === 'settled') + throw new Error('RELEASE_NOTIFICATION_LOST') + expect(fixture.mutations.recordOwnedProcessSettlement(record)).toBe(true) + }, + }))).rejects.toThrow('RELEASE_NOTIFICATION_LOST') + old.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + expect(fixture.mutations.getHazards()).toHaveLength(1) + const writer = fixture.actor('run-2', async (path, content) => { + await appendFile(join(fixture.root, 'effects.log'), 'new\n') + await writeFile(path, content) + }, 'write') + await writer.execute({ path: fixture.path, content: 'RECOVERED' }) + expect(await readFile(fixture.path, 'utf8')).toBe('RECOVERED') + expect(await readFile(join(fixture.root, 'effects.log'), 'utf8')).toBe('old\nnew\n') + await expect.poll(() => fixture.mutations.getHazards()).toEqual([]) + expect(fixture.mutationRepository.listUnreleased()).toEqual([]) + expect(fixture.mutations.getHazards()).toEqual([]) + }) + + it.each(['cancel', 'read_only'] as const)('does not start the waiting tool when %s occurs behind an active lease', async (change) => { + const fixture = await setup() + const controller = new AbortController() + const active = await fixture.mutations.acquire({ runId: 'run-1', invocationId: 'active', lifetime: 'runtime', claims: [{ kind: 'file', canonicalPath: fixture.path }], signal: controller.signal }) + const writer = fixture.actor('run-2', (path, content) => writeFile(path, content), 'write', { signal: controller.signal }) + const waiting = writer.execute({ path: fixture.path, content: 'MUST_NOT_WRITE' }) + const rejected = expect(waiting).rejects.toThrow() + try { + await expect.poll(() => fixture.mutations.getRunState('run-2').pending).toBe(1) + if (change === 'cancel') + controller.abort() + else + writer.authorization.updatePermissionSettings({ approvalPolicy: 'policy', executionProfile: 'read_only' }) + } + finally { active.markHazard('FILE_RESOURCE_SETTLEMENT_FAILED') } + await rejected + expect(await readFile(fixture.path, 'utf8')).toBe('A') + expect(fixture.repository.listCaptures('run-2')).toEqual([]) + expect(fixture.mutations.getRunState('run-2')).toEqual({ pending: 0, active: 0, hazards: [] }) + expect(fixture.mutations.getHazards()).toHaveLength(1) + }) + + it.each(['write', 'bash', 'mcp__fixture__write'])('admits a new %s while a prior external outcome remains unknown', async (toolName) => { + const fixture = await setup() + const old = await fixture.mutations.acquire({ runId: 'run-1', invocationId: 'old', lifetime: 'external', claims: [{ kind: 'host' }], signal: new AbortController().signal }) + fixture.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: 'unverified-external', settlement: 'unknown' }) + old.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + const writer = fixture.actor('run-2', (path, content) => writeFile(path, content), toolName) + await writer.execute({ path: fixture.path, content: 'NEW' }) + expect(await readFile(fixture.path, 'utf8')).toBe('NEW') + expect(fixture.mutations.getHazards()).toHaveLength(1) + expect(fixture.mutations.getOwnedProcesses('old')).toEqual([{ processId: 'unverified-external', settlement: 'unknown' }]) + expect(fixture.mutationRepository.listUnreleased().map(item => item.invocationId)).toEqual(['old']) + }) + + it('revalidates tool availability after asynchronous change capture before executing', async () => { + let available = true + const fixture = await setup({ afterCapture: () => { + available = false + } }) + const writer = fixture.actor('run-1', (path, content) => writeFile(path, content), 'mutate', { + validateBeforeExecution: async () => available ? null : { blocked: true, reason: 'TOOL_NO_LONGER_AVAILABLE' }, + }) + await expect(writer.execute({ path: fixture.path, content: 'MUST_NOT_WRITE' })).rejects.toThrow('TOOL_NO_LONGER_AVAILABLE') + expect(await readFile(fixture.path, 'utf8')).toBe('A') + expect(fixture.mutationRepository.listUnreleased()).toEqual([]) + }) + it('releases a supervised invocation rejected before process dispatch without poisoning later writes', async () => { const fixture = await setup() const shell = fixture.actor('run-1', async () => { @@ -139,7 +228,7 @@ describe('authorized mutation execution', () => { }) }) -async function setup() { +async function setup(options: { afterCapture?: () => void } = {}) { const root = await realpath(await mkdtemp(join(tmpdir(), 'buddy-tool-mutations-'))) cleanups.push(() => rm(root, { force: true, recursive: true })) const path = join(root, 'shared.txt') @@ -162,20 +251,39 @@ async function setup() { const mutations = new MutationCoordinator({ repository: mutationRepository }) cleanups.push(() => mutations.dispose()) const grants = [{ canonicalRoot: root, root, grantId: 'workspace', kind: 'workspace' as const }] - function actor(runId: string, mutate: (path: string, content: string) => Promise, toolName = 'mutate') { + function actor(runId: string, mutate: (path: string, content: string) => Promise, toolName = 'mutate', toolOptions: { signal?: AbortSignal, validateBeforeExecution?: BuddyToolClassification['validateBeforeExecution'] } = {}) { const cwd = join(root, runId) - const run: BuddyExtensionRunContext = { runId, signal: new AbortController().signal, flushProjectedEvents: async () => {}, onToolExecutionAuthorized: async () => {} } + const run: BuddyExtensionRunContext = { runId, signal: toolOptions.signal ?? new AbortController().signal, flushProjectedEvents: async () => {}, onToolExecutionAuthorized: async () => {} } const executionPermissions = new ToolExecutionPermissions() cleanups.push(() => executionPermissions.dispose()) const authorization = new ToolAuthorizationService({ approvalService: { request: async () => { throw new Error('UNEXPECTED_APPROVAL') } }, approvalAvailable: false, approvalPolicy: 'policy', executionProfile: 'workspace_write', cwd, owner: { kind: 'conversation', id: 'conversation' }, getGrants: () => grants, executionPermissions }) - const classify = (input: { path: string }) => ({ access: 'write' as const, paths: [{ path: input.path, mode: 'existing' as const }] }) + const classify = (input: { path: string }) => ({ access: 'write' as const, paths: [{ path: input.path, mode: 'existing' as const }], validateBeforeExecution: toolOptions.validateBeforeExecution }) const tool = defineTool({ name: toolName, label: 'Mutate', description: 'Write controlled files', executionMode: 'parallel', parameters: Type.Object({ path: Type.String(), content: Type.String() }), async execute(_id, input) { await mutate(input.path, input.content) return { content: [{ type: 'text', text: 'updated' }], details: {} } } }) - const wrapped = createMutationToolWrapper({ conversationId: 'conversation', cwd, authorization, changes, mutations, getRunContext: () => run, getGrants: () => grants, getWorkspaceGrants: () => grants, classify: async event => classify(event.input as { path: string }), workspaceMutationTools: [] })(tool) + const wrapped = createMutationToolWrapper({ + conversationId: 'conversation', + cwd, + authorization, + mutations, + getRunContext: () => run, + getGrants: () => grants, + getWorkspaceGrants: () => grants, + classify: async event => classify(event.input as { path: string }), + workspaceMutationTools: [], + changes: { + beginInvocation: async (input) => { + const handle = await changes.beginInvocation(input) + options.afterCapture?.() + return handle + }, + finishInvocation: (handle, input) => changes.finishInvocation(handle, input), + markPartial: runId => changes.markPartial(runId), + }, + })(tool) return { authorization, authorize: (input: { path: string, content: string }) => authorization.authorize({ input, toolCallId: 'same-provider-id', toolName: tool.name }, run, classify(input)), diff --git a/apps/buddy/service/src/agent/extensions/changeCaptureExtension.ts b/apps/buddy/service/src/agent/extensions/changeCaptureExtension.ts index 4bad37ac7..504681a36 100644 --- a/apps/buddy/service/src/agent/extensions/changeCaptureExtension.ts +++ b/apps/buddy/service/src/agent/extensions/changeCaptureExtension.ts @@ -75,6 +75,7 @@ export function createMutationToolWrapper(options: MutationToolOptions): BuddyTo let started = false let isError = true try { + executionSignal.throwIfAborted() if (!await stillCurrent(plan, permit, event, run, options)) { permit = null continue @@ -97,6 +98,10 @@ export function createMutationToolWrapper(options: MutationToolOptions): BuddyTo }) } catch { await markPartial(options, run.runId) } + executionSignal.throwIfAborted() + const finalValidation = await permit.classification.validateBeforeExecution?.() + if (finalValidation) + throw new Error(finalValidation.reason) if (!await stillCurrent(plan, permit, event, run, options)) { permit = null continue diff --git a/apps/buddy/service/src/agent/sessions/tree/BuddyConversationTree.ts b/apps/buddy/service/src/agent/sessions/tree/BuddyConversationTree.ts index 83a79fd5d..128e1558a 100644 --- a/apps/buddy/service/src/agent/sessions/tree/BuddyConversationTree.ts +++ b/apps/buddy/service/src/agent/sessions/tree/BuddyConversationTree.ts @@ -21,6 +21,7 @@ export interface BuddyConversationTreeOptions { repository: ConversationTreeRepository threads: Pick runs: Pick + isRunHistoryAvailable?: (runId: string) => boolean recovery: Pick onObserverError?: (error: unknown) => void } @@ -100,7 +101,7 @@ export class BuddyConversationTree { const throughSequence = publicMessageId ? options.repository.getMessagePublicSequence(conversationId, publicMessageId) - (position === 'before' ? 1 : 0) : source ? options.repository.getRunPublicBoundary(source.id, position) : options.repository.getPublicSequence(conversationId) const materials = options.repository.listPublications(conversationId, branchId, throughSequence) const eligible = eligiblePublicRuns(options, conversationId, branchId, targetThread, throughSequence) - const candidates = source?.threadId === targetThread + const candidates = source?.threadId === targetThread && options.isRunHistoryAvailable?.(source.id) !== false ? [{ run: source, position }, ...eligible.map(run => ({ run, position: 'after' as const }))] : eligible.map(run => ({ run, position: 'after' as const })) const checkpoint = journal @@ -213,7 +214,7 @@ export class BuddyConversationTreeCursor { const materials = this.#options.repository.listPublications(run.conversationId, run.branchId, throughSequence) const eligible = eligiblePublicRuns(this.#options, run.conversationId, run.branchId, run.threadId, throughSequence).filter(candidate => candidate.id !== run.id) const source = explicit ? this.#options.runs.findById(explicit.sourceRunId) : null - const candidates = source?.threadId === run.threadId + const candidates = source?.threadId === run.threadId && this.#options.isRunHistoryAvailable?.(source.id) !== false ? [{ run: source, position: explicit!.position }, ...eligible.map(run => ({ run, position: 'after' as const }))] : eligible.map(run => ({ run, position: 'after' as const })) for (const candidate of candidates) { @@ -267,7 +268,7 @@ function eligiblePublicRuns(options: BuddyConversationTreeOptions, conversationI const branches = new Set(options.repository.listPublicBranchIds(conversationId, branchId)) const sources = new Set(options.repository.listPublications(conversationId, branchId, throughSequence).map(material => material.sourceId)) return options.repository.listRuns(conversationId).filter((run) => { - if (run.threadId !== threadId || !branches.has(run.branchId) || run.status === 'queued' || run.status === 'running') + if (options.isRunHistoryAvailable?.(run.id) === false || run.threadId !== threadId || !branches.has(run.branchId) || run.status === 'queued' || run.status === 'running') return false if (options.repository.getRunPublicBoundary(run.id, 'after') > throughSequence) return false @@ -291,6 +292,8 @@ function isPublicCheckpoint(manager: SessionManager, entryId: string, materials: }) for (const entry of branch) { const checkpoint = readCheckpoint(entry) + if (checkpoint && options.isRunHistoryAvailable?.(checkpoint.runId) === false) + return false const completed = checkpoint?.position === 'after' ? options.runs.findById(checkpoint.runId) : null if (completed && options.repository.listPublications(completed.conversationId, completed.branchId).some(material => material.runId === completed.id && !sources.has(material.sourceId))) return false diff --git a/apps/buddy/service/src/agent/sessions/tree/__tests__/BuddyConversationTree.spec.ts b/apps/buddy/service/src/agent/sessions/tree/__tests__/BuddyConversationTree.spec.ts index b1447ae00..5db920ced 100644 --- a/apps/buddy/service/src/agent/sessions/tree/__tests__/BuddyConversationTree.spec.ts +++ b/apps/buddy/service/src/agent/sessions/tree/__tests__/BuddyConversationTree.spec.ts @@ -41,6 +41,38 @@ afterEach(async () => { }) describe('native conversation tree', () => { + it('skips quarantined native checkpoints and descendants without deleting private history', async () => { + const quarantined = new Set() + const f = await createFixture(undefined, runId => !quarantined.has(runId)) + const first = f.run('damaged', 'b0', 'q1') + const opened = await f.open(first) + await opened.cursor.begin(opened.session.session, first.id) + opened.cursor.manager.appendMessage(user('q1', 'PUBLIC_INPUT')) + opened.cursor.manager.appendMessage(assistant([{ type: 'text', text: 'UNVERIFIED_PRIVATE_NATIVE_CONTENT' }])) + f.publish(first, 'public-answer', 'PRESERVED_PUBLIC_RECORD') + opened.cursor.finish() + f.complete(first) + await opened.session.shutdown('quit') + const descendant = f.run('descendant', 'b0', 'q2') + const inherited = await f.open(descendant) + await inherited.cursor.begin(inherited.session.session, descendant.id) + inherited.cursor.finish() + f.complete(descendant) + await inherited.session.shutdown('quit') + const originalBytes = await readFile(inherited.session.piSessionFile) + quarantined.add(first.id) + const preview = await f.tree.preview('conversation', 'b0', f.root, first.threadId) + expect(JSON.stringify(preview.buildSessionContext().messages)).not.toContain('UNVERIFIED_PRIVATE_NATIVE_CONTENT') + const next = f.run('next', 'b0', 'q3') + f.source(next.id, first.id, 'after') + const fresh = await f.open(next) + await fresh.cursor.begin(fresh.session.session, next.id) + expect(JSON.stringify(fresh.cursor.manager.buildSessionContext().messages)).not.toContain('UNVERIFIED_PRIVATE_NATIVE_CONTENT') + expect(JSON.stringify(fresh.cursor.manager.buildSessionContext().messages)).toContain('PRESERVED_PUBLIC_RECORD') + expect((await readFile(inherited.session.piSessionFile)).subarray(0, originalBytes.length)).toEqual(originalBytes) + await fresh.session.shutdown('quit') + }) + it('includes new publications in later previews without changing earlier snapshots', async () => { const f = await createFixture() const first = f.run('preview-first', 'b0', 'q1') @@ -611,7 +643,7 @@ describe('native conversation tree', () => { }) }) -async function createFixture(recovery?: BuddySessionRecoveryService['create']) { +async function createFixture(recovery?: BuddySessionRecoveryService['create'], isRunHistoryAvailable?: (runId: string) => boolean) { const root = await mkdtemp(join(tmpdir(), 'buddy-native-tree-')) const database = openBuddyDatabase({ databasePath: ':memory:' }) cleanups.push(async () => { @@ -643,6 +675,7 @@ async function createFixture(recovery?: BuddySessionRecoveryService['create']) { conversationsDirectory: join(root, 'conversations'), repository, threads, + isRunHistoryAvailable, recovery: { create: recovery ?? (async () => { throw new Error('Healthy native context must not be reconstructed') diff --git a/apps/buddy/service/src/changes/MutationCoordinator.ts b/apps/buddy/service/src/changes/MutationCoordinator.ts index b1210490c..e26fabe36 100644 --- a/apps/buddy/service/src/changes/MutationCoordinator.ts +++ b/apps/buddy/service/src/changes/MutationCoordinator.ts @@ -51,7 +51,7 @@ export class MutationCoordinatorError extends Error { readonly code: 'MUTATION_RESOURCE_UNSAFE' | 'MUTATION_COORDINATOR_STOPPED' | 'MUTATION_INVOCATION_CONFLICT' constructor(code: MutationCoordinatorError['code']) { - super(code === 'MUTATION_RESOURCE_UNSAFE' ? `${code}: An earlier operation has an unverified outcome. Conflicting writes remain blocked. Review the recovery options in the task.` : code) + super(code) this.code = code this.name = 'MutationCoordinatorError' } @@ -71,6 +71,8 @@ export class MutationCoordinator { readonly #resolutions = new Map>() readonly #changes = new Emitter<{ runId: string }>(() => console.error('MUTATION_OBSERVER_FAILED')) readonly onDidChange = this.#changes.event + readonly #recoveryNeeded = new Emitter<{ invocationId: string }>(() => console.error('MUTATION_RECOVERY_OBSERVER_FAILED')) + readonly onDidNeedRecovery = this.#recoveryNeeded.event #lastWriter: { runId: string, segmentId: string } | undefined #sequence = 0 #disposed = false @@ -97,7 +99,7 @@ export class MutationCoordinator { if (this.#disposed) return Promise.reject(new MutationCoordinatorError('MUTATION_COORDINATOR_STOPPED')) input.signal.throwIfAborted() - if (this.#active.has(input.invocationId) || this.#hazards.has(input.invocationId) || this.#pending.some(pending => pending.invocationId === input.invocationId)) + if (this.#active.has(input.invocationId) || this.#hazards.has(input.invocationId) || this.#repository?.isReleased(input.invocationId) || this.#pending.some(pending => pending.invocationId === input.invocationId)) return Promise.reject(new MutationCoordinatorError('MUTATION_INVOCATION_CONFLICT')) const claims = normalizeClaims(input.claims) const lease = await new Promise((resolve, reject) => { @@ -140,13 +142,19 @@ export class MutationCoordinator { async initialize(): Promise { await this.#resources?.reconcile([...this.#hazards.values()], id => this.#repository?.isReleased(id) === true) - for (const hazard of this.#hazards.values()) { - const decision = this.#evidence?.decision(hazard.invocationId) - if (decision?.outcome === 'requested' && (decision.kind === 'waived' || decision.kind === 'quiesced')) - await this.#resolveProtection(hazard.invocationId, decision.kind).catch(() => {}) - else if (this.#canConfirmReleased(hazard.invocationId)) - await this.confirmReleased(hazard.invocationId).catch(() => {}) - } + for (const hazard of this.#hazards.values()) + void this.resumeRecovery(hazard.invocationId).catch(() => {}) + } + + async resumeRecovery(invocationId: string): Promise { + if (this.#disposed) + throw new MutationCoordinatorError('MUTATION_COORDINATOR_STOPPED') + const decision = this.#evidence?.decision(invocationId) + if (decision?.outcome === 'requested' && decision.kind === 'waived') + await this.#resolveProtection(invocationId, 'waived') + else if (this.#canConfirmReleased(invocationId)) + await this.confirmReleased(invocationId) + return !this.#hazards.has(invocationId) } getHazards(): readonly MutationHazard[] { @@ -200,12 +208,15 @@ export class MutationCoordinator { #recordProcess(input: { invocationId: string, processId: string, settlement: OwnedProcessSettlement } & MutationExecutionUpdate): boolean { const { invocationId } = input const processes = this.#processes.get(invocationId) ?? new Map() - const alreadyReleased = processes.get(input.processId) === 'released' + const previous = processes.get(input.processId) + const alreadyReleased = previous === 'released' if (alreadyReleased && (!input.launchState || input.launchState === 'authorized')) return input.launchState !== 'authorized' this.#repository?.recordProcess(invocationId, input.processId, input.settlement, input) processes.set(input.processId, alreadyReleased ? 'released' : input.settlement) this.#processes.set(invocationId, processes) + if (previous !== processes.get(input.processId)) + this.#requestRecovery(invocationId) if (this.#hazards.has(invocationId) && [...processes.values()].every(value => value === 'released')) void this.confirmReleased(invocationId).catch(() => {}) return true @@ -218,20 +229,31 @@ export class MutationCoordinator { return [...processes.values()].every(value => value === 'released') ? 'released' : 'unknown' } - async waitForRun(runId: string): Promise { + async waitForRun(runId: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const current = this.getRunState(runId) if (!current.pending && !current.active) return current - return new Promise((resolve) => { - const subscription = this.onDidChange((event) => { + return new Promise((resolve, reject) => { + let subscription: { dispose: () => void } | undefined + const abort = () => { + subscription?.dispose() + signal?.removeEventListener('abort', abort) + reject(signal?.reason) + } + subscription = this.onDidChange((event) => { if (event.runId !== runId) return const state = this.getRunState(runId) if (!state.pending && !state.active) { - subscription.dispose() + subscription?.dispose() + signal?.removeEventListener('abort', abort) resolve(state) } }) + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) + abort() }) } @@ -241,6 +263,7 @@ export class MutationCoordinator { this.#hazards.set(hazard.invocationId, copyEventSnapshot({ ...hazard, claims: normalizeClaims(hazard.claims) })) this.#publish(hazard.runId) this.#drain() + this.#requestRecovery(hazard.invocationId) } confirmReleased(invocationId: string): Promise { @@ -250,7 +273,10 @@ export class MutationCoordinator { waiveHazard(invocationId: string): Promise { if (!this.#evidence) return Promise.reject(new Error('MUTATION_RECOVERY_AUDIT_UNAVAILABLE')) - return this.#resolveProtection(invocationId, 'waived') + return this.#resolveProtection(invocationId, 'waived').catch((error) => { + this.#requestRecovery(invocationId) + throw error + }) } #canConfirmReleased(invocationId: string): boolean { @@ -284,7 +310,7 @@ export class MutationCoordinator { this.#hazards.delete(invocationId) this.#processes.delete(invocationId) this.#publish(hazard.runId) - for (const runId of this.#evidence?.blockedRuns(invocationId) ?? []) + for (const runId of this.#evidence?.relatedRuns(invocationId) ?? []) this.#publish(runId) this.#drain() } @@ -293,32 +319,23 @@ export class MutationCoordinator { return resolution } - async dispose(): Promise { + stopAdmission(): void { this.#disposed = true for (const pending of this.#pending.splice(0)) { pending.unsubscribe() pending.reject(new MutationCoordinatorError('MUTATION_COORDINATOR_STOPPED')) this.#publish(pending.runId) } + } + + async dispose(): Promise { + this.stopAdmission() await this.waitForSettlements() } #drain(): void { for (let index = 0; index < this.#pending.length;) { const pending = this.#pending[index]! - const blockers = [...this.#hazards.values()].filter(hazard => claimsOverlap(hazard.claims, pending.claims)) - if (blockers.length) { - this.#pending.splice(index, 1) - pending.unsubscribe() - try { - for (const hazard of blockers) - this.#evidence?.recordBlocker(hazard.invocationId, pending.runId) - pending.reject(new MutationCoordinatorError('MUTATION_RESOURCE_UNSAFE')) - } - catch (error) { pending.reject(error) } - this.#publish(pending.runId) - continue - } if ([...this.#active.values()].some(lease => claimsOverlap(lease.claims, pending.claims)) || this.#pending.slice(0, index).some(earlier => claimsOverlap(earlier.claims, pending.claims))) { index++ @@ -347,11 +364,24 @@ export class MutationCoordinator { continue } this.#active.set(lease.invocationId, lease) + this.#recordOverlaps(lease) pending.resolve(lease) this.#publish(lease.runId) } } + #recordOverlaps(lease: MutationLease): void { + if (!this.#evidence) + return + try { + for (const hazard of this.#hazards.values()) { + if (hazard.runId !== lease.runId && claimsOverlap(hazard.claims, lease.claims)) + this.#evidence.recordOverlap(hazard.invocationId, lease.runId) + } + } + catch { console.error('MUTATION_OVERLAP_RECORD_FAILED') } + } + #release(lease: MutationLease, hazard?: string): void { if (this.#active.get(lease.invocationId) !== lease || this.#closing.has(lease.invocationId)) return @@ -380,6 +410,8 @@ export class MutationCoordinator { } this.#publish(lease.runId) this.#drain() + if (reason) + this.#requestRecovery(lease.invocationId) } if (!this.#resources) { finish(hazard) @@ -405,6 +437,11 @@ export class MutationCoordinator { return pending } + #requestRecovery(invocationId: string): void { + if (!this.#disposed && this.#hazards.has(invocationId)) + this.#recoveryNeeded.fire(Object.freeze({ invocationId })) + } + #publish(runId: string): void { this.#changes.fire(Object.freeze({ runId })) } diff --git a/apps/buddy/service/src/changes/MutationRecoveryService.ts b/apps/buddy/service/src/changes/MutationRecoveryService.ts index 3aceea6df..c11e01b23 100644 --- a/apps/buddy/service/src/changes/MutationRecoveryService.ts +++ b/apps/buddy/service/src/changes/MutationRecoveryService.ts @@ -19,6 +19,24 @@ const identitySchema = z.object({ type ProcessIdentity = z.infer +interface RecoveryWork { + invocationId: string + priority: boolean + run: () => Promise + promise: Promise + resolve: () => void + reject: (error: unknown) => void +} + +interface RecoveryRetry { + attempt: number + dirty: boolean + timer?: ReturnType +} + +const retryDelays = [1_000, 5_000, 15_000] +const verificationTimeout = 5_000 + export interface MutationRecoveryOptions { mutations: MutationCoordinator verify?: typeof reverifyOwnedProcess @@ -36,31 +54,158 @@ export interface MutationRecoveryOptions { export class MutationRecoveryService { readonly #options: MutationRecoveryOptions readonly #record: ApplicationDiagnosticReporter - #tail: Promise = Promise.resolve() + readonly #stopped = new AbortController() + readonly #pending: RecoveryWork[] = [] + readonly #running = new Set() + readonly #checks = new Map() + readonly #retries = new Map() + readonly #unregister: (() => void)[] = [] + #startPromise: Promise | undefined + #disposePromise: Promise | undefined constructor(options: MutationRecoveryOptions) { this.#options = options this.#record = safeDiagnosticReporter(options.record) } + start(): Promise { + if (this.#stopped.signal.aborted) + return Promise.resolve() + this.#startPromise ??= this.#start() + return this.#startPromise + } + + async #start(): Promise { + const mutations = this.#options.mutations + const subscription = mutations.onDidNeedRecovery(({ invocationId }) => { + if (this.#stopped.signal.aborted) + return + const previous = this.#retries.get(invocationId) + clearTimeout(previous?.timer) + this.#retries.set(invocationId, { attempt: 0, dirty: this.#checks.has(invocationId) }) + if (!this.#checks.has(invocationId)) + void this.#check([invocationId], false) + }) + this.#unregister.push(() => subscription.dispose()) + for (const hazard of mutations.getHazards()) + this.#retries.set(hazard.invocationId, { attempt: 0, dirty: false }) + void this.#check(mutations.getHazards().map(hazard => hazard.invocationId), false) + } + + dispose(): Promise { + this.#disposePromise ??= this.#dispose() + return this.#disposePromise + } + + async #dispose(): Promise { + this.#stopped.abort() + for (const unregister of this.#unregister.splice(0).reverse()) + unregister() + for (const retry of this.#retries.values()) + clearTimeout(retry.timer) + this.#retries.clear() + for (const work of this.#pending.splice(0)) { + this.#checks.delete(work.invocationId) + work.reject(this.#stopped.signal.reason) + } + await Promise.allSettled([...this.#running].map(work => work.promise)) + } + recheck(input: { signal?: AbortSignal, invocationIds?: readonly string[] } = {}): Promise { - const signal = input.signal ?? AbortSignal.timeout(15_000) - return this.#serialize(async () => { - for (const hazard of this.#options.mutations.getHazards()) { - if (signal.aborted) - break - if (input.invocationIds && !input.invocationIds.includes(hazard.invocationId)) - continue - try { - await this.#recheck(hazard, signal) - } - catch { - if (signal.aborted) - break - this.#observe(hazard, 'unknown', 'RECOVERY_VERIFICATION_FAILED') - } + return this.#check(input.invocationIds ?? this.#options.mutations.getHazards().map(hazard => hazard.invocationId), true, input.signal) + } + + async #check(invocationIds: readonly string[], priority: boolean, callerSignal?: AbortSignal): Promise { + if (this.#stopped.signal.aborted || callerSignal?.aborted) + return + const signal = callerSignal ? AbortSignal.any([callerSignal, this.#stopped.signal]) : this.#stopped.signal + const checks = [...new Set(invocationIds)].map((invocationId) => { + const existing = this.#checks.get(invocationId) + if (existing) { + existing.priority ||= priority + return existing.promise } + clearTimeout(this.#retries.get(invocationId)?.timer) + const work = this.#enqueue(invocationId, () => this.#verify(invocationId), priority) + this.#checks.set(invocationId, work) + return work.promise }) + this.#drain() + await waitForVerification(Promise.allSettled(checks).then(() => {}), signal).catch(() => {}) + } + + async #verify(invocationId: string): Promise { + this.#stopped.signal.throwIfAborted() + const hazard = this.#options.mutations.getHazards().find(item => item.invocationId === invocationId) + if (!hazard) + return + const timeout = new AbortController() + const timer = setTimeout(() => timeout.abort(), verificationTimeout) + timer.unref() + const signal = AbortSignal.any([this.#stopped.signal, timeout.signal]) + try { + await waitForVerification(this.#recheck(hazard, signal), signal) + } + catch { + if (!signal.aborted) + this.#observe(hazard, 'unknown', 'RECOVERY_VERIFICATION_FAILED') + } + finally { clearTimeout(timer) } + } + + #enqueue(invocationId: string, run: () => Promise, priority: boolean): RecoveryWork { + const deferred = Promise.withResolvers() + const work = { invocationId, run, priority, ...deferred } + this.#pending.push(work) + return work + } + + #drain(): void { + while (!this.#stopped.signal.aborted && this.#running.size < 2) { + const eligible = (work: RecoveryWork, index: number) => + ![...this.#running].some(active => active.invocationId === work.invocationId) + && !this.#pending.slice(0, index).some(earlier => earlier.invocationId === work.invocationId) + let index = this.#pending.findIndex((work, index) => work.priority && eligible(work, index)) + if (index < 0) + index = this.#pending.findIndex(eligible) + if (index < 0) + return + const work = this.#pending.splice(index, 1)[0]! + this.#running.add(work) + void Promise.resolve().then(work.run).then(work.resolve, work.reject).finally(() => { + this.#running.delete(work) + if (this.#checks.get(work.invocationId) === work) { + this.#checks.delete(work.invocationId) + this.#scheduleRetry(work.invocationId) + } + this.#drain() + }) + } + } + + #scheduleRetry(invocationId: string): void { + const retry = this.#retries.get(invocationId) + if (!retry || this.#stopped.signal.aborted) + return + if (!this.#hasHazard(invocationId)) { + clearTimeout(retry.timer) + this.#retries.delete(invocationId) + return + } + if (retry.dirty) { + retry.dirty = false + void this.#check([invocationId], false) + return + } + const delay = retryDelays[retry.attempt++] + if (delay === undefined) { + this.#retries.delete(invocationId) + return + } + retry.timer = setTimeout(() => { + void this.#check([invocationId], false) + }, delay) + retry.timer.unref() } list(input: ChangeRecoveryListRequest): LocalChangeRecoveryItem[] { @@ -70,7 +215,7 @@ export class MutationRecoveryService { throw new BuddyServiceError('VALIDATION_FAILED') const runIds = new Set(context.runs.listIdsForConversation(input.conversationId)) return this.#options.mutations.getHazards().flatMap((hazard) => { - if (!runIds.has(hazard.runId) && !this.#options.evidence?.blockedRuns(hazard.invocationId).some(runId => runIds.has(runId))) + if (!runIds.has(hazard.runId) && !this.#options.evidence?.relatedRuns(hazard.invocationId).some(runId => runIds.has(runId))) return [] const run = context.runs.findById(hazard.runId) const owner = run ? context.conversations.findById(run.conversationId) : null @@ -97,40 +242,47 @@ export class MutationRecoveryService { if (input.invocationIds.some(id => !visible.has(id))) throw new BuddyServiceError('VALIDATION_FAILED') await this.recheck({ signal, invocationIds: input.invocationIds }) + this.#stopped.signal.throwIfAborted() return this.list(input) } - resolve(input: ChangeRecoveryResolveRequest, signal?: AbortSignal): Promise { - return this.#serialize(async () => { - signal?.throwIfAborted() - const visible = new Map(this.list(input).map(item => [item.invocationId, item])) - if (input.items.some((item) => { - const current = visible.get(item.invocationId) - return !current || current.status !== 'unknown' || current.revision !== item.revision - })) { - throw new BuddyServiceError('VALIDATION_FAILED') - } - if (!this.#options.evidence) - throw new BuddyServiceError('RUNTIME_UNAVAILABLE') - for (const item of input.items) { - signal?.throwIfAborted() + async resolve(input: ChangeRecoveryResolveRequest, callerSignal?: AbortSignal): Promise { + const signal = callerSignal ? AbortSignal.any([callerSignal, this.#stopped.signal]) : this.#stopped.signal + signal.throwIfAborted() + const visible = new Map(this.list(input).map(item => [item.invocationId, item])) + if (input.items.some((item) => { + const current = visible.get(item.invocationId) + return !current || current.status !== 'unknown' || current.revision !== item.revision + })) { + throw new BuddyServiceError('VALIDATION_FAILED') + } + if (!this.#options.evidence) + throw new BuddyServiceError('RUNTIME_UNAVAILABLE') + for (const item of input.items) { + signal.throwIfAborted() + const work = this.#enqueue(item.invocationId, async () => { + signal.throwIfAborted() const current = this.list(input).find(candidate => candidate.invocationId === item.invocationId) if (!current || current.status !== 'unknown' || current.revision !== item.revision) throw new BuddyServiceError('VALIDATION_FAILED') await this.#options.mutations.waiveHazard(item.invocationId) - this.#options.onChange?.(visible.get(item.invocationId)!.runId) - } - return this.list(input) - }) + signal.throwIfAborted() + this.#options.onChange?.(current.runId) + }, true) + this.#drain() + await waitForVerification(work.promise, signal) + } + signal.throwIfAborted() + return this.list(input) } - #serialize(operation: () => Promise): Promise { - const result = this.#tail.catch(() => {}).then(operation) - this.#tail = result - return result + #hasHazard(invocationId: string): boolean { + return this.#options.mutations.getHazards().some(hazard => hazard.invocationId === invocationId) } #observe(hazard: MutationHazard, outcome: 'active' | 'unknown', reason: string): void { + if (this.#stopped.signal.aborted || !this.#hasHazard(hazard.invocationId)) + return this.#options.evidence?.append({ invocationId: hazard.invocationId, kind: 'observed', outcome, reason, actor: 'system' }) this.#record({ event: outcome === 'active' ? 'mutation.recovery.active' : 'mutation.recovery.protected', level: 'warn', runId: hazard.runId, operationId: hazard.invocationId, errorCode: reason }) this.#options.onChange?.(hazard.runId) @@ -138,7 +290,16 @@ export class MutationRecoveryService { async #recheck(hazard: MutationHazard, signal: AbortSignal): Promise { const mutations = this.#options.mutations - if (!mutations.getHazards().some(item => item.invocationId === hazard.invocationId)) + signal.throwIfAborted() + if (!this.#hasHazard(hazard.invocationId)) + return + const resumed = await waitForVerification(mutations.resumeRecovery(hazard.invocationId), signal) + signal.throwIfAborted() + if (resumed) { + this.#options.onChange?.(hazard.runId) + return + } + if (!this.#hasHazard(hazard.invocationId)) return const records = mutations.getOwnedProcesses(hazard.invocationId).map(record => ({ ...record, identity: parseIdentity(record.processId) })) const checkpoint = records.find(record => record.identity?.kind === 'recovery-boot') @@ -154,12 +315,17 @@ export class MutationRecoveryService { if (signal.aborted) return const outcome = await verify(record.processId, { signal, checkpoint: checkpoint?.processId, receiptDirectory: this.#options.receiptDirectory }) + signal.throwIfAborted() + if (!this.#hasHazard(hazard.invocationId)) + return if (outcome.status === 'active' || (!active && outcome.status === 'unknown')) reason = outcome.reason active ||= outcome.status === 'active' this.#options.evidence?.append({ invocationId: hazard.invocationId, kind: outcome.status === 'released' ? 'quiesced' : 'observed', outcome: outcome.status === 'released' ? 'applied' : outcome.status, reason: outcome.reason, actor: 'system', processId: record.processId }) - if (outcome.status === 'released') + if (outcome.status === 'released') { released.add(record.processId) + mutations.recordOwnedProcessSettlement({ invocationId: hazard.invocationId, processId: record.processId, settlement: 'released' }) + } } const sandboxDispatches = records.filter(record => record.identity?.kind === 'sandbox-dispatch') const dispatchRecords = [...records.filter(record => record.identity?.kind === 'native-shell-dispatch'), ...sandboxDispatches] @@ -190,6 +356,7 @@ export class MutationRecoveryService { try { signal.throwIfAborted() await waitForVerification(mutations.confirmReleased(hazard.invocationId), signal) + signal.throwIfAborted() this.#record({ event: 'mutation.recovery.released', level: 'info', runId: hazard.runId, operationId: hazard.invocationId }) this.#options.onChange?.(hazard.runId) } @@ -207,6 +374,9 @@ export class MutationRecoveryService { && withoutCheckpoint.every(record => record.identity && ['native-shell', 'sandbox-job', 'native-shell-dispatch', 'sandbox-dispatch'].includes(record.identity.kind)) if (!checkpoint && canCheckpoint && !signal.aborted) { const baseline = await (this.#options.checkpoint ?? captureProcessRecoveryCheckpoint)({ signal }) + signal.throwIfAborted() + if (!this.#hasHazard(hazard.invocationId)) + return if (baseline) { mutations.recordRecoveryCheckpoint(hazard.invocationId, baseline) this.#record({ event: 'mutation.recovery.checkpoint_saved', level: 'info', runId: hazard.runId, operationId: hazard.invocationId }) @@ -216,7 +386,7 @@ export class MutationRecoveryService { } } -async function waitForVerification(operation: Promise, signal: AbortSignal): Promise { +async function waitForVerification(operation: Promise, signal: AbortSignal): Promise { let abort: () => void = () => {} const cancelled = new Promise((_resolve, reject) => { abort = () => reject(signal.reason) @@ -225,7 +395,7 @@ async function waitForVerification(operation: Promise, signal: AbortSignal abort() }) try { - await Promise.race([operation, cancelled]) + return await Promise.race([operation, cancelled]) } finally { signal.removeEventListener('abort', abort) } } diff --git a/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts b/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts index ad6fd24cf..6a7a9cb8b 100644 --- a/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/MutationCoordinator.spec.ts @@ -1,3 +1,6 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { openBuddyDatabase } from '../../storage/database' import { MutationCoordinator } from '../MutationCoordinator' @@ -13,6 +16,105 @@ function setup() { } describe('mutation ownership', () => { + it('keeps historical shell and MCP facts across database reopen while admitting only new identities', async () => { + const root = await mkdtemp(join(tmpdir(), 'mutation-reopen-')) + const databasePath = join(root, 'buddy.sqlite3') + const input = { lifetime: 'external' as const, claims: [{ kind: 'host' as const }], signal: new AbortController().signal } + let database = openBuddyDatabase({ databasePath }) + try { + const first = new MutationCoordinator({ repository: createMutationRepository(database) }) + for (const id of ['shell', 'mcp']) { + const lease = await first.acquire({ ...input, runId: id, invocationId: id }) + first.recordOwnedProcessSettlement({ invocationId: id, processId: id, settlement: id === 'shell' ? 'pending' : 'unknown' }) + first.recordOwnedProcessSettlement({ invocationId: id, processId: `${id}-released`, settlement: 'released' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + } + await first.dispose() + database.close() + database = openBuddyDatabase({ databasePath }) + const repository = createMutationRepository(database) + const restored = new MutationCoordinator({ repository }) + try { + const before = repository.listUnreleased() + const next = await restored.acquire({ ...input, runId: 'next', invocationId: 'next' }) + next.release() + expect(repository.listUnreleased()).toEqual(before) + expect(restored.getOwnedProcesses('shell')).toEqual([ + { processId: 'shell', settlement: 'pending' }, + { processId: 'shell-released', settlement: 'released' }, + ]) + expect(restored.getOwnedProcesses('mcp')).toEqual([ + { processId: 'mcp', settlement: 'unknown' }, + { processId: 'mcp-released', settlement: 'released' }, + ]) + for (const invocationId of ['shell', 'mcp', 'next']) + await expect(restored.acquire({ ...input, runId: 'replay', invocationId })).rejects.toMatchObject({ code: 'MUTATION_INVOCATION_CONFLICT' }) + } + finally { await restored.dispose() } + } + finally { + database.close() + await rm(root, { recursive: true, force: true }) + } + }) + + it('keeps overlapping pending claims in FIFO order while an active lease settles', async () => { + const mutations = setup() + const input = { lifetime: 'runtime' as const, claims: [{ kind: 'host' as const }], signal: new AbortController().signal } + const first = await mutations.acquire({ ...input, runId: 'first', invocationId: 'first' }) + const admitted: string[] = [] + const second = mutations.acquire({ ...input, runId: 'second', invocationId: 'second' }).then((lease) => { + admitted.push('second') + return lease + }) + const third = mutations.acquire({ ...input, runId: 'third', invocationId: 'third' }).then((lease) => { + admitted.push('third') + return lease + }) + expect(mutations.getRunState('second').pending).toBe(1) + expect(mutations.getRunState('third').pending).toBe(1) + first.markHazard('UNCONFIRMED') + const secondLease = await second + expect(admitted).toEqual(['second']) + expect(mutations.getRunState('third').pending).toBe(1) + secondLease.release() + const thirdLease = await third + expect(admitted).toEqual(['second', 'third']) + thirdLease.release() + expect(mutations.getHazards()).toHaveLength(1) + }) + + it('initializes and admits new work while a historical resource reconciliation is still settling', async () => { + const entered = Promise.withResolvers() + const released = Promise.withResolvers() + const mutations = new MutationCoordinator({ resources: { + acquire: async () => {}, + reconcile: async () => {}, + release: async (id) => { + if (id === 'old') { + entered.resolve() + await released.promise + } + }, + } }) + coordinators.push(mutations) + const input = { lifetime: 'runtime' as const, claims: [{ kind: 'host' as const }], signal: new AbortController().signal } + mutations.restoreHazard({ runId: 'old', invocationId: 'old', claims: input.claims, reason: 'FILE_RESOURCE_SETTLEMENT_FAILED' }) + try { + await mutations.initialize() + await entered.promise + const next = await mutations.acquire({ ...input, runId: 'next', invocationId: 'next' }) + expect(mutations.getHazards()).toHaveLength(1) + await expect(mutations.acquire({ ...input, runId: 'old', invocationId: 'old' })).rejects.toMatchObject({ code: 'MUTATION_INVOCATION_CONFLICT' }) + next.release() + } + finally { + released.resolve() + await mutations.waitForSettlements() + } + expect(mutations.getHazards()).toEqual([]) + }) + it('keeps conflicting writers queued during native settlement but fences late process dispatch', async () => { const released = Promise.withResolvers() const mutations = new MutationCoordinator({ resources: { acquire: async () => {}, reconcile: async () => {}, release: async () => released.promise } }) @@ -74,7 +176,9 @@ describe('mutation ownership', () => { const recovered = new MutationCoordinator({ repository }) coordinators.push(recovered) expect(recovered.getRunState(input.runId)).toMatchObject({ active: 0, pending: 0, hazards: [{ invocationId: input.invocationId, reason: 'MUTATION_INTERRUPTED' }] }) - await expect(recovered.acquire({ ...input, invocationId: 'overlap' })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const overlap = await recovered.acquire({ ...input, invocationId: 'overlap' }) + overlap.release() + await expect(recovered.acquire(input)).rejects.toMatchObject({ code: 'MUTATION_INVOCATION_CONFLICT' }) recovered.recordOwnedProcessSettlement({ invocationId: input.invocationId, processId: 'parent', settlement: 'unknown' }) expect(recovered.getRunState(input.runId).hazards).toHaveLength(1) recovered.recordOwnedProcessSettlement({ invocationId: input.invocationId, processId: 'child', settlement: 'released' }) @@ -113,7 +217,7 @@ describe('mutation ownership', () => { finally { database.close() } }) - it('keeps a supervised lease unsafe when the process handshake was not durably recorded', async () => { + it('retains an unknown supervised outcome without fencing a new invocation', async () => { const database = openBuddyDatabase({ databasePath: ':memory:' }) try { const repository = createMutationRepository(database) @@ -124,7 +228,9 @@ describe('mutation ownership', () => { first.dispose() const recovered = new MutationCoordinator({ repository }) coordinators.push(recovered) - await expect(recovered.acquire({ ...input, invocationId: 'next' })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const next = await recovered.acquire({ ...input, invocationId: 'next' }) + next.release() + await expect(recovered.acquire(input)).rejects.toMatchObject({ code: 'MUTATION_INVOCATION_CONFLICT' }) expect(repository.listUnreleased()).toHaveLength(1) } finally { database.close() } @@ -148,22 +254,24 @@ describe('mutation ownership', () => { resumed.release() }) - it('settles an unsafe execution without releasing its resource and restores that hazard independently', async () => { + it('retains an unknown outcome without turning it into a persistent admission fence', async () => { const mutations = setup() const input = { lifetime: 'external' as const, claims: [{ kind: 'host' as const }], signal: new AbortController().signal } const lease = await mutations.acquire({ ...input, runId: 'failed', invocationId: 'host-1' }) const settled = mutations.waitForRun('failed') const waiting = mutations.acquire({ ...input, runId: 'next', invocationId: 'next-1' }) - const blocked = expect(waiting).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + expect(mutations.getRunState('next')).toMatchObject({ pending: 1, active: 0 }) lease.markHazard('PROCESS_CLEANUP_UNCONFIRMED') - await blocked + const next = await waiting + next.release() const state = await settled expect(state).toMatchObject({ pending: 0, active: 0, hazards: [{ invocationId: 'host-1', runId: 'failed' }] }) lease.release() expect(mutations.getRunState('failed').hazards).toHaveLength(1) const restored = setup() restored.restoreHazard(state.hazards[0]!) - await expect(restored.acquire({ ...input, runId: 'other', invocationId: 'other-1' })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const other = await restored.acquire({ ...input, runId: 'other', invocationId: 'other-1' }) + other.release() await expect(restored.confirmReleased('host-1')).rejects.toThrow('PROCESS_RELEASE_EVIDENCE_MISSING') restored.recordOwnedProcessSettlement({ invocationId: 'host-1', processId: 'verified-original', settlement: 'released' }) await restored.confirmReleased('host-1') diff --git a/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts b/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts index f74da77e1..50e6983ee 100644 --- a/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/MutationRecoveryService.spec.ts @@ -56,11 +56,13 @@ describe('mutation recovery evidence', () => { writer.release() }) - it('keeps live descendants, malformed identities, and unmatched dispatches protected', async () => { + it('retains observed descendants and malformed identities as warnings without blocking new work', async () => { const f = await fixture([dispatch('other-request'), native('nonce', 'request'), 'invalid']) const recovery = new MutationRecoveryService({ mutations: f.mutations, verify: async () => ({ status: 'active', reason: 'job_active' }), checkpoint: async () => null }) await recovery.recheck() - await expect(f.mutations.acquire({ runId: 'next', invocationId: 'next', lifetime: 'runtime', claims: [{ kind: 'host' }], signal })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const next = await f.mutations.acquire({ runId: 'next', invocationId: 'next', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + next.release() + expect(f.mutations.getOwnedProcesses('invocation').every(item => item.settlement === 'unknown')).toBe(true) expect(f.repository.listUnreleased()).toHaveLength(1) }) @@ -68,6 +70,8 @@ describe('mutation recovery evidence', () => { const f = await fixture([]) const recovery = new MutationRecoveryService({ mutations: f.mutations, verify: async () => ({ status: 'released', reason: 'boot_changed' }), checkpoint: async () => checkpoint }) await recovery.recheck() + const next = await f.mutations.acquire({ runId: 'next', invocationId: 'next', lifetime: 'external', claims: [{ kind: 'host' }], signal }) + next.release() expect(f.mutations.getOwnedProcesses('invocation')).toEqual([]) expect(f.repository.listUnreleased()).toHaveLength(1) }) diff --git a/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts b/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts index d4e6f6f90..f276d63e5 100644 --- a/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts +++ b/apps/buddy/service/src/changes/__tests__/mutationRecoveryDecisions.spec.ts @@ -108,18 +108,22 @@ describe('durable mutation recovery decisions', () => { const restored = new MutationCoordinator({ repository: f.repository, evidence: f.evidence, resources }) cleanups.push(() => restored.dispose()) await restored.initialize() + await restored.waitForSettlements() expect(restored.getHazards()).toEqual([]) expect(f.evidence.decision('old')).toMatchObject({ kind: 'waived', outcome: 'applied' }) expect(f.database.prepare('SELECT settlement FROM run_mutation_processes').get()).toEqual({ settlement: 'unknown' }) }) - it('limits cross-task recovery to recorded blockers and rejects stale or active decisions', async () => { + it('shows admitted cross-task overlaps without blocking and rejects stale or active decisions', async () => { const f = await memoryFixture() const scope = context(f.database) await hazard(f.mutations) const recovery = new MutationRecoveryService({ mutations: f.mutations, evidence: f.evidence, context: scope, checkpoint: async () => null }) expect(recovery.list({ conversationId: 'next' })).toEqual([]) - await expect(f.mutations.acquire({ invocationId: 'next-attempt', runId: 'next-run', lifetime: 'runtime', claims: [{ kind: 'host' }], signal })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const next = await f.mutations.acquire({ invocationId: 'next-attempt', runId: 'next-run', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + expect(f.mutations.getRunState('next-run').active).toBe(1) + next.release() + expect(f.database.prepare('SELECT reason, related_run_id FROM run_mutation_evidence WHERE related_run_id IS NOT NULL').all()).toEqual([{ reason: 'RESOURCE_OVERLAP', related_run_id: 'next-run' }]) const item = recovery.list({ conversationId: 'next' })[0]! expect(item).toMatchObject({ invocationId: 'old', conversationId: 'old' }) expect(recovery.list({ conversationId: 'unrelated' })).toEqual([]) @@ -134,6 +138,85 @@ describe('durable mutation recovery decisions', () => { expect(f.database.prepare('SELECT settlement FROM run_mutation_processes').get()).toEqual({ settlement: 'unknown' }) }) + it('records only admitted overlapping requests and publishes the notice without weakening active ownership', async () => { + const f = await memoryFixture() + const scope = context(f.database) + const recovery = new MutationRecoveryService({ mutations: f.mutations, evidence: f.evidence, context: scope }) + const root = join(tmpdir(), 'mutation-overlap', 'shared') + const old = await f.mutations.acquire({ invocationId: 'old', runId: 'old-run', lifetime: 'external', claims: [{ kind: 'tree', canonicalPath: root }], signal }) + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: 'unverified-owner', settlement: 'unknown' }) + const observed: { active: number, invocationIds: string[] }[] = [] + const subscription = f.mutations.onDidChange(({ runId }) => { + if (runId === 'next-run') + observed.push({ active: f.mutations.getRunState(runId).active, invocationIds: recovery.list({ conversationId: 'next' }).map(item => item.invocationId) }) + }) + cleanups.push(() => subscription.dispose()) + const input = { runId: 'next-run', lifetime: 'runtime' as const, claims: [{ kind: 'file' as const, canonicalPath: join(root, 'note.txt') }], signal } + const pending = f.mutations.acquire({ ...input, invocationId: 'next' }) + const controller = new AbortController() + const cancelled = f.mutations.acquire({ ...input, runId: 'unrelated-run', invocationId: 'cancelled', signal: controller.signal }) + const rejected = expect(cancelled).rejects.toMatchObject({ name: 'AbortError' }) + controller.abort() + await rejected + const unrelated = await f.mutations.acquire({ ...input, runId: 'unrelated-run', invocationId: 'unrelated', claims: [{ kind: 'file', canonicalPath: join(`${root}-other`, 'note.txt') }] }) + unrelated.release() + expect(f.mutations.getRunState('next-run')).toMatchObject({ active: 0, pending: 1 }) + expect(f.evidence.relatedRuns('old')).toEqual([]) + old.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + const next = await pending + expect(observed).toContainEqual({ active: 1, invocationIds: ['old'] }) + next.release() + const repeated = await f.mutations.acquire({ ...input, invocationId: 'next-repeat' }) + repeated.release() + expect(f.evidence.relatedRuns('old')).toEqual(['next-run']) + expect(f.evidence.latest('old')).toBeNull() + expect(recovery.list({ conversationId: 'unrelated' })).toEqual([]) + expect(f.database.prepare('SELECT reason, related_run_id FROM run_mutation_evidence').all()).toEqual([{ reason: 'RESOURCE_OVERLAP', related_run_id: 'next-run' }]) + expect(f.mutations.getOwnedProcesses('old')).toEqual([{ processId: 'unverified-owner', settlement: 'unknown' }]) + }) + + it('preserves new overlap notices and legacy blocked associations after reopening without rewriting evidence', async () => { + const root = await mkdtemp(join(tmpdir(), 'mutation-overlap-reopen-')) + cleanups.push(() => rm(root, { recursive: true, force: true })) + const databasePath = join(root, 'buddy.sqlite3') + let database = openBuddyDatabase({ databasePath }) + cleanups.push(() => database.close()) + let f = setup(database) + context(database) + await hazard(f.mutations) + const next = await f.mutations.acquire({ invocationId: 'next', runId: 'next-run', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + next.release() + f.evidence.append({ invocationId: 'old', kind: 'observed', outcome: 'unknown', reason: 'WRITE_BLOCKED', actor: 'system', relatedRunId: 'unrelated-run' }) + const before = database.prepare('SELECT * FROM run_mutation_evidence ORDER BY sequence').all() + expect(before.map(row => row.reason)).toEqual(['RESOURCE_OVERLAP', 'WRITE_BLOCKED']) + await f.mutations.dispose() + database.close() + database = openBuddyDatabase({ databasePath }) + f = setup(database) + const recovery = new MutationRecoveryService({ mutations: f.mutations, evidence: f.evidence, context: { + conversations: createConversationRepository(database), + runs: createRunRepository(database), + } }) + for (const conversationId of ['next', 'unrelated']) + expect(recovery.list({ conversationId })).toEqual([expect.objectContaining({ invocationId: 'old', status: 'unknown', conversationId: 'old' })]) + expect(database.prepare('SELECT * FROM run_mutation_evidence ORDER BY sequence').all()).toEqual(before) + expect(f.mutations.getInvocationSettlement('old')).toBe('unknown') + }) + + it('does not turn a failed risk-notice write into an admission lock or a successful recovery', async () => { + const f = await memoryFixture() + await hazard(f.mutations) + f.database.exec(`CREATE TRIGGER reject_overlap BEFORE INSERT ON run_mutation_evidence + WHEN NEW.related_run_id IS NOT NULL BEGIN SELECT RAISE(ABORT, 'notice unavailable'); END`) + const next = await f.mutations.acquire({ invocationId: 'next', runId: 'next-run', lifetime: 'runtime', claims: [{ kind: 'host' }], signal }) + expect(f.mutations.getRunState('next-run').active).toBe(1) + expect(f.evidence.relatedRuns('old')).toEqual([]) + expect(f.evidence.decision('old')).toBeNull() + expect(f.mutations.getInvocationSettlement('old')).toBe('unknown') + next.release() + expect(f.repository.listUnreleased().map(item => item.invocationId)).toEqual(['old']) + }) + it('atomically binds launch authorization and never rolls it back to prepared', async () => { const f = await memoryFixture() const executionId = randomUUID() @@ -195,6 +278,7 @@ describe('durable mutation recovery decisions', () => { cleanups.push(() => restored.dispose()) expect(restored.getHazards()).toHaveLength(1) await restored.initialize() + await restored.waitForSettlements() expect(restored.getHazards()).toEqual([]) expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) }) diff --git a/apps/buddy/service/src/changes/__tests__/mutationRecoveryScheduling.spec.ts b/apps/buddy/service/src/changes/__tests__/mutationRecoveryScheduling.spec.ts new file mode 100644 index 000000000..cd1c3a657 --- /dev/null +++ b/apps/buddy/service/src/changes/__tests__/mutationRecoveryScheduling.spec.ts @@ -0,0 +1,509 @@ +import type { OwnedProcessVerification } from '../../../../platform/process/ownedProcessRecovery' +import type { MutationRecoveryOptions } from '../MutationRecoveryService' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createConversationRepository } from '../../storage/conversationRepository' +import { openBuddyDatabase } from '../../storage/database' +import { createRunRepository } from '../../storage/runRepository' +import { MutationCoordinator } from '../MutationCoordinator' +import { createMutationEvidenceRepository } from '../mutationEvidenceRepository' +import { MutationRecoveryService } from '../MutationRecoveryService' +import { createMutationRepository } from '../mutationRepository' + +const cleanups: (() => void | Promise)[] = [] +beforeEach(() => { + vi.useFakeTimers() +}) +afterEach(async () => { + try { + for (const cleanup of cleanups.splice(0).reverse()) + await cleanup() + } + finally { vi.useRealTimers() } +}) + +const signal = new AbortController().signal +const unknown: OwnedProcessVerification = { status: 'unknown', reason: 'receipt_missing' } +const released: OwnedProcessVerification = { status: 'released', reason: 'supervisor_receipt' } +const active: OwnedProcessVerification = { status: 'active', reason: 'supervisor_active' } +const native = (id: string) => JSON.stringify({ kind: 'native-shell', platform: process.platform, pid: 123, startTime: '123', nonce: id }) +const claims = (id: string) => [{ kind: 'tree' as const, canonicalPath: join(tmpdir(), 'mutation-recovery-scheduling', id) }] +const flush = () => vi.advanceTimersByTimeAsync(0) + +function fixture(resources?: NonNullable[0]>['resources']) { + const database = openBuddyDatabase({ databasePath: ':memory:' }) + let closed = false + const close = () => { + if (!closed) { + closed = true + database.close() + } + } + cleanups.push(close) + const repository = createMutationRepository(database) + const evidence = createMutationEvidenceRepository(database) + const mutations = new MutationCoordinator({ repository, evidence, resources }) + cleanups.push(() => mutations.dispose()) + return { + database, + repository, + evidence, + mutations, + close, + async hazard(id: string, effect: () => void = () => {}) { + const lease = await mutations.acquire({ runId: `${id}-run`, invocationId: id, claims: claims(id), lifetime: 'external', signal }) + await mutations.withInvocation(lease, async () => { + effect() + }) + mutations.recordOwnedProcessSettlement({ invocationId: id, processId: native(id), settlement: 'unknown' }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + await mutations.waitForSettlements() + }, + recovery(options: Omit = {}) { + const recovery = new MutationRecoveryService({ mutations, evidence, checkpoint: async () => null, ...options }) + cleanups.push(async () => { + mutations.stopAdmission() + await recovery.dispose() + }) + return recovery + }, + } +} + +function verificationGate() { + const gate = Promise.withResolvers() + cleanups.push(() => gate.resolve(unknown)) + return gate +} + +function context(f: ReturnType) { + const conversations = createConversationRepository(f.database) + const runs = createRunRepository(f.database) + const now = new Date().toISOString() + conversations.create({ id: 'task', branchId: 'task', createdAt: now, approvalPolicy: 'policy', executionProfile: 'workspace_write', spaceId: null, title: 'Recovery' }) + conversations.createMessage({ id: 'question', conversationId: 'task', branchId: 'task', role: 'user', runId: null, createdAt: now, content: { text: 'Fixture' } }) + runs.create({ id: 'old-run', conversationId: 'task', branchId: 'task', triggeringMessageId: 'question', provider: 'fixture', model: 'fixture', purpose: 'chat', status: 'failed', startedAt: now, approvalPolicy: 'policy', executionProfile: 'workspace_write', piSessionFile: null }) + return { conversations, runs } +} + +describe('bounded mutation recovery scheduling', () => { + it('starts immediately and continues bounded background checks without blocking new work', async () => { + const f = fixture() + for (const id of ['one', 'two', 'three', 'four']) + await f.hazard(id) + const gate = verificationGate() + const visited = new Map() + const recovery = f.recovery({ verify: async (id) => { + const attempt = (visited.get(id) ?? 0) + 1 + visited.set(id, attempt) + return attempt === 1 && [native('one'), native('two')].includes(id) ? gate.promise : released + } }) + let started = false + const starting = recovery.start().then(() => { + started = true + }) + await flush() + expect([...visited.keys()]).toEqual([native('one'), native('two')]) + await vi.advanceTimersByTimeAsync(4_999) + expect(started).toBe(true) + const next = await f.mutations.acquire({ runId: 'next-run', invocationId: 'next', claims: claims('one'), lifetime: 'runtime', signal }) + next.release() + expect(f.mutations.getOwnedProcesses('one')).toEqual([{ processId: native('one'), settlement: 'unknown' }]) + await vi.advanceTimersByTimeAsync(1) + await starting + expect(started).toBe(true) + expect(f.repository.listUnreleased().map(item => item.invocationId).sort()).toEqual(['one', 'two']) + await vi.advanceTimersByTimeAsync(1_000) + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.decision('four')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) + gate.resolve(active) + await flush() + expect(f.repository.listUnreleased()).toEqual([]) + }) + + it('admits new work after exhausted verification without replaying or settling the old effect', async () => { + const f = fixture() + const effects: string[] = [] + await f.hazard('old', () => effects.push('old')) + let outcome = unknown + const recovery = f.recovery({ verify: async () => outcome }) + await recovery.start() + await vi.advanceTimersByTimeAsync(60_000) + expect(f.repository.listUnreleased()).toHaveLength(1) + outcome = released + const lease = await f.mutations.acquire({ runId: 'next-run', invocationId: 'next', claims: claims('old'), lifetime: 'runtime', signal }) + await f.mutations.withInvocation(lease, async () => { + effects.push('next') + }) + lease.release() + await f.mutations.waitForSettlements() + expect(effects).toEqual(['old', 'next']) + expect(f.evidence.relatedRuns('old')).toEqual(['next-run']) + expect(f.repository.listUnreleased().map(item => item.invocationId)).toEqual(['old']) + expect(f.mutations.getOwnedProcesses('old')).toEqual([{ processId: native('old'), settlement: 'unknown' }]) + await recovery.recheck() + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied', actor: 'system' }) + }) + + it('shares verification for one invocation while cancellation only ends that caller\'s wait', async () => { + const f = fixture() + await f.hazard('old') + const gate = verificationGate() + const entered = Promise.withResolvers() + let checks = 0 + const recovery = f.recovery({ verify: async (_id, options) => { + checks++ + entered.resolve(options!.signal!) + return gate.promise + } }) + const controller = new AbortController() + const first = recovery.recheck({ invocationIds: ['old'], signal: controller.signal }) + const sharedSignal = await entered.promise + let secondFinished = false + const second = recovery.recheck({ invocationIds: ['old'] }).then(() => { + secondFinished = true + }) + controller.abort() + await first + expect(sharedSignal.aborted).toBe(false) + expect(secondFinished).toBe(false) + gate.resolve(released) + await second + expect(checks).toBe(1) + expect(f.repository.listUnreleased()).toEqual([]) + }) + + it('preserves observed active and unknown facts while bounding background retries', async () => { + const f = fixture() + await f.hazard('old') + let outcome = active + let checks = 0 + const recovery = f.recovery({ verify: async () => { + checks++ + return outcome + } }) + await recovery.start() + await flush() + const next = await f.mutations.acquire({ runId: 'next-run', invocationId: 'next', claims: claims('old'), lifetime: 'runtime', signal }) + next.release() + expect(f.evidence.latest('old')).toMatchObject({ outcome: 'active' }) + outcome = unknown + await vi.advanceTimersByTimeAsync(60_000) + const exhausted = checks + expect(f.evidence.latest('old')).toMatchObject({ outcome: 'unknown' }) + expect(f.repository.listUnreleased()).toHaveLength(1) + await vi.advanceTimersByTimeAsync(600_000) + expect(checks).toBe(exhausted) + outcome = released + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: native('new-child'), settlement: 'unknown' }) + await flush() + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) + }) + + it('persists each verified process so a multi-process invocation can recover across verification budgets', async () => { + const f = fixture() + await f.hazard('old') + for (const id of ['second', 'third']) + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: native(id), settlement: 'unknown' }) + const recovery = f.recovery({ verify: async () => { + await new Promise(resolve => setTimeout(resolve, 2_000)) + return released + } }) + const starting = recovery.start() + await vi.advanceTimersByTimeAsync(5_000) + await starting + expect(f.repository.listUnreleased()[0]?.processes).toEqual(expect.arrayContaining([ + { processId: native('old'), settlement: 'released' }, + { processId: native('second'), settlement: 'released' }, + { processId: native('third'), settlement: 'unknown' }, + ])) + expect(f.mutations.getHazards()).toHaveLength(1) + await vi.advanceTimersByTimeAsync(2_000) + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) + }) + + it('does not restart exhausted retries for duplicate notices about an already released process', async () => { + const f = fixture() + await f.hazard('old') + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: native('unknown-child'), settlement: 'unknown' }) + const recovery = f.recovery({ verify: async id => id === native('old') ? released : unknown }) + await recovery.start() + await vi.advanceTimersByTimeAsync(60_000) + const observations = f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get() + for (let index = 0; index < 3; index++) { + f.mutations.recordOwnedProcessSettlement({ + invocationId: 'old', + processId: native('old'), + settlement: 'unknown', + launchState: 'settled', + executionId: '11111111-1111-4111-8111-111111111111', + dispatchId: '22222222-2222-4222-8222-222222222222', + }) + } + await vi.advanceTimersByTimeAsync(60_000) + expect(f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get()).toEqual(observations) + expect(f.repository.listUnreleased()[0]?.processes).toEqual(expect.arrayContaining([ + { processId: native('old'), settlement: 'released' }, + { processId: native('unknown-child'), settlement: 'unknown' }, + ])) + expect(f.evidence.decision('old')).toBeNull() + }) + + it('stops polling unverifiable legacy records without silently waiving their protection', async () => { + const f = fixture() + const lease = await f.mutations.acquire({ runId: 'legacy-run', invocationId: 'legacy', claims: claims('legacy'), lifetime: 'external', signal }) + lease.markHazard('OWNED_PROCESS_CLEANUP_UNCONFIRMED') + const recovery = f.recovery() + await recovery.start() + await vi.advanceTimersByTimeAsync(60_000) + const observations = f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get() + await vi.advanceTimersByTimeAsync(600_000) + expect(f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get()).toEqual(observations) + expect(f.evidence.latest('legacy')).toMatchObject({ outcome: 'unknown', reason: 'PROCESS_IDENTITY_MISSING' }) + expect(f.evidence.decision('legacy')).toBeNull() + expect(f.repository.listUnreleased()).toHaveLength(1) + }) + + it.each(['quiesced', 'waived'] as const)('resumes a requested %s decision after transient native release failure', async (kind) => { + let failing = true + const f = fixture({ + acquire: async () => {}, + reconcile: async () => {}, + release: async (_id, protectedState) => { + if (!protectedState && failing) + throw new Error('NATIVE_RELEASE_FAILED') + }, + }) + await f.hazard('old') + if (kind === 'waived') + await expect(f.mutations.waiveHazard('old')).rejects.toThrow('NATIVE_RELEASE_FAILED') + const recovery = f.recovery({ verify: async () => released }) + await recovery.start() + await flush() + expect(f.evidence.decision('old')).toMatchObject({ kind, outcome: 'requested' }) + expect(f.repository.listUnreleased()).toHaveLength(1) + failing = false + await vi.advanceTimersByTimeAsync(1_000) + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.decision('old')).toMatchObject({ kind, outcome: 'applied', actor: kind === 'waived' ? 'user' : 'system' }) + expect(f.database.prepare('SELECT settlement FROM run_mutation_processes').get()).toEqual({ settlement: kind === 'waived' ? 'unknown' : 'released' }) + }) + + it('keeps requested release protected past timeout until the coordinator finishes native settlement', async () => { + const release = Promise.withResolvers() + const f = fixture({ + acquire: async () => {}, + reconcile: async () => {}, + release: async (_id, protectedState) => { + if (!protectedState) + await release.promise + }, + }) + cleanups.push(() => release.resolve()) + await f.hazard('old') + const recovery = f.recovery({ verify: async () => released }) + const starting = recovery.start() + await vi.advanceTimersByTimeAsync(5_000) + await starting + expect(f.repository.listUnreleased()).toHaveLength(1) + expect(f.evidence.decision('old')).toMatchObject({ kind: 'quiesced', outcome: 'requested' }) + await recovery.dispose() + expect(f.repository.listUnreleased()).toHaveLength(1) + release.resolve() + await f.mutations.waitForSettlements() + expect(f.repository.listUnreleased()).toEqual([]) + }) + + it('cancels a pending acquire behind a real lease while background verification remains independent', async () => { + const f = fixture() + await f.hazard('old') + const gate = verificationGate() + const recovery = f.recovery({ verify: async () => gate.promise }) + await recovery.start() + const running = await f.mutations.acquire({ runId: 'running-run', invocationId: 'running', claims: claims('old'), lifetime: 'runtime', signal }) + const effects: string[] = [] + const controller = new AbortController() + const first = f.mutations.acquire({ runId: 'cancelled-run', invocationId: 'cancelled', claims: claims('old'), lifetime: 'runtime', signal: controller.signal }).then((lease) => { + effects.push('cancelled') + lease.release() + }) + const cancelled = expect(first).rejects.toMatchObject({ name: 'AbortError' }) + const second = f.mutations.acquire({ runId: 'next-run', invocationId: 'next', claims: claims('old'), lifetime: 'runtime', signal }).then((lease) => { + effects.push('next') + lease.release() + }) + await flush() + controller.abort() + await cancelled + expect(f.mutations.getRunState('cancelled-run')).toMatchObject({ pending: 0, active: 0 }) + expect(f.mutations.getRunState('next-run')).toMatchObject({ pending: 1, active: 0 }) + running.release() + await second + expect(effects).toEqual(['next']) + expect(f.repository.listUnreleased().map(item => item.invocationId)).toEqual(['old']) + gate.resolve(released) + await recovery.recheck() + expect(f.repository.listUnreleased()).toEqual([]) + }) + + it('disposes background verification without blocking or rewriting subsequent operations', async () => { + const f = fixture() + const gate = verificationGate() + const recovery = f.recovery({ verify: async () => gate.promise }) + await recovery.start() + await f.hazard('old') + const next = await f.mutations.acquire({ runId: 'next-run', invocationId: 'next', claims: claims('old'), lifetime: 'runtime', signal }) + next.release() + await flush() + await recovery.dispose() + gate.resolve(released) + await flush() + expect(f.mutations.getRunState('next-run')).toMatchObject({ pending: 0, active: 0 }) + expect(f.repository.listUnreleased().map(item => item.invocationId)).toEqual(['old']) + expect(f.mutations.getOwnedProcesses('old')).toEqual([{ processId: native('old'), settlement: 'unknown' }]) + const later = await f.mutations.acquire({ runId: 'later-run', invocationId: 'later', claims: claims('old'), lifetime: 'runtime', signal }) + later.release() + }) + + it.each(['verify', 'checkpoint'] as const)('discards a late %s result after disposal and database close', async (stage) => { + const f = fixture() + await f.hazard('old') + const gate = Promise.withResolvers() + const entered = Promise.withResolvers() + let writes = 0 + const append = f.evidence.append + f.evidence.append = (value) => { + writes++ + append(value) + } + const recovery = f.recovery({ + verify: async () => { + if (stage === 'verify') { + entered.resolve() + await gate.promise + } + return stage === 'verify' ? released : unknown + }, + checkpoint: async () => { + entered.resolve() + await gate.promise + return JSON.stringify({ kind: 'recovery-boot', version: 1, platform: process.platform, bootId: '11111111-1111-4111-8111-111111111111' }) + }, + }) + const checking = recovery.recheck() + await entered.promise + await recovery.dispose() + await checking + const writesAtDisposal = writes + expect(f.repository.listUnreleased()[0]?.processes).toEqual([{ processId: native('old'), settlement: 'unknown' }]) + f.close() + gate.resolve() + await vi.advanceTimersByTimeAsync(600_000) + expect(writes).toBe(writesAtDisposal) + expect(f.mutations.getOwnedProcesses('old')).toEqual([{ processId: native('old'), settlement: 'unknown' }]) + }) + + it.each(['verify', 'checkpoint'] as const)('does not overwrite completed settlement with a late %s observation', async (stage) => { + const f = fixture() + await f.hazard('old') + const gate = Promise.withResolvers() + const entered = Promise.withResolvers() + const recovery = f.recovery({ + verify: async () => { + if (stage === 'verify') { + entered.resolve() + await gate.promise + } + return stage === 'verify' ? active : unknown + }, + checkpoint: async () => { + entered.resolve() + await gate.promise + return JSON.stringify({ kind: 'recovery-boot', version: 1, platform: process.platform, bootId: '11111111-1111-4111-8111-111111111111' }) + }, + }) + const checking = recovery.recheck() + await entered.promise + f.mutations.recordOwnedProcessSettlement({ invocationId: 'old', processId: native('old'), settlement: 'released' }) + await f.mutations.waitForSettlements() + gate.resolve() + await checking + expect(f.repository.listUnreleased()).toEqual([]) + expect(f.evidence.latest('old')).toMatchObject({ kind: 'quiesced', outcome: 'applied' }) + }) + + it('revalidates a queued waiver after the same invocation becomes active', async () => { + const f = fixture() + await f.hazard('old') + const gate = verificationGate() + const recovery = f.recovery({ context: context(f), verify: async () => gate.promise }) + const item = recovery.list({ conversationId: 'task' })[0]! + const checking = recovery.recheck() + const resolving = recovery.resolve({ conversationId: 'task', decision: 'waived', items: [item] }) + const rejected = expect(resolving).rejects.toMatchObject({ code: 'VALIDATION_FAILED' }) + await flush() + gate.resolve(active) + await checking + await rejected + expect(f.evidence.decision('old')).toBeNull() + expect(recovery.list({ conversationId: 'task' })[0]).toMatchObject({ status: 'active' }) + expect(f.repository.listUnreleased()).toHaveLength(1) + }) + + it('allows new work immediately while historical verification keeps its background order', async () => { + const f = fixture() + for (const id of ['one', 'two', 'three', 'four', 'five']) + await f.hazard(id) + const gate = verificationGate() + const visited: string[] = [] + const recovery = f.recovery({ verify: async (id) => { + visited.push(id) + return [native('one'), native('two')].includes(id) ? gate.promise : id === native('five') ? released : unknown + } }) + const starting = recovery.start() + await flush() + const writer = await f.mutations.acquire({ runId: 'writer-run', invocationId: 'writer', claims: claims('five'), lifetime: 'runtime', signal }) + writer.release() + await flush() + expect(visited).toEqual([native('one'), native('two')]) + const unrelated = await f.mutations.acquire({ runId: 'free-run', invocationId: 'free', claims: claims('free'), lifetime: 'runtime', signal }) + unrelated.release() + gate.resolve(unknown) + await starting + await flush() + expect(visited).toEqual(['one', 'two', 'three', 'four', 'five'].map(native)) + expect(f.repository.isReleased('five')).toBe(true) + expect(f.repository.listUnreleased().map(item => item.invocationId).sort()).toEqual(['four', 'one', 'three', 'two']) + }) + + it('does not schedule background work until explicitly started', async () => { + const f = fixture() + await f.hazard('old') + const recovery = f.recovery({ verify: async () => unknown }) + await recovery.recheck() + const observations = f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get() + await vi.advanceTimersByTimeAsync(600_000) + expect(f.database.prepare('SELECT count(*) AS count FROM run_mutation_evidence').get()).toEqual(observations) + expect(f.repository.listUnreleased()).toHaveLength(1) + }) + + it('isolates a failed invocation verification from other recoverable hazards', async () => { + const f = fixture() + await f.hazard('bad') + await f.hazard('good') + const recovery = f.recovery({ verify: async (id) => { + if (id === native('bad')) + throw new Error('VERIFIER_UNAVAILABLE') + return released + } }) + await recovery.recheck() + expect(f.repository.isReleased('good')).toBe(true) + expect(f.repository.listUnreleased().map(item => item.invocationId)).toEqual(['bad']) + expect(f.evidence.latest('bad')).toMatchObject({ outcome: 'unknown', reason: 'RECOVERY_VERIFICATION_FAILED' }) + }) +}) diff --git a/apps/buddy/service/src/changes/mutationEvidenceRepository.ts b/apps/buddy/service/src/changes/mutationEvidenceRepository.ts index b3dfd928a..509c87f87 100644 --- a/apps/buddy/service/src/changes/mutationEvidenceRepository.ts +++ b/apps/buddy/service/src/changes/mutationEvidenceRepository.ts @@ -20,8 +20,8 @@ export interface MutationEvidenceRepository { append: (evidence: MutationEvidence) => void latest: (invocationId: string) => StoredMutationEvidence | null decision: (invocationId: string) => StoredMutationEvidence | null - recordBlocker: (invocationId: string, runId: string) => void - blockedRuns: (invocationId: string) => string[] + recordOverlap: (invocationId: string, runId: string) => void + relatedRuns: (invocationId: string) => string[] applyResolution: (invocationId: string, kind: 'quiesced' | 'waived') => void createdAt: (invocationId: string) => string | null } @@ -52,7 +52,7 @@ export function createMutationEvidenceRepository(database: DatabaseSync): Mutati SELECT * FROM run_mutation_evidence WHERE invocation_id = ? AND outcome IN ('requested', 'applied') AND process_id IS NULL AND related_run_id IS NULL ORDER BY sequence DESC LIMIT 1 `) - const blockedRuns = database.prepare('SELECT related_run_id FROM run_mutation_evidence WHERE invocation_id = ? AND related_run_id IS NOT NULL') + const relatedRuns = database.prepare('SELECT related_run_id FROM run_mutation_evidence WHERE invocation_id = ? AND related_run_id IS NOT NULL') const createdAt = database.prepare('SELECT created_at FROM run_mutation_leases WHERE invocation_id = ?') const settle = database.prepare('UPDATE run_mutation_leases SET status = \'released\', reason = ?, updated_at = ? WHERE invocation_id = ?') const append = (evidence: MutationEvidence) => { @@ -62,8 +62,8 @@ export function createMutationEvidenceRepository(database: DatabaseSync): Mutati append, latest: invocationId => toEvidence(latest.get(invocationId) as EvidenceRow | undefined), decision: invocationId => toEvidence(decision.get(invocationId) as EvidenceRow | undefined), - recordBlocker: (invocationId, runId) => append({ invocationId, kind: 'observed', outcome: 'unknown', actor: 'system', reason: 'WRITE_BLOCKED', relatedRunId: runId }), - blockedRuns: invocationId => (blockedRuns.all(invocationId) as { related_run_id: string }[]).map(row => row.related_run_id), + recordOverlap: (invocationId, runId) => append({ invocationId, kind: 'observed', outcome: 'unknown', actor: 'system', reason: 'RESOURCE_OVERLAP', relatedRunId: runId }), + relatedRuns: invocationId => (relatedRuns.all(invocationId) as { related_run_id: string }[]).map(row => row.related_run_id), createdAt: invocationId => (createdAt.get(invocationId) as { created_at: string } | undefined)?.created_at ?? null, applyResolution(invocationId, kind) { withTransaction(database, () => { diff --git a/apps/buddy/service/src/conversations/registerConversationRpc.ts b/apps/buddy/service/src/conversations/registerConversationRpc.ts index c2b59b440..c1754a11a 100644 --- a/apps/buddy/service/src/conversations/registerConversationRpc.ts +++ b/apps/buddy/service/src/conversations/registerConversationRpc.ts @@ -1,6 +1,7 @@ import type { AttachmentService } from '../attachments/AttachmentService' import type { ChangeCaptureService } from '../changes/ChangeCaptureService' import type { BuddyRunEvent } from '../events/BuddyRunEvent' +import type { RunEventAvailability } from '../events/RunEventPorts' import type { RuntimeRequestRegistrar } from '../rpc/runtimeRequest' import type { ArtifactRepository, @@ -45,7 +46,7 @@ export interface RegisterConversationRpcOptions { conversations: ConversationRpcRepository metadata: Pick deleteConversation: (conversationId: string) => Promise - eventLog: { listForRuns: (runIds: readonly string[]) => BuddyRunEvent[] } + eventLog: { listForRuns: (runIds: readonly string[]) => BuddyRunEvent[] } & Partial> isDeleting: (conversationId: string) => boolean rpc: RuntimeRequestRegistrar runInputs: Pick @@ -160,6 +161,7 @@ export function registerConversationRpc(options: RegisterConversationRpcOptions) ) const runEvents = options.eventLog.listForRuns(runs.map(run => run.id)) const runIds = runs.map(run => run.id) + const quarantined = new Set(options.eventLog.getRecoveryFailures?.().map(failure => failure.runId)) return { changeSets: options.changes.listSummariesForRuns(runIds), items, @@ -176,7 +178,7 @@ export function registerConversationRpc(options: RegisterConversationRpcOptions) ), runEvents: runEvents.map(toPublicRunEvent), runs: runs.map(run => toPublicRun( - run, + quarantined.has(run.id) ? { ...run, errorCode: 'EVENT_LOG_CORRUPTED' } : run, options.runInputs.findByRunId(run.id)?.reasoning ?? null, )), } diff --git a/apps/buddy/service/src/events/RunEventLog.ts b/apps/buddy/service/src/events/RunEventLog.ts index 9f7515be7..235bcf3c7 100644 --- a/apps/buddy/service/src/events/RunEventLog.ts +++ b/apps/buddy/service/src/events/RunEventLog.ts @@ -113,9 +113,8 @@ export class RunEventLog implements RunEventLogPort { return Object.freeze([...this.#quarantined.values()].map(entry => entry.failure)) } - assertConversationWritable(conversationId: string): void { + assertConversationWritable(_conversationId: string): void { this.#assertMutationAllowed() - this.#assertConversationAvailable(conversationId) } assertRunExecutable(runId: string): void { @@ -189,21 +188,16 @@ export class RunEventLog implements RunEventLogPort { options: Pick = {}, ): BuddyRunEvent[] { this.#assertOpen() - this.#assertConversationAvailable(conversationId) - return this.#queries.listForConversation(conversationId, options) + return this.#queries.listForConversation(conversationId, { ...options, excludedRunIds: [...this.#quarantined.keys()] }) } listForRuns(runIds: readonly string[], options?: { latestActivityOnly?: boolean }): BuddyRunEvent[] { this.#assertOpen() - for (const runId of runIds) - this.#assertRunAvailable(runId) - return this.#queries.listForRuns(runIds, options) + return this.#queries.listForRuns(runIds.filter(runId => !this.#quarantined.has(runId)), options) } listToolCounts(runIds: readonly string[]): ReadonlyMap { this.#assertOpen() - for (const runId of runIds) - this.#assertRunAvailable(runId) return this.#queries.listToolCounts(runIds) } @@ -500,22 +494,10 @@ export class RunEventLog implements RunEventLogPort { throw entry.error } - #assertConversationAvailable(conversationId: string): void { - for (const entry of this.#quarantined.values()) { - if (entry.failure.conversationId === conversationId) - throw entry.error - } - } - #assertRunAvailable(runId: string): void { const entry = this.#quarantined.get(runId) if (entry) throw entry.error - if (!this.#quarantined.size) - return - const conversationId = this.#queries.findConversationId(runId) - if (conversationId) - this.#assertConversationAvailable(conversationId) } #isTerminalRun(runId: string): boolean { diff --git a/apps/buddy/service/src/events/RunEventQueries.ts b/apps/buddy/service/src/events/RunEventQueries.ts index e63bbc8d5..aaec204b5 100644 --- a/apps/buddy/service/src/events/RunEventQueries.ts +++ b/apps/buddy/service/src/events/RunEventQueries.ts @@ -38,9 +38,10 @@ export class RunEventQueries { listForConversation( conversationId: string, - options: Pick = {}, + options: Pick & { excludedRunIds?: readonly string[] } = {}, ): BuddyRunEvent[] { const limit = options.limit ?? 1_000 + const excluded = options.excludedRunIds ?? [] const rows = this.#database.prepare(` SELECT * FROM ( SELECT @@ -52,11 +53,12 @@ export class RunEventQueries { FROM run_events AS events INNER JOIN runs ON runs.id = events.run_id WHERE runs.conversation_id = ? AND runs.execution_kind IN ('root', 'maintenance') + ${excluded.length ? `AND runs.id NOT IN (${excluded.map(() => '?').join(', ')})` : ''} ORDER BY events.created_at DESC, events.run_id DESC, events.sequence DESC LIMIT ? ) ORDER BY created_at, run_id, sequence - `).all(conversationId, limit) + `).all(conversationId, ...excluded, limit) return (rows as unknown as RunEventRow[]).map(toRunEvent) } diff --git a/apps/buddy/service/src/events/RunEventStore.ts b/apps/buddy/service/src/events/RunEventStore.ts index f0f4d2aa9..07ee93bdc 100644 --- a/apps/buddy/service/src/events/RunEventStore.ts +++ b/apps/buddy/service/src/events/RunEventStore.ts @@ -127,27 +127,21 @@ export class RunEventStore { { cause: error }, ) } - let parsed: ReturnType + let events: BuddyRunEvent[] try { - parsed = parseEventLines(content, runId) + events = parseEventLines(content, runId) } catch (cause) { throw new RunEventCorruptionError(runId, { cause }) } - const scope = runEventFailureScope(runId, parsed.events) - const repairBytes = parsed.repairBytes - if (repairBytes !== null) { - await this.#repairEventFile(scope, 'truncate', file => ( - file.truncate(repairBytes) - )) - } - else if (content.length > 0 && !content.endsWith('\n')) { + const scope = runEventFailureScope(runId, events) + if (content.length > 0 && !content.endsWith('\n')) { const appendPosition = Buffer.byteLength(content, 'utf8') await this.#repairEventFile(scope, 'write', file => ( file.write('\n', appendPosition, 'utf8').then(() => undefined) )) } - return parsed.events + return events } async replace( @@ -362,34 +356,20 @@ export class RunEventStore { function parseEventLines( content: string, runId: string, -): { events: BuddyRunEvent[], repairBytes: number | null } { - const hasFinalNewline = content.endsWith('\n') +): BuddyRunEvent[] { const lines = content.split('\n') if (lines.at(-1) === '') lines.pop() const events: BuddyRunEvent[] = [] - for (const [index, line] of lines.entries()) { + for (const line of lines) { if (!line.trim()) continue - let event: BuddyRunEvent - try { - event = buddyRunEventSchema.parse(JSON.parse(line)) - } - catch (error) { - if (index === lines.length - 1 && !hasFinalNewline) { - const validPrefix = lines.slice(0, -1).join('\n') - return { - events, - repairBytes: Buffer.byteLength(validPrefix ? `${validPrefix}\n` : '', 'utf8'), - } - } - throw error - } + const event = buddyRunEventSchema.parse(JSON.parse(line)) if (event.runId !== runId || event.sequence <= (events.at(-1)?.sequence ?? 0)) throw new Error(`Lexora Buddy run event sequence is invalid: ${runId}`) events.push(event) } - return { events, repairBytes: null } + return events } async function writeDurableFile( diff --git a/apps/buddy/service/src/events/__tests__/RunEventLog.spec.ts b/apps/buddy/service/src/events/__tests__/RunEventLog.spec.ts index c3d3536f9..2c9406c2c 100644 --- a/apps/buddy/service/src/events/__tests__/RunEventLog.spec.ts +++ b/apps/buddy/service/src/events/__tests__/RunEventLog.spec.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from 'node:sqlite' import type { RunEventLogCallbacks } from '../RunEventLog' -import { appendFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' @@ -90,16 +90,17 @@ describe('runEventLog', () => { dispose() }) - it('repairs an incomplete tail before assigning the next sequence', async () => { + it('preserves an incomplete tail and rejects further writes to that run', async () => { const fixture = await createFixture() await fixture.log.append({ payload: {}, runId: 'run-1', type: 'run.started' }) await appendFile(fixture.eventFile, '{"runId":"run-1"') - await expect(fixture.log.read('run-1')).resolves.toHaveLength(1) + const damaged = await readFile(fixture.eventFile, 'utf8') + await expect(fixture.log.read('run-1')).rejects.toMatchObject({ code: 'EVENT_LOG_CORRUPTED' }) await expect(fixture.log.append({ payload: {}, runId: 'run-1', type: 'audit.recorded' })) - .resolves - .toMatchObject({ sequence: 2 }) - await expect(fixture.log.read('run-1')).resolves.toHaveLength(2) + .rejects + .toMatchObject({ code: 'EVENT_LOG_CORRUPTED' }) + expect(await readFile(fixture.eventFile, 'utf8')).toBe(damaged) }) it('quarantines a corrupt durable log without entering the global fatal channel', async () => { diff --git a/apps/buddy/service/src/events/__tests__/RunEventQuarantine.spec.ts b/apps/buddy/service/src/events/__tests__/RunEventQuarantine.spec.ts index a78a1cfbc..07f7dd30d 100644 --- a/apps/buddy/service/src/events/__tests__/RunEventQuarantine.spec.ts +++ b/apps/buddy/service/src/events/__tests__/RunEventQuarantine.spec.ts @@ -44,13 +44,14 @@ async function fixture() { const corruptions = [ ['terminated invalid JSON', '{broken}\n'], + ['torn final line', '{broken'], ['invalid middle line', `{broken}\n${JSON.stringify({ runId: 'child', sequence: 3, type: 'audit.after', payload: {}, createdAt: executionNow })}\n`], ['wrong run identity', `${JSON.stringify({ runId: 'root', sequence: 2, type: 'audit.after', payload: {}, createdAt: executionNow })}\n`], ['non-monotonic sequence', `${JSON.stringify({ runId: 'child', sequence: 1, type: 'audit.after', payload: {}, createdAt: executionNow })}\n`], ] as const describe('run event quarantine', () => { - it.each(corruptions)('isolates %s to its conversation without rewriting evidence or accepting stale projections', async (_, corruption) => { + it.each(corruptions)('isolates %s to its run without rewriting evidence or blocking healthy siblings', async (_, corruption) => { const f = await fixture() await appendFile(f.file('child'), corruption) const damaged = await readFile(f.file('child'), 'utf8') @@ -62,21 +63,33 @@ describe('run event quarantine', () => { expect(await readFile(f.file('child'), 'utf8')).toBe(damaged) expect(f.database.prepare('SELECT * FROM run_events WHERE run_id = ?').all('child')).toEqual(before) await expect(f.events.append({ runId: 'healthy-run', type: 'audit.after', payload: {} })).resolves.toMatchObject({ sequence: 2 }) - for (const runId of ['root', 'child']) { + for (const runId of ['child']) { await expect(f.events.append({ runId, type: 'run.completed', payload: {} })).rejects.toBeInstanceOf(RunEventCorruptionError) await expect(f.events.read(runId)).rejects.toBeInstanceOf(RunEventCorruptionError) await expect(f.events.list(runId)).rejects.toBeInstanceOf(RunEventCorruptionError) expect(() => f.events.assertRunWritable(runId)).toThrow(RunEventCorruptionError) } - expect(() => f.events.listForConversation('conversation')).toThrow(RunEventCorruptionError) - expect(() => f.events.listForRuns(['root'])).toThrow(RunEventCorruptionError) - expect(() => f.events.listToolCounts(['child'])).toThrow(RunEventCorruptionError) - expect(() => f.events.assertConversationWritable('conversation')).toThrow(RunEventCorruptionError) + await expect(f.events.append({ runId: 'root', type: 'audit.after', payload: {} })).resolves.toMatchObject({ sequence: 2 }) + expect(await f.events.read('root')).toHaveLength(2) + expect(f.events.listForConversation('conversation').map(event => event.runId)).toEqual(['root', 'root']) + expect(f.events.listForRuns(['root', 'child'])).toHaveLength(2) + expect(() => f.events.assertConversationWritable('conversation')).not.toThrow() await expect(f.events.replayAll()).rejects.toBeInstanceOf(RunEventCorruptionError) expect(f.events.getRecoveryFailures()).toHaveLength(1) expect(f.runs.findById('root')?.status).toBe('running') }) + it('excludes quarantined roots before limiting aggregate history and keeps recorded tool counts', async () => { + const f = await fixture() + f.runs.create({ ...f.runs.findById('root')!, id: 'other-root', rootRunId: 'other-root', status: 'completed' }) + await f.events.append({ runId: 'other-root', type: 'audit.healthy', payload: {}, createdAt: executionNow }) + await f.events.append({ runId: 'root', type: 'tool.started', payload: { toolName: 'read', toolCallId: 'recorded-tool' } }) + await appendFile(f.file('root'), '{broken}\n') + await f.events.recoverAll() + expect(f.events.listForConversation('conversation', { limit: 1 })).toMatchObject([{ runId: 'other-root', type: 'audit.healthy' }]) + expect(f.events.listToolCounts(['root']).get('root')).toBe(1) + }) + it('continues terminal-run compaction without touching a quarantined historical log', async () => { const f = await fixture() for (const runId of ['child', 'healthy-run']) { @@ -110,7 +123,7 @@ describe('run event quarantine', () => { await expect(f.events.read('child')).rejects.toBeInstanceOf(RunEventCorruptionError) await expect(f.events.replay('child')).resolves.toBe(1) expect(f.events.getRecoveryFailures().map(failure => failure.runId)).toEqual(['root']) - expect(() => f.events.assertRunWritable('child')).toThrow(RunEventCorruptionError) + expect(() => f.events.assertRunWritable('child')).not.toThrow() await writeFile(f.file('root'), originalRoot) expect(await f.events.recoverAll()).toMatchObject({ failed: 0, failures: [] }) await expect(f.events.append({ runId: 'child', type: 'audit.resumed', payload: {} })).resolves.toMatchObject({ sequence: 2 }) @@ -128,7 +141,7 @@ describe('run event quarantine', () => { const reopened = createRunEventLog({ database: f.database, conversationsDirectory: f.directory }) try { expect(await reopened.recoverAll()).toMatchObject({ failed: 1, failures: [{ runId: 'child', conversationId: 'conversation', code: 'EVENT_LOG_CORRUPTED' }] }) - expect(() => reopened.assertRunExecutable('root')).toThrow(RunEventCorruptionError) + expect(() => reopened.assertRunExecutable('root')).not.toThrow() expect(f.database.prepare('SELECT * FROM run_events WHERE run_id = ?').all('child')).toEqual(before) await expect(reopened.append({ runId: 'healthy-run', type: 'audit.healthy', payload: {} })).resolves.toMatchObject({ sequence: 2 }) } @@ -190,19 +203,21 @@ describe('run event quarantine', () => { }, usage: { record: async () => { throw new Error('Unexpected compaction') } }, }) - expect(await recovery.recoverInterruptedRuns()).toBe(1) - expect(captured).toEqual(['healthy-run']) - expect(cancellable).toEqual(['healthy-run']) + expect(await recovery.recoverInterruptedRuns()).toBe(3) + expect(captured.toSorted()).toEqual(['healthy-run', 'root']) + expect(cancellable).toEqual(['root', 'healthy-run']) expect(f.runs.findById('healthy-run')).toMatchObject({ status: 'failed', errorCode: 'RUNTIME_RESTARTED' }) - expect(f.runs.findById('root')?.status).toBe('running') - expect(f.runs.findById('child')?.status).toBe('queued') - expect(await readFile(f.file('root'), 'utf8')).not.toContain('run.failed') + expect(f.runs.findById('root')).toMatchObject({ status: 'failed', errorCode: 'RUNTIME_RESTARTED' }) + expect(f.runs.findById('child')).toMatchObject({ status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' }) + expect(f.executions.find('root')).toMatchObject({ phase: 'closed', cleanupState: 'degraded', cleanupErrorCode: 'EVENT_LOG_CORRUPTED' }) + expect(f.executions.find('child')).toMatchObject({ phase: 'closed', cleanupState: 'degraded' }) + expect(await readFile(f.file('root'), 'utf8')).toContain('run.failed') expect(await readFile(f.file('child'), 'utf8')).toBe(damaged) expect(await recovery.recoverInterruptedRuns()).toBe(0) await lifecycle.dispose() }) - it('preserves quarantined approvals while durably cancelling healthy approvals during startup', async () => { + it('cancels stale approvals without inventing events in a quarantined log', async () => { const f = await fixture() const repository = createApprovalRepository(f.database) const approvals = new ApprovalService({ eventLog: f.events, repository }) @@ -231,13 +246,13 @@ describe('run event quarantine', () => { cancelPendingApprovals: canRecover => approvals.cancelPendingApprovals('recovery', canRecover), usage: { record: async () => { throw new Error('Unexpected compaction') } }, }) - expect(await recovery.recoverInterruptedRuns()).toBe(1) - expect(repository.findById('approval-child')?.status).toBe('pending') + expect(await recovery.recoverInterruptedRuns()).toBe(3) + expect(repository.findById('approval-child')?.status).toBe('cancelled') expect(repository.findById('approval-healthy-run')?.status).toBe('cancelled') expect(await readFile(f.file('child'), 'utf8')).toBe(damaged) await lifecycle.dispose() await expect(approvals.dispose()).resolves.toBeUndefined() - expect(repository.findById('approval-child')?.status).toBe('pending') + expect(repository.findById('approval-child')?.status).toBe('cancelled') expect(await readFile(f.file('child'), 'utf8')).toBe(damaged) }) @@ -286,7 +301,7 @@ describe('run event quarantine', () => { await mutations.dispose() }) - it('rejects child and launcher admission before projection or model side effects while still allowing cleanup', async () => { + it('settles a quarantined launch without planning, model calls or invented events', async () => { const f = await fixture() const mutations = new MutationCoordinator() const lifecycle = new RunLifecycleService({ eventLog: f.events, repository: f.runs }) @@ -305,10 +320,11 @@ describe('run event quarantine', () => { return false }, listIncomplete: () => f.runs.listIncomplete(), + recoverQuarantinedRun: runId => f.runs.reconcileQuarantined(runId, executionNow), }) await appendFile(f.file('child'), '{broken}\n') await f.events.recoverAll() - const before = f.executions.listTree('root') + const original = await readFile(f.file('child')) const launcher = new BuddyTurnLauncher({ executions: coordinator, planner: { resolve: async () => { throw new Error('Unexpected planning') } }, @@ -317,17 +333,12 @@ describe('run event quarantine', () => { startCompaction: () => { throw new Error('Unexpected model') }, }, }) - await expect(launcher.launch('child')).rejects.toBeInstanceOf(RunEventCorruptionError) - await expect(coordinator.admitChild(f.child('another-child'))).rejects.toBeInstanceOf(RunEventCorruptionError) - await expect(coordinator.begin('root', new AbortController().signal)).rejects.toBeInstanceOf(RunEventCorruptionError) - expect(f.executions.listTree('root')).toEqual(before) - expect(f.runs.findById('another-child')).toBeNull() - await coordinator.cancel('root', 'RUN_CANCELLED', async (runId) => { - stopped.push(runId) - return false - }) - expect(stopped).toEqual(['root', 'child']) - expect(f.executions.listTree('root')).toEqual(before) + const launched = await launcher.launch('child') + await expect(launched.completion).resolves.toMatchObject({ status: 'failed', errorCode: 'EVENT_LOG_CORRUPTED' }) + await expect(f.events.read('root')).resolves.toHaveLength(1) + expect(f.executions.find('child')).toMatchObject({ phase: 'closed', cleanupState: 'degraded' }) + expect(f.executions.find('root')?.cleanupState).toBe('degraded') + expect(await readFile(f.file('child'))).toEqual(original) await coordinator.dispose() await lifecycle.dispose() await mutations.dispose() diff --git a/apps/buddy/service/src/events/__tests__/RunEventRecovery.spec.ts b/apps/buddy/service/src/events/__tests__/RunEventRecovery.spec.ts index ca705aea9..4068db2cd 100644 --- a/apps/buddy/service/src/events/__tests__/RunEventRecovery.spec.ts +++ b/apps/buddy/service/src/events/__tests__/RunEventRecovery.spec.ts @@ -264,15 +264,16 @@ describe('startup run event recovery', () => { expect(f.database.prepare('SELECT content_json FROM messages WHERE id = \'answer-1\'').get()).toEqual({ content_json: '{"text":"edited"}' }) }) - it('repairs a torn tail but does not certify the changed file until a subsequent clean read', async () => { + it('quarantines a torn tail without changing evidence across repeated recovery', async () => { const f = await fixture() await seedHistory(f.log) await f.log.recoverAll() await appendFile(f.eventFile, '{"runId":"run-1"') - expect(await f.log.recoverAll()).toMatchObject({ rebuilt: 1, skipped: 0 }) + const damaged = await readFile(f.eventFile, 'utf8') + expect(await f.log.recoverAll()).toMatchObject({ rebuilt: 0, skipped: 0, failed: 1 }) expect(checkpoint(f.database)).toBeUndefined() - expect(await f.log.recoverAll()).toMatchObject({ rebuilt: 1 }) - expect(await f.log.recoverAll()).toMatchObject({ skipped: 1 }) + expect(await f.log.recoverAll()).toMatchObject({ rebuilt: 0, failed: 1 }) + expect(await readFile(f.eventFile, 'utf8')).toBe(damaged) }) it.each([ @@ -305,7 +306,8 @@ describe('startup run event recovery', () => { await rm(f.eventFile) await writeFile(join(f.conversationsDirectory, 'conversation-1', 'events', 'unknown-run.jsonl'), '{broken}\n') expect(await f.log.recoverAll()).toMatchObject({ scanned: 0, skipped: 0, rebuilt: 0, failed: 1 }) - expect(() => f.log.assertConversationWritable('conversation-1')).toThrow(expect.objectContaining({ code: 'EVENT_LOG_CORRUPTED' })) + expect(() => f.log.assertConversationWritable('conversation-1')).not.toThrow() + expect(() => f.log.assertRunWritable('run-1')).toThrow(expect.objectContaining({ code: 'EVENT_LOG_CORRUPTED' })) expect(checkpoint(f.database)).toBeUndefined() expect(projection(f.database)).toEqual(before) expect(f.database.prepare('SELECT id FROM runs').all()).toEqual([{ id: 'run-1' }]) diff --git a/apps/buddy/service/src/files/NativeFileEngine.ts b/apps/buddy/service/src/files/NativeFileEngine.ts index 5eb3792b9..0416404ea 100644 --- a/apps/buddy/service/src/files/NativeFileEngine.ts +++ b/apps/buddy/service/src/files/NativeFileEngine.ts @@ -29,133 +29,365 @@ export interface NativeFileClaim { root: string, path: string, subtree: boolean export interface NativeFileProtection { path: string, subtree: boolean } export interface NativeFileMutation { operationId: string, root: string, path: string, leaseId?: string, action: FileAction } const nativeDocumentSchema = z.object({ text: z.string(), etag: z.string().length(64), identity: z.string() }) +const nativeRootSchema = z.object({ identity: z.string(), path: z.string() }) const notificationSchema = z.discriminatedUnion('event', [ z.object({ event: z.literal('changed'), roots: z.array(z.string()) }), z.object({ event: z.literal('operation'), operation: nativeOperationSchema }), z.object({ event: z.literal('progress'), operationId: z.string(), bytes: z.number() }), ]) +const startupTimeout = 10_000 +const shutdownTimeout = 5_000 +const terminationTimeout = 5_000 + +interface FileEngineConnection { + child: ChildProcessWithoutNullStreams + ready: PromiseWithResolvers + closed: PromiseWithResolvers + pending: Map void, reject: (reason: Error) => void }> + buffer: Buffer + failed: boolean + exited: boolean + startupTimer?: ReturnType +} + +interface RegisteredRoot { + identity: string + path: string + watch: boolean + connection: FileEngineConnection +} class FileEngineError extends Error { readonly code: string + readonly responseReceived: boolean - constructor(code: string) { - super(code) + constructor(code: string, responseReceived = false) { + super(code === 'FILE_ENGINE_INTERRUPTED' ? `${code}: 文件操作可能已产生副作用，请重新读取后发起新操作。` : code) this.code = code + this.responseReceived = responseReceived this.name = 'FileEngineError' } } +async function within(promise: Promise, milliseconds: number, code: string): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new FileEngineError(code)), milliseconds) + }), + ]) + } + finally { clearTimeout(timer) } +} + export class NativeFileEngine { - readonly #child: ChildProcessWithoutNullStreams - readonly #ready = Promise.withResolvers() - readonly #closed = Promise.withResolvers() - readonly #pending = new Map void, reject: (reason: Error) => void }>() + readonly #directory: string + readonly #executable: string + readonly #roots = new Map() + readonly #revoked = new Set() + readonly #rootQueues = new Map>() + readonly #leases = new Map() readonly #events = new Emitter>(() => console.error('FILE_ENGINE_OBSERVER_FAILED')) readonly onDidChange = this.#events.event + #connection: FileEngineConnection + #starting?: Promise + #disposing?: Promise #sequence = 0 - #buffer = Buffer.alloc(0) #stopping = false - #failed = false constructor(directory: string, executable = process.env.LEXORA_BUDDY_FILE_READER) { if (!executable) throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') - const env = Object.fromEntries(['SystemRoot', 'HOME', 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'XDG_DATA_HOME', 'XDG_RUNTIME_DIR', 'TMPDIR', 'TEMP', 'TMP'].flatMap(key => process.env[key] ? [[key, process.env[key]!]] : [])) - this.#child = spawn(executable, ['--serve', directory], { env, windowsHide: true, stdio: 'pipe' }) - this.#child.stdout.on('data', (chunk: Buffer) => this.#receive(chunk)) - this.#child.stderr.resume() - this.#child.stdin.on('error', () => this.#fail()) - this.#child.on('error', () => this.#fail()) - this.#child.on('close', () => { - this.#fail() - this.#closed.resolve() - }) - void this.#ready.promise.catch(() => {}) + this.#directory = directory + this.#executable = executable + this.#connection = this.#spawn() } - ready(): Promise { return this.#ready.promise } + async ready(): Promise { await this.#connect() } - register(key: string, path: string, watch = false) { - return this.#request({ type: 'register', key, path, watch }, z.object({ identity: z.string(), path: z.string() })) + async register(key: string, path: string, watch = false) { + const connection = await this.#connect() + return this.#rootQueue(key, () => this.#register(connection, key, path, watch)) + } + + async revoke(key: string) { + this.#revoked.add(key) + this.#roots.delete(key) + const connection = this.#connection + if (connection.failed) { + await this.#waitForExit(connection) + return null + } + return this.#send(connection, { type: 'revoke', key }, z.null()) + } + + list(root: string, path: string, cursor?: string) { return this.#request({ type: 'list', root, path, cursor }, directoryPageSchema, [root]) } + read(root: string, path: string) { return this.#request({ type: 'read', root, path }, nativeDocumentSchema, [root]) } + mutate(request: NativeFileMutation) { + const roots = [request.root] + if (request.action.kind === 'copy' || request.action.kind === 'move') + roots.push(request.action.destinationRoot) + return this.#request({ type: 'mutate', request }, nativeOperationSchema, roots) } - revoke(key: string) { return this.#request({ type: 'revoke', key }, z.null()) } - list(root: string, path: string, cursor?: string) { return this.#request({ type: 'list', root, path, cursor }, directoryPageSchema) } - read(root: string, path: string) { return this.#request({ type: 'read', root, path }, nativeDocumentSchema) } - mutate(request: NativeFileMutation) { return this.#request({ type: 'mutate', request }, nativeOperationSchema) } query(operationId: string) { return this.#request({ type: 'query', operationId }, nativeOperationSchema.nullable()) } - history(root: string) { return this.#request({ type: 'history', root }, z.array(nativeOperationSchema)) } + history(root: string) { return this.#request({ type: 'history', root }, z.array(nativeOperationSchema), [root]) } protect(leaseId: string, claims: NativeFileProtection[]) { return this.#request({ type: 'protect', leaseId, claims }, z.null()) } - acknowledge(root: string, operationId: string) { return this.#request({ type: 'acknowledge', root, operationId }, nativeOperationSchema) } - cancel(operationId: string) { return this.#request({ type: 'cancel', operationId }, z.null()) } + acknowledge(root: string, operationId: string) { return this.#request({ type: 'acknowledge', root, operationId }, nativeOperationSchema, [root]) } + cancel(operationId: string) { return this.#send(this.#connection, { type: 'cancel', operationId }, z.null()) } leases() { return this.#request({ type: 'leases' }, z.array(z.string())) } - release(leaseId: string, hazard = false) { return this.#request({ type: 'release', leaseId, hazard }, z.null()) } + + async release(leaseId: string, hazard = false) { + const owner = this.#leases.get(leaseId) + const connection = owner ?? this.#connection + const relinquish = () => { + if (this.#leases.get(leaseId) === owner) + this.#leases.delete(leaseId) + } + try { + await this.#send(connection, { type: 'release', leaseId, hazard }, z.null(), Boolean(owner)) + } + catch (error) { + if (!connection.failed) { + if (error instanceof FileEngineError && error.responseReceived) + relinquish() + throw error + } + void connection.closed.promise.then(relinquish) + await this.#waitForExit(connection) + } + relinquish() + return null + } async acquire(leaseId: string, claims: NativeFileClaim[], external = false, signal?: AbortSignal): Promise { signal?.throwIfAborted() + const connection = await this.#connect() + await this.#restoreRoots(connection, claims.map(claim => claim.root).filter(root => root !== '*')) + signal?.throwIfAborted() + if (this.#stopping) + throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') + if (this.#leases.has(leaseId)) + throw new FileEngineError('FILE_LEASE_INVALID') + this.#leases.set(leaseId, connection) const cancel = () => { void this.release(leaseId).catch(() => {}) } - const acquired = this.#request({ type: 'acquire', leaseId, claims, external }, z.null()) signal?.addEventListener('abort', cancel, { once: true }) try { - await acquired + await this.#send(connection, { type: 'acquire', leaseId, claims, external }, z.null()) if (signal?.aborted) { await this.release(leaseId) signal.throwIfAborted() } + if (this.#leases.get(leaseId) !== connection) + throw new FileEngineError('FILE_OPERATION_CANCELLED') + } + catch (error) { + if (this.#leases.get(leaseId) === connection) + this.#leases.delete(leaseId) + throw error } finally { signal?.removeEventListener('abort', cancel) } } - async dispose(): Promise { - if (this.#stopping) - return this.#closed.promise + dispose(): Promise { this.#stopping = true - if (!this.#failed) { - while (this.#pending.size) - await new Promise(resolve => setTimeout(resolve, 10)) - await this.#request({ type: 'shutdown' }, z.null(), true).catch(() => {}) - this.#child.stdin.end() + this.#disposing ??= this.#dispose().catch((error) => { + this.#disposing = undefined + throw error + }) + return this.#disposing + } + + async #dispose(): Promise { + const connection = this.#connection + try { + const shutdown = async () => { + if (!connection.failed) { + await this.#send(connection, { type: 'shutdown' }, z.null()) + connection.child.stdin.end() + } + await connection.closed.promise + } + try { + await within(shutdown(), shutdownTimeout, 'FILE_ENGINE_STOP_TIMEOUT') + } + catch { + this.#fail(connection) + await this.#waitForExit(connection) + } } - await this.#closed.promise - this.#events.dispose() + finally { this.#events.dispose() } } - async #request(command: object, schema: ZodType, shutdown = false): Promise { - if (this.#failed || (this.#stopping && !shutdown)) + #spawn(): FileEngineConnection { + const env = Object.fromEntries(['SystemRoot', 'HOME', 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'XDG_DATA_HOME', 'XDG_RUNTIME_DIR', 'TMPDIR', 'TEMP', 'TMP'].flatMap(key => process.env[key] ? [[key, process.env[key]!]] : [])) + const child = spawn(this.#executable, ['--serve', this.#directory], { env, windowsHide: true, stdio: 'pipe' }) + const connection: FileEngineConnection = { + child, + ready: Promise.withResolvers(), + closed: Promise.withResolvers(), + pending: new Map(), + buffer: Buffer.alloc(0), + failed: false, + exited: false, + } + connection.startupTimer = setTimeout(() => this.#fail(connection), startupTimeout) + child.stdout.on('data', (chunk: Buffer) => this.#receive(connection, chunk)) + child.stdout.on('error', () => this.#fail(connection)) + child.stdout.on('end', () => this.#fail(connection)) + child.stderr.resume() + child.stdin.on('error', () => this.#fail(connection)) + child.on('error', () => this.#fail(connection)) + child.on('close', () => { + connection.exited = true + this.#fail(connection) + const error = new FileEngineError('FILE_ENGINE_INTERRUPTED') + for (const pending of connection.pending.values()) + pending.reject(error) + connection.pending.clear() + connection.buffer = Buffer.alloc(0) + connection.closed.resolve() + }) + void connection.ready.promise.catch(() => {}) + return connection + } + + async #connect(): Promise { + if (this.#stopping) + throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') + if (this.#starting) + return this.#starting + const connection = this.#connection + if (!connection.failed) { + await connection.ready.promise + if (connection.failed) + throw new FileEngineError('FILE_ENGINE_INTERRUPTED') + if (this.#stopping) + throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') + return connection + } + this.#starting = this.#restart(connection) + try { + return await this.#starting + } + finally { this.#starting = undefined } + } + + async #restart(previous: FileEngineConnection): Promise { + await this.#waitForExit(previous) + if (this.#stopping) + throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') + if (this.#leases.size) + throw new FileEngineError('FILE_ENGINE_ACTIVE_LEASES') + const connection = this.#spawn() + this.#connection = connection + await connection.ready.promise + if (connection.failed) + throw new FileEngineError('FILE_ENGINE_INTERRUPTED') + const roots = [...this.#roots].filter(([key, root]) => root.watch && !this.#revoked.has(key)).map(([key]) => key) + if (roots.length) + this.#events.fire({ event: 'changed', roots }) + return connection + } + + #waitForExit(connection: FileEngineConnection): Promise { + return within(connection.closed.promise, terminationTimeout, 'FILE_ENGINE_STOP_TIMEOUT') + } + + #rootQueue(key: string, action: () => Promise): Promise { + const result = (this.#rootQueues.get(key) ?? Promise.resolve()).catch(() => {}).then(action) + this.#rootQueues.set(key, result) + void result.finally(() => { + if (this.#rootQueues.get(key) === result) + this.#rootQueues.delete(key) + }).catch(() => {}) + return result + } + + async #register(connection: FileEngineConnection, key: string, path: string, watch: boolean) { + if (this.#stopping) throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') - await this.#ready.promise - if (this.#failed) + if (this.#revoked.has(key)) + throw new FileEngineError('FILE_PERMISSION_REVOKED') + const previous = this.#roots.get(key) + const result = await this.#send(connection, { type: 'register', key, path, watch: watch || previous?.watch === true }, nativeRootSchema) + if (this.#revoked.has(key) || (previous && (previous.identity !== result.identity || previous.path !== result.path))) { + this.#revoked.add(key) + await this.#send(connection, { type: 'revoke', key }, z.null()) + throw new FileEngineError('FILE_PERMISSION_REVOKED') + } + this.#roots.set(key, { ...result, watch: watch || previous?.watch === true, connection }) + return result + } + + async #restoreRoots(connection: FileEngineConnection, roots: string[]): Promise { + await Promise.all([...new Set(roots)].map(key => this.#rootQueue(key, async () => { + if (this.#revoked.has(key)) + throw new FileEngineError('FILE_PERMISSION_REVOKED') + const root = this.#roots.get(key) + if (root && root.connection !== connection) + await this.#register(connection, key, root.path, root.watch) + }))) + } + + async #request(command: object, schema: ZodType, roots: string[] = []): Promise { + const connection = await this.#connect() + await this.#restoreRoots(connection, roots) + if (this.#stopping) throw new FileEngineError('FILE_ENGINE_UNAVAILABLE') - if (this.#pending.size >= 128) + return this.#send(connection, command, schema) + } + + async #send(connection: FileEngineConnection, command: object, schema: ZodType, releaseOwner = false): Promise { + if (connection.failed) + throw new FileEngineError('FILE_ENGINE_INTERRUPTED') + await connection.ready.promise + if (connection.failed) + throw new FileEngineError('FILE_ENGINE_INTERRUPTED') + if (connection.pending.size >= 128 && !releaseOwner) throw new FileEngineError('FILE_ENGINE_BUSY') const id = ++this.#sequence const frame = `${JSON.stringify({ id, command })}\n` if (Buffer.byteLength(frame) > 128 * 1024 * 1024) throw new FileEngineError('FILE_INVALID_REQUEST') const value = await new Promise((resolve, reject) => { - this.#pending.set(id, { resolve, reject }) - this.#child.stdin.write(frame, error => error && this.#fail()) + connection.pending.set(id, { resolve, reject }) + try { + connection.child.stdin.write(frame, error => error && this.#fail(connection)) + } + catch { this.#fail(connection) } }) - return schema.parse(value) + try { + return schema.parse(value) + } + catch { + this.#fail(connection) + await connection.closed.promise + throw new FileEngineError('FILE_ENGINE_INTERRUPTED') + } } - #receive(chunk: Buffer): void { - this.#buffer = Buffer.concat([this.#buffer, chunk]) - if (this.#buffer.length > 128 * 1024 * 1024) { - this.#fail() - this.#child.kill() + #receive(connection: FileEngineConnection, chunk: Buffer): void { + if (connection.failed || connection !== this.#connection) + return + connection.buffer = Buffer.concat([connection.buffer, chunk]) + if (connection.buffer.length > 128 * 1024 * 1024) { + this.#fail(connection) return } - for (let end = this.#buffer.indexOf(10); end !== -1; end = this.#buffer.indexOf(10)) { - const frame = this.#buffer.subarray(0, end) - this.#buffer = this.#buffer.subarray(end + 1) + for (let end = connection.buffer.indexOf(10); end !== -1; end = connection.buffer.indexOf(10)) { + const frame = connection.buffer.subarray(0, end) + connection.buffer = connection.buffer.subarray(end + 1) try { const value: unknown = JSON.parse(frame.toString('utf8')) const ready = z.object({ event: z.literal('ready'), version: z.literal(2) }).safeParse(value) if (ready.success) { - this.#ready.resolve() + clearTimeout(connection.startupTimer) + connection.ready.resolve() continue } const notification = notificationSchema.safeParse(value) @@ -164,28 +396,28 @@ export class NativeFileEngine { continue } const response = z.object({ id: z.number().int(), result: z.unknown().optional(), error: z.string().optional() }).parse(value) - const pending = this.#pending.get(response.id) + const pending = connection.pending.get(response.id) if (!pending) throw new FileEngineError('FILE_PROTOCOL_FAILED') - this.#pending.delete(response.id) + connection.pending.delete(response.id) if (response.error) - pending.reject(new FileEngineError(response.error)) + pending.reject(new FileEngineError(response.error, true)) else pending.resolve(response.result) } catch { - this.#fail() - this.#child.kill() + this.#fail(connection) return } } } - #fail(): void { - this.#failed = true - const error = new FileEngineError('FILE_ENGINE_INTERRUPTED') - this.#ready.reject(error) - for (const pending of this.#pending.values()) - pending.reject(error) - this.#pending.clear() + #fail(connection: FileEngineConnection): void { + if (connection.failed) + return + connection.failed = true + clearTimeout(connection.startupTimer) + connection.ready.reject(new FileEngineError('FILE_ENGINE_INTERRUPTED')) + if (!connection.exited) + connection.child.kill('SIGKILL') } } diff --git a/apps/buddy/service/src/files/__tests__/FileResourceCoordinator.spec.ts b/apps/buddy/service/src/files/__tests__/FileResourceCoordinator.spec.ts index 8c91ac8aa..45d596fc5 100644 --- a/apps/buddy/service/src/files/__tests__/FileResourceCoordinator.spec.ts +++ b/apps/buddy/service/src/files/__tests__/FileResourceCoordinator.spec.ts @@ -51,8 +51,17 @@ it('reconciles overlapping and offline hazards without acquiring write rights, i const restarted = new NativeFileEngine(journal) cleanup.push(() => restarted.dispose()) await restarted.ready() - await new FileResourceCoordinator(restarted).reconcile(hazards, () => false) + const restoredResources = new FileResourceCoordinator(restarted) + await restoredResources.reconcile(hazards, () => false) expect(new Set(await restarted.leases())).toEqual(new Set([orphanId, invocationId, missingId, emptyId])) expect((await restarted.query(record.operationId))?.status).toBe('needs-recovery') expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('original') + await expect(restoredResources.write(invocationId, join(workspace, 'note.txt'), 'not-authorized')).rejects.toThrow('FILE_LEASE_INVALID') + const next = randomUUID() + await restoredResources.acquire(next, hazards[0]!.claims, 'runtime') + await restoredResources.write(next, join(workspace, 'note.txt'), 'new-operation') + await restoredResources.release(next) + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('new-operation') + expect((await restarted.query(record.operationId))?.status).toBe('needs-recovery') + expect(new Set(await restarted.leases())).toEqual(new Set([orphanId, invocationId, missingId, emptyId])) }) diff --git a/apps/buddy/service/src/files/__tests__/NativeFileEngine.spec.ts b/apps/buddy/service/src/files/__tests__/NativeFileEngine.spec.ts new file mode 100644 index 000000000..42f373e04 --- /dev/null +++ b/apps/buddy/service/src/files/__tests__/NativeFileEngine.spec.ts @@ -0,0 +1,505 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' +import { Buffer } from 'node:buffer' +import { spawn } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { EventEmitter, once } from 'node:events' +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { PassThrough } from 'node:stream' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { NativeFileEngine } from '../NativeFileEngine' + +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, spawn: vi.fn(actual.spawn) } +}) + +const cleanup: Array<() => Promise> = [] +afterEach(async () => { + vi.useRealTimers() + for (const dispose of cleanup.splice(0).reverse()) + await dispose() +}) + +async function temporaryDirectory() { + const directory = await mkdtemp(join(tmpdir(), 'native-file-engine-')) + cleanup.push(() => rm(directory, { recursive: true, force: true })) + return directory +} + +function latestHelper() { + const result = vi.mocked(spawn).mock.results.at(-1) + if (result?.type !== 'return') + throw new Error('Missing owned helper') + return result.value as ChildProcessWithoutNullStreams +} + +async function killHelper(child: ChildProcessWithoutNullStreams) { + const closed = once(child, 'close') + expect(child.kill('SIGKILL')).toBe(true) + await closed + expect(child.signalCode).toBe('SIGKILL') +} + +async function nativeEngine() { + const directory = await temporaryDirectory() + const workspace = join(directory, 'workspace') + await mkdir(workspace) + await writeFile(join(workspace, 'note.txt'), 'original') + const engine = new NativeFileEngine(join(directory, 'journal')) + cleanup.push(() => engine.dispose()) + await engine.ready() + await engine.register('workspace', workspace) + return { directory, workspace, engine, child: latestHelper() } +} + +function sentCommands(child: ChildProcessWithoutNullStreams) { + const commands: Array<{ type: string, leaseId?: string, request?: { operationId: string } }> = [] + const write = child.stdin.write.bind(child.stdin) + vi.spyOn(child.stdin, 'write').mockImplementation(((...args: Parameters) => { + commands.push(JSON.parse(String(args[0])).command) + return write(...args) + }) as typeof write) + return commands +} + +describe('real native helper recovery', () => { + it('restarts once after kill + close and supports concurrent reads and a new write', async () => { + const { engine, child, workspace } = await nativeEngine() + const before = vi.mocked(spawn).mock.calls.length + await killHelper(child) + const documents = await Promise.all(Array.from({ length: 24 }, () => engine.read('workspace', 'note.txt'))) + expect(documents.every(document => document.text === 'original')).toBe(true) + expect(vi.mocked(spawn).mock.calls.length).toBe(before + 1) + expect(latestHelper().pid).not.toBe(child.pid) + const record = await engine.mutate({ operationId: randomUUID(), root: 'workspace', path: 'note.txt', action: { kind: 'write', text: 'new request' } }) + expect(record.status).toBe('applied') + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('new request') + }) + + it('invalidates watched roots after reconnect and restores watching when refreshed', async () => { + const { engine, child, workspace } = await nativeEngine() + await engine.register('workspace', workspace, true) + const changed: string[][] = [] + engine.onDidChange((event) => { + if (event.event === 'changed') + changed.push(event.roots) + }) + await killHelper(child) + await engine.query(randomUUID()) + expect(changed).toContainEqual(['workspace']) + await engine.read('workspace', 'note.txt') + changed.length = 0 + await writeFile(join(workspace, 'note.txt'), 'external change') + await vi.waitFor(() => expect(changed).toContainEqual(['workspace'])) + expect((await engine.read('workspace', 'note.txt')).text).toBe('external change') + }) + + it('preserves an applied write whose response was lost, without replaying it on a later read', async () => { + const { engine, child, workspace } = await nativeEngine() + const operationId = randomUUID() + const applied = Promise.withResolvers() + const receivers = child.stdout.listeners('data') + child.stdout.removeAllListeners('data') + let buffer = '' + child.stdout.on('data', (chunk: Buffer) => { + buffer += chunk.toString() + for (let end = buffer.indexOf('\n'); end !== -1; end = buffer.indexOf('\n')) { + const frame = buffer.slice(0, end + 1) + buffer = buffer.slice(end + 1) + const response = JSON.parse(frame) + if (response.id && response.result?.operationId === operationId) { + expect(response.result.status).toBe('applied') + applied.resolve() + } + else { + for (const receive of receivers) + receive.call(child.stdout, Buffer.from(frame)) + } + } + }) + const interrupted = engine.mutate({ operationId, root: 'workspace', path: 'note.txt', action: { kind: 'write', text: 'applied before disconnect' } }).catch(error => error) + await applied.promise + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('applied before disconnect') + await killHelper(child) + expect(await interrupted).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + await writeFile(join(workspace, 'note.txt'), 'subsequent external edit') + expect((await engine.read('workspace', 'note.txt')).text).toBe('subsequent external edit') + expect(await engine.query(operationId)).toMatchObject({ operationId, status: 'applied' }) + expect((await engine.history('workspace')).filter(record => record.operationId === operationId)).toHaveLength(1) + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('subsequent external edit') + }) + + it('fails queued mutation/acquire without replay and retains the existing lease until its owner releases', async () => { + const { engine, child, workspace } = await nativeEngine() + const owner = randomUUID() + const waiting = randomUUID() + const operationId = randomUUID() + const claims = [{ root: 'workspace', path: 'note.txt', subtree: false }] + await engine.acquire(owner, claims) + const commands = sentCommands(child) + const mutation = engine.mutate({ operationId, root: 'workspace', path: 'note.txt', action: { kind: 'write', text: 'must not replay' } }).catch(error => error) + const acquisition = engine.acquire(waiting, claims, true).catch(error => error) + await vi.waitFor(() => { + expect(commands.some(command => command.request?.operationId === operationId)).toBe(true) + expect(commands.some(command => command.type === 'acquire' && command.leaseId === waiting)).toBe(true) + }) + expect(await engine.query(operationId)).toBeNull() + await killHelper(child) + expect(await mutation).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + expect(await acquisition).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + await expect(engine.read('workspace', 'note.txt')).rejects.toThrow('FILE_ENGINE_ACTIVE_LEASES') + await engine.release(owner) + expect((await engine.read('workspace', 'note.txt')).text).toBe('original') + expect(await engine.query(operationId)).toBeNull() + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('original') + expect((await engine.mutate({ operationId: randomUUID(), root: 'workspace', path: 'note.txt', action: { kind: 'write', text: 'fresh' } })).status).toBe('applied') + }) + + it('cancels a blocked acquire on its original connection without losing the existing owner', async () => { + const { engine, child } = await nativeEngine() + const owner = randomUUID() + const waiting = randomUUID() + const claims = [{ root: 'workspace', path: '', subtree: true }] + await engine.acquire(owner, claims) + const commands = sentCommands(child) + const controller = new AbortController() + const acquisition = engine.acquire(waiting, claims, true, controller.signal).catch(error => error) + await vi.waitFor(() => expect(commands.some(command => command.type === 'acquire' && command.leaseId === waiting)).toBe(true)) + controller.abort() + expect(await acquisition).toMatchObject({ code: 'FILE_OPERATION_CANCELLED' }) + await engine.release(owner) + const next = randomUUID() + await engine.acquire(next, claims) + await engine.release(next) + expect((await engine.read('workspace', 'note.txt')).text).toBe('original') + }) + + it('does not turn a still-running external owner into a historical lease after helper death', async () => { + const { engine, child } = await nativeEngine() + const owner = randomUUID() + await engine.acquire(owner, [{ root: 'workspace', path: '', subtree: true }], true) + const external = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'pipe' }) + cleanup.push(async () => { + if (external.exitCode === null && external.signalCode === null) + await killHelper(external) + await engine.release(owner) + }) + await once(external, 'spawn') + await killHelper(child) + await expect(engine.ready()).rejects.toThrow('FILE_ENGINE_ACTIVE_LEASES') + expect(external.exitCode).toBeNull() + expect(external.signalCode).toBeNull() + await killHelper(external) + await engine.release(owner) + expect((await engine.read('workspace', 'note.txt')).text).toBe('original') + const next = randomUUID() + await engine.acquire(next, [{ root: 'workspace', path: '', subtree: true }], true) + await engine.release(next) + }) + + it('rejects a replaced root including concurrent access without poisoning another root', async () => { + const { directory, engine, child, workspace } = await nativeEngine() + const healthy = join(directory, 'healthy') + await mkdir(healthy) + await writeFile(join(healthy, 'note.txt'), 'healthy') + await engine.register('healthy', healthy) + await killHelper(child) + await rename(workspace, join(directory, 'previous-workspace')) + await mkdir(workspace) + await writeFile(join(workspace, 'note.txt'), 'replacement') + const results = await Promise.allSettled([ + engine.read('workspace', 'note.txt'), + engine.read('workspace', 'note.txt'), + engine.read('healthy', 'note.txt'), + ]) + expect(results[0]).toMatchObject({ status: 'rejected', reason: { code: 'FILE_PERMISSION_REVOKED' } }) + expect(results[1]).toMatchObject({ status: 'rejected', reason: { code: 'FILE_PERMISSION_REVOKED' } }) + expect(results[2]).toMatchObject({ status: 'fulfilled', value: { text: 'healthy' } }) + await expect(engine.register('workspace', workspace)).rejects.toThrow('FILE_PERMISSION_REVOKED') + await expect(engine.mutate({ operationId: randomUUID(), root: 'healthy', path: 'note.txt', action: { kind: 'copy', destinationRoot: 'workspace', destination: 'copied.txt' } })).rejects.toThrow('FILE_PERMISSION_REVOKED') + expect(await readFile(join(workspace, 'note.txt'), 'utf8')).toBe('replacement') + expect((await engine.mutate({ operationId: randomUUID(), root: 'healthy', path: 'note.txt', action: { kind: 'write', text: 'still healthy' } })).status).toBe('applied') + }) + + it('keeps an offline root isolated and rechecks it when the original directory returns', async () => { + const { directory, engine, child, workspace } = await nativeEngine() + const healthy = join(directory, 'healthy') + await mkdir(healthy) + await writeFile(join(healthy, 'note.txt'), 'healthy') + await engine.register('healthy', healthy) + const offline = join(directory, 'offline') + await killHelper(child) + await rename(workspace, offline) + expect((await engine.read('healthy', 'note.txt')).text).toBe('healthy') + await expect(engine.read('workspace', 'note.txt')).rejects.toThrow('FILE_IO_FAILED') + await rename(offline, workspace) + expect((await engine.read('workspace', 'note.txt')).text).toBe('original') + }) + + it('does not restore explicitly revoked authority after restart', async () => { + const { engine, child, workspace } = await nativeEngine() + await engine.revoke('workspace') + await killHelper(child) + await expect(engine.read('workspace', 'note.txt')).rejects.toThrow('FILE_PERMISSION_REVOKED') + await expect(engine.register('workspace', workspace)).rejects.toThrow('FILE_PERMISSION_REVOKED') + await engine.register('new-authority', workspace) + expect((await engine.read('new-authority', 'note.txt')).text).toBe('original') + }) + + it.skipIf(process.platform === 'win32')('bounds disposal of a stopped real helper and confirms its forced exit', async () => { + const { engine, child } = await nativeEngine() + expect(child.kill('SIGSTOP')).toBe(true) + const commands = sentCommands(child) + const interrupted = engine.query(randomUUID()).catch(error => error) + await vi.waitFor(() => expect(commands.some(command => command.type === 'query')).toBe(true)) + await engine.dispose() + expect(child.signalCode).toBe('SIGKILL') + expect(await interrupted).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + await expect(engine.ready()).rejects.toThrow('FILE_ENGINE_UNAVAILABLE') + }) + + it('shuts down with a blocked acquire, rejects the pending request and leaves no owned helper', async () => { + const { engine, child } = await nativeEngine() + const owner = randomUUID() + const waiting = randomUUID() + const claims = [{ root: 'workspace', path: '', subtree: true }] + await engine.acquire(owner, claims) + const commands = sentCommands(child) + const acquisition = engine.acquire(waiting, claims).catch(error => error) + await vi.waitFor(() => expect(commands.some(command => command.leaseId === waiting)).toBe(true)) + await Promise.all([engine.dispose(), engine.dispose()]) + expect(await acquisition).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + expect(child.exitCode !== null || child.signalCode !== null).toBe(true) + await expect(engine.ready()).rejects.toThrow('FILE_ENGINE_UNAVAILABLE') + await engine.release(owner) + }) +}) + +// Transport-only fixture: it models protocol/exit ordering, not native disk effects or atomicity. +class TransportChild extends EventEmitter { + stdin = new PassThrough() + stdout = new PassThrough() + stderr = new PassThrough() + commands: Array<{ id: number, command: { type: string, key?: string, path?: string } }> = [] + kill = vi.fn(() => true) + closed = false + respond = true + shutdown = true + + constructor() { + super() + this.stdin.on('data', (frame: Buffer) => { + const request = JSON.parse(frame.toString()) + this.commands.push(request) + if (request.command.type === 'shutdown' && this.shutdown) { + this.reply(request.id, null) + queueMicrotask(() => this.close()) + } + else if (request.command.type === 'register') { + this.reply(request.id, { identity: 'transport-identity', path: request.command.path }) + } + else if (this.respond && request.command.type === 'read') { + this.reply(request.id, { text: 'transport-result', identity: 'transport-file', etag: '0'.repeat(64) }) + } + }) + } + + connect(ready = true) { + if (ready) + queueMicrotask(() => this.stdout.write('{"event":"ready","version":2}\n')) + return this as unknown as ChildProcessWithoutNullStreams + } + + reply(id: number, result: unknown) { + this.stdout.write(`${JSON.stringify({ id, result })}\n`) + } + + close() { + if (this.closed) + return + this.closed = true + this.emit('close') + } +} + +function transportEngine(ready = true) { + const transport = new TransportChild() + vi.mocked(spawn).mockImplementationOnce(() => transport.connect(ready)) + const engine = new NativeFileEngine('/transport-only-journal', '/transport-only-helper') + cleanup.push(async () => { + transport.close() + await engine.dispose().catch(() => {}) + }) + return { engine, transport } +} + +describe('transport-only fault ordering', () => { + it('retires an explicitly released lease after late close without retiring another live owner', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + await engine.register('root', '/transport-only-root') + const ended = randomUUID() + const active = randomUUID() + for (const leaseId of [ended, active]) { + const acquired = engine.acquire(leaseId, [{ root: 'root', path: leaseId, subtree: false }], true) + await vi.waitFor(() => expect(transport.commands.at(-1)?.command.type).toBe('acquire')) + transport.reply(transport.commands.at(-1)!.id, null) + await acquired + transport.commands.length = 0 + } + transport.stdout.write('broken protocol\n') + vi.useFakeTimers() + const release = engine.release(ended).catch(error => error) + await vi.advanceTimersByTimeAsync(5_000) + expect(await release).toMatchObject({ code: 'FILE_ENGINE_STOP_TIMEOUT' }) + transport.close() + await expect(engine.ready()).rejects.toThrow('FILE_ENGINE_ACTIVE_LEASES') + await engine.release(active) + const replacement = new TransportChild() + vi.mocked(spawn).mockImplementationOnce(() => replacement.connect()) + vi.useRealTimers() + expect((await engine.read('root', 'next')).text).toBe('transport-result') + expect(replacement.commands.map(request => request.command.type)).toEqual(['register', 'read']) + await engine.dispose() + }) + + it('allows owner cleanup through a full request queue and retains a release accounting error without a stale lease', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + await engine.register('root', '/transport-only-root') + const owner = randomUUID() + const acquired = engine.acquire(owner, [{ root: 'root', path: '', subtree: true }]) + await vi.waitFor(() => expect(transport.commands.at(-1)?.command.type).toBe('acquire')) + transport.reply(transport.commands.at(-1)!.id, null) + await acquired + transport.respond = false + const pending = Array.from({ length: 128 }, (_, index) => engine.read('root', String(index)).catch(error => error)) + await vi.waitFor(() => expect(transport.commands.filter(request => request.command.type === 'read')).toHaveLength(128)) + const release = engine.release(owner).catch(error => error) + await vi.waitFor(() => expect(transport.commands.at(-1)?.command.type).toBe('release')) + transport.stdout.write(`${JSON.stringify({ id: transport.commands.at(-1)!.id, error: 'FILE_IO_FAILED' })}\n`) + expect(await release).toMatchObject({ code: 'FILE_IO_FAILED' }) + transport.close() + expect((await Promise.all(pending)).every(error => error.code === 'FILE_ENGINE_INTERRUPTED')).toBe(true) + const replacement = new TransportChild() + vi.mocked(spawn).mockImplementationOnce(() => replacement.connect()) + expect((await engine.read('root', 'next')).text).toBe('transport-result') + await engine.dispose() + }) + + it('bounds startup without sending business requests to an unready helper', async () => { + vi.useFakeTimers() + const { engine, transport } = transportEngine(false) + const readiness = engine.ready().catch(error => error) + await vi.advanceTimersByTimeAsync(10_000) + expect(await readiness).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + expect(transport.kill).toHaveBeenCalledWith('SIGKILL') + expect(transport.commands).toEqual([]) + transport.close() + await engine.dispose() + }) + + it('does not time out a legitimate acquire wait, but terminates a malformed response before settling', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + await engine.register('root', '/transport-only-root') + let settled = false + const acquisition = engine.acquire(randomUUID(), [{ root: 'root', path: '', subtree: true }]).catch((error) => { + settled = true + return error + }) + await vi.waitFor(() => expect(transport.commands.some(request => request.command.type === 'acquire')).toBe(true)) + vi.useFakeTimers() + await vi.advanceTimersByTimeAsync(60 * 60 * 1000) + expect(transport.kill).not.toHaveBeenCalled() + expect(settled).toBe(false) + const request = transport.commands.find(request => request.command.type === 'acquire')! + transport.reply(request.id, { invalid: 'acquire result' }) + await vi.advanceTimersByTimeAsync(0) + expect(transport.kill).toHaveBeenCalledWith('SIGKILL') + expect(settled).toBe(false) + transport.close() + expect(await acquisition).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + }) + + it('does not settle interrupted I/O or restart before confirmed exit, and ignores old connection events', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + await engine.register('root', '/transport-only-root') + transport.respond = false + let settled = false + const pending = engine.read('root', 'old').catch((error) => { + settled = true + return error + }) + await vi.waitFor(() => expect(transport.commands.some(request => request.command.type === 'read')).toBe(true)) + transport.stdout.write('not json\n') + expect(transport.kill).toHaveBeenCalledWith('SIGKILL') + const before = vi.mocked(spawn).mock.calls.length + const replacement = new TransportChild() + vi.mocked(spawn).mockImplementationOnce(() => replacement.connect()) + const next = engine.read('root', 'new') + await Promise.resolve() + expect(settled).toBe(false) + expect(vi.mocked(spawn).mock.calls.length).toBe(before) + transport.close() + expect(await pending).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + expect((await next).text).toBe('transport-result') + const events: unknown[] = [] + engine.onDidChange(event => events.push(event)) + transport.stdout.write('{"event":"changed","roots":["root"]}\n') + transport.emit('error', new Error('late old error')) + transport.emit('close') + expect(events).toEqual([]) + expect(replacement.commands.map(request => request.command.type)).toEqual(['register', 'read']) + expect((await engine.read('root', 'later')).text).toBe('transport-result') + await engine.dispose() + expect(replacement.closed).toBe(true) + }) + + it('kills a helper on stdin failure but waits for actual close before rejecting the request', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + transport.respond = false + let settled = false + const pending = engine.query(randomUUID()).catch((error) => { + settled = true + return error + }) + await vi.waitFor(() => expect(transport.commands).toHaveLength(1)) + transport.stdin.emit('error', new Error('broken pipe')) + expect(transport.kill).toHaveBeenCalledWith('SIGKILL') + await Promise.resolve() + expect(settled).toBe(false) + transport.close() + expect(await pending).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + }) + + it('bounds shutdown, reports unconfirmed termination and keeps I/O pending until exit evidence', async () => { + const { engine, transport } = transportEngine() + await engine.ready() + transport.respond = false + transport.shutdown = false + let settled = false + const pending = engine.query(randomUUID()).catch((error) => { + settled = true + return error + }) + await vi.waitFor(() => expect(transport.commands).toHaveLength(1)) + vi.useFakeTimers() + const disposal = engine.dispose().catch(error => error) + await vi.advanceTimersByTimeAsync(5_000) + expect(transport.kill).toHaveBeenCalledWith('SIGKILL') + expect(settled).toBe(false) + await vi.advanceTimersByTimeAsync(5_000) + expect(await disposal).toMatchObject({ code: 'FILE_ENGINE_STOP_TIMEOUT' }) + expect(settled).toBe(false) + transport.close() + expect(await pending).toMatchObject({ code: 'FILE_ENGINE_INTERRUPTED' }) + await expect(engine.dispose()).resolves.toBeUndefined() + await expect(engine.ready()).rejects.toThrow('FILE_ENGINE_UNAVAILABLE') + }) +}) diff --git a/apps/buddy/service/src/runs/RunRecoveryService.ts b/apps/buddy/service/src/runs/RunRecoveryService.ts index 523b4a2e8..c079744eb 100644 --- a/apps/buddy/service/src/runs/RunRecoveryService.ts +++ b/apps/buddy/service/src/runs/RunRecoveryService.ts @@ -16,12 +16,12 @@ export interface RunRecoveryServiceOptions { captureInterruptedChanges?: (runId: string) => Promise conversations: Pick eventLog: Pick & Pick - & Partial> + & Partial> inspectCommittedCompaction?: ( run: RunRecord, ) => Promise lifecycle: Pick - repository: Pick + repository: Pick usage: Pick } @@ -35,9 +35,8 @@ export class RunRecoveryService { async recoverInterruptedRuns(): Promise { const interrupted = this.#options.repository.listIncomplete() - const runs = new Map(interrupted.map(run => [run.id, run])) const histories = new Map() - const blockedConversations = new Set() + const quarantined = new Set(this.#options.eventLog.getRecoveryFailures?.().map(failure => failure.runId)) for (const run of interrupted) { try { this.#options.eventLog.assertRunWritable?.(run.id) @@ -46,18 +45,20 @@ export class RunRecoveryService { catch (error) { if (!(error instanceof RunEventCorruptionError)) throw error - blockedConversations.add(run.conversationId) + quarantined.add(run.id) } } const canRecoverRun = (runId: string): boolean => { - const run = runs.get(runId) + if (quarantined.has(runId)) + return false try { this.#options.eventLog.assertRunWritable?.(runId) - return !run || !blockedConversations.has(run.conversationId) + return true } catch (error) { if (!(error instanceof RunEventCorruptionError)) throw error + quarantined.add(runId) return false } } @@ -101,12 +102,28 @@ export class RunRecoveryService { catch (error) { if (!(error instanceof RunEventCorruptionError)) throw error - blockedConversations.add(run.conversationId) + quarantined.add(run.id) } } + for (const runId of quarantined) { + if (this.recoverQuarantinedRun(runId)) + recovered++ + } return recovered } + recoverQuarantinedRun(runId: string): boolean { + try { + this.#options.eventLog.assertRunWritable?.(runId) + return false + } + catch (error) { + if (!(error instanceof RunEventCorruptionError)) + throw error + return this.#options.repository.reconcileQuarantined(runId, this.#timestamp()) + } + } + async #recoverInterruptedCompaction(run: RunRecord, events: readonly BuddyRunEvent[]): Promise { if (run.purpose !== 'conversation.compaction') return false diff --git a/apps/buddy/service/src/runs/registerRunRpc.ts b/apps/buddy/service/src/runs/registerRunRpc.ts index 561d5b1d0..346d8c51b 100644 --- a/apps/buddy/service/src/runs/registerRunRpc.ts +++ b/apps/buddy/service/src/runs/registerRunRpc.ts @@ -1,4 +1,4 @@ -import type { RunEventReader } from '../events/RunEventPorts' +import type { RunEventAvailability, RunEventReader } from '../events/RunEventPorts' import type { RuntimeRequestRegistrar } from '../rpc/runtimeRequest' import type { RunInputRepository } from '../storage/runInputRepository' import type { RunRecord } from '../storage/runRecord' @@ -15,7 +15,7 @@ import { toPublicRun } from './publicRun' export interface RegisterRunRpcOptions { isBusy: () => boolean getCacheWarmingStatus?: ConversationStatusOptions['getCacheWarmingStatus'] - eventLog: Pick + eventLog: Pick & Partial> inputs: Pick repository: Pick< RunRepository, @@ -27,7 +27,9 @@ export interface RegisterRunRpcOptions { export function registerRunRpc(options: RegisterRunRpcOptions): () => void { const publicRun = (run: RunRecord) => toPublicRun( - run, + options.eventLog.getRecoveryFailures?.().some(failure => failure.runId === run.id) + ? { ...run, errorCode: 'EVENT_LOG_CORRUPTED' } + : run, options.inputs.findByRunId(run.id)?.reasoning ?? null, ) const status = new ConversationStatusService({ diff --git a/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts b/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts index e54806bf7..1d7855f92 100644 --- a/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts +++ b/apps/buddy/service/src/sandbox/__tests__/shellProcessOwnership.spec.ts @@ -102,7 +102,11 @@ describe.skipIf(process.platform !== 'linux')('real shell process ownership', () f.lease.markHazard('SHELL_PROCESS_TREE_UNKNOWN') const restored = new MutationCoordinator({ repository: f.repository }) expect(restored.getRunState('owner').hazards).toMatchObject([{ invocationId: 'invocation', reason: 'SHELL_PROCESS_TREE_UNKNOWN' }]) - await expect(restored.acquire({ lifetime: 'external' as const, runId: 'other', invocationId: 'other-call', claims: [{ kind: 'host' }], signal: new AbortController().signal })).rejects.toThrow('MUTATION_RESOURCE_UNSAFE') + const next = await restored.acquire({ lifetime: 'external' as const, runId: 'other', invocationId: 'other-call', claims: [{ kind: 'host' }], signal: new AbortController().signal }) + next.release() + expect(restored.getRunState('owner').hazards).toHaveLength(1) + expect(f.processes()[0]?.settlement).toBe('unknown') + await restored.dispose() } finally { for (const pid of pids) { diff --git a/apps/buddy/service/src/spaces/__tests__/SpaceFileRecovery.spec.ts b/apps/buddy/service/src/spaces/__tests__/SpaceFileRecovery.spec.ts index a4e8cc211..8c8cc2150 100644 --- a/apps/buddy/service/src/spaces/__tests__/SpaceFileRecovery.spec.ts +++ b/apps/buddy/service/src/spaces/__tests__/SpaceFileRecovery.spec.ts @@ -68,17 +68,29 @@ describe('space file acknowledgement', () => { await expect(f.files.acknowledge(f.input)).rejects.toMatchObject({ code: 'FILE_PERMISSION_REVOKED' }) expect((await f.engine.query(f.record.operationId))?.status).toBe('needs-recovery') await f.engine.register('replacement-probe', f.workspace) - await expect(f.engine.acquire(randomUUID(), [{ root: 'replacement-probe', path: 'note.txt', subtree: false }])).rejects.toThrow('FILE_RECOVERY_REQUIRED') - expect(await readFile(join(f.workspace, 'note.txt'), 'utf8')).toBe('replacement') + await expect(f.files.mutate({ ...f.input, operationId: randomUUID(), path: 'note.txt', action: { kind: 'rename', destination: 'rejected.txt' } })).rejects.toMatchObject({ code: 'FILE_PERMISSION_REVOKED' }) + const leaseId = randomUUID() + await f.engine.acquire(leaseId, [{ root: 'replacement-probe', path: 'note.txt', subtree: false }]) + try { + const current = await f.engine.read('replacement-probe', 'note.txt') + expect(current.text).toBe('replacement') + expect((await f.engine.mutate({ operationId: randomUUID(), leaseId, root: 'replacement-probe', path: 'note.txt', action: { kind: 'write', text: 'newly authorized', expected: current.etag } })).status).toBe('applied') + } + finally { await f.engine.release(leaseId) } + expect((await f.engine.query(f.record.operationId))?.status).toBe('needs-recovery') + expect(await readFile(join(f.workspace, 'note.txt'), 'utf8')).toBe('newly authorized') expect(await readFile(join(f.root, 'retained', 'note.txt'), 'utf8')).toBe('original') }) - it('keeps cross-directory operations protected when only the primary directory is authorized', async () => { + it('retains unreviewed cross-directory outcomes while allowing new operations in the authorized directory', async () => { const f = await fixture(true) expect(await f.files.history(f.input)).toContainEqual(expect.objectContaining({ path: 'note.txt', destination: null })) await expect(f.files.acknowledge(f.input)).rejects.toMatchObject({ code: 'FILE_PERMISSION_REVOKED' }) expect((await f.engine.query(f.record.operationId))?.status).toBe('needs-recovery') - await expect(f.files.mutate({ ...f.input, operationId: randomUUID(), path: 'note.txt', action: { kind: 'rename', destination: 'moved.txt' } })).rejects.toThrow('FILE_RECOVERY_REQUIRED') + expect((await f.files.mutate({ ...f.input, operationId: randomUUID(), path: 'note.txt', action: { kind: 'rename', destination: 'moved.txt' } })).status).toBe('applied') + expect(await readFile(join(f.workspace, 'moved.txt'), 'utf8')).toBe('original') + expect(await readFile(join(f.outside, 'copied.txt'), 'utf8')).toBe('original') + expect((await f.engine.query(f.record.operationId))?.status).toBe('needs-recovery') }) it('accepts current state idempotently without replaying effects or deleting staging evidence', async () => { @@ -97,7 +109,7 @@ describe('space file acknowledgement', () => { expect((await f.files.mutate({ ...f.input, operationId: randomUUID(), path: 'note.txt', action: { kind: 'rename', destination: 'moved.txt' } })).status).toBe('applied') }) - it.each(['source', 'destination', 'staging'] as const)('rejects a symlink redirect at the %s endpoint and retains protection', async (endpoint) => { + it.each(['source', 'destination', 'staging'] as const)('rejects a symlink redirect at the %s endpoint and retains the unknown outcome', async (endpoint) => { const f = await fixture() await f.files.history(f.input) const path = endpoint === 'staging' ? f.record.receipt!.staging as string : f.record[endpoint]! diff --git a/apps/buddy/service/src/storage/__tests__/schema.spec.ts b/apps/buddy/service/src/storage/__tests__/schema.spec.ts index 5df791324..9c186179d 100644 --- a/apps/buddy/service/src/storage/__tests__/schema.spec.ts +++ b/apps/buddy/service/src/storage/__tests__/schema.spec.ts @@ -571,7 +571,10 @@ describe('buddy schema', { timeout: MIGRATION_TEST_TIMEOUT }, () => { const signal = new AbortController().signal const lease = await coordinator.acquire({ runId: 'new', invocationId: 'new', lifetime: 'runtime' as const, claims: [{ kind: 'file', canonicalPath: '/fixture/file' }], signal }) lease.release() - await expect(coordinator.acquire({ runId: 'new', invocationId: 'blocked', lifetime: 'runtime' as const, claims: [{ kind: 'host' }], signal })).rejects.toMatchObject({ code: 'MUTATION_RESOURCE_UNSAFE' }) + const next = await coordinator.acquire({ runId: 'new', invocationId: 'next', lifetime: 'runtime' as const, claims: [{ kind: 'host' }], signal }) + next.release() + expect(repository.listUnreleased().map(lease => lease.invocationId)).toEqual(['shell', 'unrecorded']) + await expect(coordinator.acquire({ runId: 'new', invocationId: 'shell', lifetime: 'external', claims: [{ kind: 'host' }], signal })).rejects.toMatchObject({ code: 'MUTATION_INVOCATION_CONFLICT' }) expect(migrated.prepare('SELECT * FROM run_change_invocations').all()).toEqual(captures) expect(migrated.prepare('PRAGMA foreign_key_check').all()).toEqual([]) } diff --git a/apps/buddy/service/src/storage/runRepository.ts b/apps/buddy/service/src/storage/runRepository.ts index 3adb08869..878d395aa 100644 --- a/apps/buddy/service/src/storage/runRepository.ts +++ b/apps/buddy/service/src/storage/runRepository.ts @@ -50,6 +50,7 @@ export interface RunRepository { ) => RunRecord[] listIdsForConversation: (conversationId: string) => string[] markRunning: (id: string, startedAt: string) => boolean + reconcileQuarantined: (id: string, completedAt: string) => boolean reconcileTerminal: ( id: string, status: Extract, @@ -217,6 +218,20 @@ export function createRunRepository(database: DatabaseSync): RunRepository { markRunning(id, startedAt) { return Number(markRunning.run(startedAt, id).changes) === 1 }, + reconcileQuarantined(id, completedAt) { + return withTransaction(database, () => { + const run = find.get(id) as RunRow | undefined + if (!run) + return false + database.prepare(`UPDATE execution_state SET cleanup_state = 'degraded', cleanup_error_code = 'EVENT_LOG_CORRUPTED' + WHERE run_id IN (?, ?)`).run(id, run.root_run_id) + database.prepare(`UPDATE approvals SET status = 'cancelled', resolved_at = ? + WHERE run_id = ? AND status = 'pending'`).run(completedAt, id) + database.prepare('UPDATE runs SET pi_session_file = NULL WHERE id = ?').run(id) + return Number(database.prepare(`UPDATE runs SET status = 'failed', completed_at = ?, error_code = 'EVENT_LOG_CORRUPTED' + WHERE id = ? AND status IN ('queued', 'running')`).run(completedAt, id).changes) === 1 + }) + }, reconcileTerminal(id, status, completedAt, errorCode) { return Number(reconcileTerminal.run( status, diff --git a/apps/buddy/src/i18n/locales/en-US/chat.ts b/apps/buddy/src/i18n/locales/en-US/chat.ts index e8d2934af..1bd64859f 100644 --- a/apps/buddy/src/i18n/locales/en-US/chat.ts +++ b/apps/buddy/src/i18n/locales/en-US/chat.ts @@ -533,24 +533,24 @@ export default { 'desktop.chat.progressApproval': 'Approval pending', 'desktop.chat.progressPreparing': 'Preparing', 'desktop.chat.progressToolExecuting': 'Running', - 'desktop.chat.changeRecovery.title': 'Some locations are protected from changes', - 'desktop.chat.changeRecovery.description': 'An earlier operation still needs verification. These locations are protected to avoid concurrent changes. You can still view tasks and files.', - 'desktop.chat.changeRecovery.active': 'The earlier operation is still running. Changes cannot be resumed yet.', - 'desktop.chat.changeRecovery.unknown': 'We cannot yet confirm whether the earlier operation has stopped.', + 'desktop.chat.changeRecovery.title': 'Earlier operations need review', + 'desktop.chat.changeRecovery.description': 'These historical records do not block new operations, but earlier operations may have had side effects or may still be running. Reread files and check external state before new work.', + 'desktop.chat.changeRecovery.active': 'The latest check found the earlier operation still running. Be aware of concurrent changes.', + 'desktop.chat.changeRecovery.unknown': 'We cannot yet confirm whether the earlier operation stopped or what side effects it had.', 'desktop.chat.changeRecovery.unknownLocation': 'The affected locations are not yet known', 'desktop.chat.changeRecovery.untitledTask': 'Untitled task', 'desktop.chat.changeRecovery.deletedTask': 'Deleted task', 'desktop.chat.changeRecovery.recheck': 'Check again', 'desktop.chat.changeRecovery.more': 'More options', - 'desktop.chat.changeRecovery.acceptRisk': 'Accept risk and resume changes', - 'desktop.chat.changeRecovery.confirmTitle': 'Accept risk and resume changes?', - 'desktop.chat.changeRecovery.confirmDescription': 'Resume changes for only the following {count} items. Earlier operations may still be running, so further changes could conflict or overwrite files. Continue only if you accept these risks.', - 'desktop.chat.changeRecovery.noReplay': 'This will not stop earlier operations or rerun old commands. Accepting the risk does not confirm that those operations have ended.', - 'desktop.chat.changeRecovery.waived': 'You accepted the risk and resumed changes for the selected locations. Earlier operations have not been confirmed stopped. Old commands will not rerun automatically.', + 'desktop.chat.changeRecovery.acceptRisk': 'Mark reviewed and accept risk', + 'desktop.chat.changeRecovery.confirmTitle': 'Mark reviewed and accept risk?', + 'desktop.chat.changeRecovery.confirmDescription': 'Mark only the following {count} items as reviewed. Earlier operations may still be running, so further changes could conflict or overwrite files. Check the current state and confirm that you accept these risks.', + 'desktop.chat.changeRecovery.noReplay': 'This will not stop processes or rerun old operations. Marking items as reviewed does not confirm that those operations have ended. Historical records are retained.', + 'desktop.chat.changeRecovery.waived': 'Your review and risk acceptance were recorded for the selected items. Earlier operations have not been confirmed stopped. Historical records are retained, and old operations will not rerun automatically.', 'desktop.chat.changeRecovery.changed': 'These items have changed. Close this confirmation and check again before choosing.', - 'desktop.chat.changeRecovery.loadFailed': 'Could not load protected items. Please check again.', - 'desktop.chat.changeRecovery.recheckFailed': 'The check did not complete. Protection remains in place. Please check again later.', - 'desktop.chat.changeRecovery.resolveFailed': 'Could not resume changes for all selected items. Check again before choosing. New items will not be accepted automatically.', + 'desktop.chat.changeRecovery.loadFailed': 'Could not load items for review. New operations are not blocked. Please check again.', + 'desktop.chat.changeRecovery.recheckFailed': 'The check did not complete. Items remain available for review. Please check again later.', + 'desktop.chat.changeRecovery.resolveFailed': 'Could not record the review for all selected items. Check again before choosing. New items will not be marked reviewed automatically.', 'desktop.chat.draftRestoration.restoring': 'Restoring unsent input. Your input will be saved once restoration finishes.', 'desktop.chat.draftRestoration.failed': 'Unsent input could not be restored. Current input is not saved yet. Retry before sending or quitting.', 'desktop.chat.draftRestoration.conflict': 'Saved content conflicts with input entered during restoration. New input cannot be saved until this is resolved.', diff --git a/apps/buddy/src/i18n/locales/en-US/diagnostics.ts b/apps/buddy/src/i18n/locales/en-US/diagnostics.ts index f65677130..49ef8996f 100644 --- a/apps/buddy/src/i18n/locales/en-US/diagnostics.ts +++ b/apps/buddy/src/i18n/locales/en-US/diagnostics.ts @@ -79,9 +79,9 @@ export default { 'desktop.error.credentialStore': 'Encrypted credential storage is unavailable', 'desktop.error.credentialStoreFailure': 'Could not read or save encrypted credentials', 'desktop.error.directoryNotAuthorized': 'Authorize this directory first', - 'desktop.error.taskRecoveryRequired': 'This task’s records cannot be read, so it is unavailable. Other tasks are unaffected. Export diagnostics from Logs if you need help.', + 'desktop.error.taskRecoveryRequired': 'Some records for this execution cannot be read. The original records are preserved, and you can start a new turn. Export diagnostics from Logs if you need help.', 'desktop.error.fileRecoveryRequired': 'The outcome of an earlier file operation is unconfirmed, so related writes are paused. Export diagnostics from Logs if you need help.', - 'desktop.error.resourceRecoveryRequired': 'The outcome of an earlier operation is unconfirmed, so related writes are paused. See the logs for details.', + 'desktop.error.resourceRecoveryRequired': 'An earlier operation has an unverified outcome. Check the files and external state before starting a new operation; the old operation will not be replayed automatically.', 'desktop.error.modelSyncFailed': 'Could not sync models. Check the API endpoint and authentication, then retry.', 'desktop.error.modelSyncUnsupported': 'This API protocol does not support model synchronization.', 'desktop.error.pathOutsideDirectory': 'The target is outside the authorized directory', diff --git a/apps/buddy/src/i18n/locales/zh-CN/chat.ts b/apps/buddy/src/i18n/locales/zh-CN/chat.ts index f0bcf1aa3..c54611a32 100644 --- a/apps/buddy/src/i18n/locales/zh-CN/chat.ts +++ b/apps/buddy/src/i18n/locales/zh-CN/chat.ts @@ -531,24 +531,24 @@ export default { 'desktop.chat.progressApproval': '有操作待批准', 'desktop.chat.progressPreparing': '准备中', 'desktop.chat.progressToolExecuting': '正在执行', - 'desktop.chat.changeRecovery.title': '部分位置暂时无法修改', - 'desktop.chat.changeRecovery.description': '此前操作的状态仍需核验。为避免同时修改，下面的位置暂时受到保护；你仍可查看任务和文件。', - 'desktop.chat.changeRecovery.active': '此前的操作仍在运行，暂不允许恢复修改。', - 'desktop.chat.changeRecovery.unknown': '暂时无法确认此前的操作是否已停止。', + 'desktop.chat.changeRecovery.title': '此前操作有待核对', + 'desktop.chat.changeRecovery.description': '这些历史记录不会阻止新操作，但此前操作可能已有副作用或仍在运行。请在新操作前重新读取文件并核对外部状态。', + 'desktop.chat.changeRecovery.active': '最近核验发现此前的操作仍在运行。请注意并发修改风险。', + 'desktop.chat.changeRecovery.unknown': '暂时无法确认此前的操作是否已停止或产生了哪些副作用。', 'desktop.chat.changeRecovery.unknownLocation': '影响范围暂时无法确定', 'desktop.chat.changeRecovery.untitledTask': '未命名任务', 'desktop.chat.changeRecovery.deletedTask': '已删除的任务', 'desktop.chat.changeRecovery.recheck': '重新检查', 'desktop.chat.changeRecovery.more': '更多选项', - 'desktop.chat.changeRecovery.acceptRisk': '接受风险并恢复修改', - 'desktop.chat.changeRecovery.confirmTitle': '接受风险并恢复修改？', - 'desktop.chat.changeRecovery.confirmDescription': '仅对以下 {count} 项恢复修改。此前的操作可能仍在运行，继续修改可能产生冲突或覆盖文件。请仅在愿意承担这些风险时继续。', - 'desktop.chat.changeRecovery.noReplay': '此操作不会结束此前的操作，也不会重新运行旧命令。接受风险不代表已确认此前的操作结束。', - 'desktop.chat.changeRecovery.waived': '已按你的选择接受风险并恢复所选位置的修改；此前操作仍未确认结束，旧命令不会自动重跑。', + 'desktop.chat.changeRecovery.acceptRisk': '标记已核对并接受风险', + 'desktop.chat.changeRecovery.confirmTitle': '标记已核对并接受风险？', + 'desktop.chat.changeRecovery.confirmDescription': '仅标记以下 {count} 项为已核对。此前操作可能仍在运行，继续修改可能产生冲突或覆盖文件。请先核对当前状态，并确认接受这些风险。', + 'desktop.chat.changeRecovery.noReplay': '此操作不会停止进程，也不会重新运行旧操作。标记已核对不代表已确认此前的操作结束，历史记录会保留。', + 'desktop.chat.changeRecovery.waived': '已记录你对所选事项的核对和风险确认；此前操作仍未确认结束，历史记录会保留，旧操作不会自动重跑。', 'desktop.chat.changeRecovery.changed': '待处理事项已变化。请关闭此确认，重新检查后再选择。', - 'desktop.chat.changeRecovery.loadFailed': '暂时无法读取受保护事项。请重新检查。', - 'desktop.chat.changeRecovery.recheckFailed': '检查未完成，已保留保护状态。请稍后重新检查。', - 'desktop.chat.changeRecovery.resolveFailed': '未能恢复全部所选事项的修改。请重新检查后再选择；不会自动放行新增事项。', + 'desktop.chat.changeRecovery.loadFailed': '暂时无法读取待核对记录，不影响新操作。请重新检查。', + 'desktop.chat.changeRecovery.recheckFailed': '检查未完成，已保留待核对记录。请稍后重新检查。', + 'desktop.chat.changeRecovery.resolveFailed': '未能记录全部所选事项的核对。请重新检查后再选择；新增事项不会自动标记为已核对。', 'desktop.chat.draftRestoration.restoring': '正在恢复未发送内容，完成后将自动保存当前输入。', 'desktop.chat.draftRestoration.failed': '未发送内容尚未恢复，当前输入还未保存。请重试恢复后再发送或退出。', 'desktop.chat.draftRestoration.conflict': '已保存内容与恢复期间的输入有冲突。处理前，新输入还无法保存。', diff --git a/apps/buddy/src/i18n/locales/zh-CN/diagnostics.ts b/apps/buddy/src/i18n/locales/zh-CN/diagnostics.ts index df5df0ccb..04b51e58d 100644 --- a/apps/buddy/src/i18n/locales/zh-CN/diagnostics.ts +++ b/apps/buddy/src/i18n/locales/zh-CN/diagnostics.ts @@ -77,9 +77,9 @@ export default { 'desktop.error.credentialStore': '系统凭据加密服务不可用', 'desktop.error.credentialStoreFailure': '系统凭据读取或保存失败', 'desktop.error.directoryNotAuthorized': '请先授权这个目录', - 'desktop.error.taskRecoveryRequired': '此任务的记录无法读取，已暂停使用，其他任务不受影响。如需协助，可在日志中导出诊断信息。', + 'desktop.error.taskRecoveryRequired': '此执行的部分记录无法读取，原始记录已保留。可以继续新一轮任务；如需协助，可在日志中导出诊断信息。', 'desktop.error.fileRecoveryRequired': '上次文件操作的结果尚未确认，相关写入已暂停。如需协助，可在日志中导出诊断信息。', - 'desktop.error.resourceRecoveryRequired': '上次操作的结果尚未确认，相关写入已暂停。可在日志中查看详情。', + 'desktop.error.resourceRecoveryRequired': '上次操作的结果尚未确认。请核对文件和外部状态后发起新操作，旧操作不会自动重跑。', 'desktop.error.modelSyncFailed': '模型同步失败，请检查 API 地址和认证信息后重试', 'desktop.error.modelSyncUnsupported': '当前 API 协议不支持模型同步', 'desktop.error.pathOutsideDirectory': '目标路径超出已授权目录', diff --git a/apps/buddy/src/modules/files/explorer/explorerLabels.ts b/apps/buddy/src/modules/files/explorer/explorerLabels.ts index 5233b5fdd..38f79ee6a 100644 --- a/apps/buddy/src/modules/files/explorer/explorerLabels.ts +++ b/apps/buddy/src/modules/files/explorer/explorerLabels.ts @@ -24,7 +24,7 @@ export function explorerLabels(language: string) { cancelled: '操作已取消', unsupported: '此位置或文件类型暂不支持该操作。', permission: '目录授权已变更，请重新打开文件目录。', - pendingRecovery: '相关文件的操作或后台进程尚未确认结束，请核对文件和任务执行状态。', + pendingRecovery: '相关操作的结果尚未确认，请核对文件和任务状态后发起新操作。', unsaved: '打开的文件未能保存，请先处理编辑器中的冲突。', recovery: '文件操作结果未确认', operation: '文件操作', @@ -34,7 +34,7 @@ export function explorerLabels(language: string) { move: '移动', restore: '还原', temporary: '临时文件', - recoveryDetail: '确认表示你接受当前文件状态，并允许后续修改。不会修复文件、重做上次操作或删除临时文件。无法确认时请取消，可在日志中导出诊断信息寻求协助。', + recoveryDetail: '此记录不阻止新操作。确认只表示你已核对并接受当前文件状态，不会修复文件、重做上次操作或删除临时文件。无法确认时可暂不处理，修改前请重新读取文件。', recoveryFailure: '无法确认文件操作的状态。可在日志中导出诊断信息寻求协助。', acknowledge: '接受当前状态', trashDetail: '文件将移到回收站，可恢复。', @@ -64,7 +64,7 @@ export function explorerLabels(language: string) { cancelled: 'Operation cancelled', unsupported: 'This operation is not supported for this location or file type.', permission: 'Directory access changed. Reopen the file directory.', - pendingRecovery: 'A file operation or background process has not settled. Check the files and task execution.', + pendingRecovery: 'An earlier operation has an unverified outcome. Check the files and task state before starting a new operation.', unsaved: 'An open file could not be saved. Resolve its editor conflict first.', recovery: 'File operation outcome unconfirmed', operation: 'File operation', @@ -74,7 +74,7 @@ export function explorerLabels(language: string) { move: 'Move', restore: 'Restore', temporary: 'Temporary file', - recoveryDetail: 'Confirm only if you accept the current file state and want to allow further changes. This does not repair files, replay the operation or delete temporary files. If unsure, cancel and export diagnostics from Logs for help.', + recoveryDetail: 'This record does not block new operations. Confirmation only records that you reviewed and accept the current file state; it does not repair files, replay the operation or delete temporary files. If unsure, leave it unconfirmed and reread the files before changing them.', recoveryFailure: 'The file operation’s state could not be confirmed. Export diagnostics from Logs for help.', acknowledge: 'Accept current state', trashDetail: 'The file will be moved to the trash and can be restored.', diff --git a/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionGroup.vue b/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionGroup.vue index 4a8850d76..9d4afb62f 100644 --- a/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionGroup.vue +++ b/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionGroup.vue @@ -13,10 +13,10 @@ const BuddyExecutionCanvas = defineAsyncComponent(() => import('./BuddyExecution const canvas = useTemplateRef<{ focus: (id: string) => void }>('canvas') const canvasId = useId() const tree = useExecutionTree() -const labels = computed(() => executionLabels(props.language)) const group = computed(() => tree?.groups.value.byRoot.get(props.rootRunId)) const nodes = computed(() => projectExecutionCanvas(group.value?.records ?? [], props.rootRunId)) const root = computed(() => group.value?.root) +const labels = computed(() => executionLabels(props.language, root.value?.errorCode)) const children = computed(() => nodes.value.slice(1).map(node => node.execution)) const collaborating = computed(() => group.value?.mode === 'collaboration') const npcCount = computed(() => group.value?.subtasks.length ?? 0) diff --git a/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionNode.vue b/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionNode.vue index ac39ab8af..88c077810 100644 --- a/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionNode.vue +++ b/apps/buddy/src/modules/tasks/widgets/executions/BuddyExecutionNode.vue @@ -11,7 +11,7 @@ import { executionPhaseLabel, executionStatus } from './executionPresentation' const props = defineProps<{ execution: LocalExecution, language: BuddyLocale, now: number, selected: boolean, stopping: boolean, failed: boolean, enabled: boolean }>() const emit = defineEmits<{ open: [], stop: [] }>() -const labels = computed(() => executionLabels(props.language)) +const labels = computed(() => executionLabels(props.language, props.execution.errorCode)) const identity = useBuddyIdentity(computed(() => props.execution.buddyId)) const root = computed(() => props.execution.kind === 'root') const actor = computed(() => props.execution.kind !== 'npc_child') diff --git a/apps/buddy/src/modules/tasks/widgets/executions/DesktopExecutionDetail.vue b/apps/buddy/src/modules/tasks/widgets/executions/DesktopExecutionDetail.vue index 027164b16..a59ed05a8 100644 --- a/apps/buddy/src/modules/tasks/widgets/executions/DesktopExecutionDetail.vue +++ b/apps/buddy/src/modules/tasks/widgets/executions/DesktopExecutionDetail.vue @@ -27,7 +27,7 @@ const state = useExecutionDetail(target, computed(() => props.visible)) const detail = state.detail const execution = computed(() => detail.value?.execution) const displayName = computed(() => execution.value ? executionDisplayName(execution.value) : null) -const labels = computed(() => executionLabels(props.language)) +const labels = computed(() => executionLabels(props.language, execution.value?.errorCode)) const supplementLabels = computed(() => execution.value?.kind === 'actor_child' ? { title: labels.value.actorSupplement, placeholder: labels.value.actorSupplementPlaceholder, send: labels.value.actorSend } : { title: labels.value.supplement, placeholder: labels.value.supplementPlaceholder, send: labels.value.send }) diff --git a/apps/buddy/src/modules/tasks/widgets/executions/executionLabels.ts b/apps/buddy/src/modules/tasks/widgets/executions/executionLabels.ts index e9c485baf..466d5410f 100644 --- a/apps/buddy/src/modules/tasks/widgets/executions/executionLabels.ts +++ b/apps/buddy/src/modules/tasks/widgets/executions/executionLabels.ts @@ -15,7 +15,10 @@ const english = { unfinishedCount: '{count} unfinished', needsYouCount: '{count} need you', issuesCount: '{count} issues', - cleanupBadge: 'Cleanup issue', + cleanupBadge: 'Needs review', + incompleteHistoryBadge: 'Incomplete history', + incompleteHistory: 'Some history could not be read. The original records are preserved, and new tasks can continue. Tool counts include only recorded activity.', + recordedToolCount: 'Recorded tools {count}', fit: 'Fit canvas', zoomIn: 'Zoom in', zoomOut: 'Zoom out', @@ -69,7 +72,7 @@ const english = { questionClosed: 'Question closed', actionFailed: 'The operation failed. The task may have changed.', loadFailed: 'Could not load the result.', - cleanup: 'Some operations have not fully stopped. Conflicting writes remain blocked.', + cleanup: 'Cleanup of some earlier operations could not be verified. New tasks can continue; check the current state before making further changes.', host: 'System commands', } const chinese: typeof english = { @@ -87,7 +90,10 @@ const chinese: typeof english = { unfinishedCount: '{count} 个未结束', needsYouCount: '{count} 项需你处理', issuesCount: '{count} 项异常', - cleanupBadge: '清理异常', + cleanupBadge: '待核对', + incompleteHistoryBadge: '记录不完整', + incompleteHistory: '部分历史记录无法读取，原始记录已保留。可以继续新任务，工具次数仅包含已记录的活动。', + recordedToolCount: '已记录工具 {count} 次', fit: '适应画布', zoomIn: '放大', zoomOut: '缩小', @@ -141,10 +147,13 @@ const chinese: typeof english = { questionClosed: '问题已关闭', actionFailed: '操作失败，任务状态可能已改变。', loadFailed: '无法读取结果。', - cleanup: '部分操作尚未完全停止，冲突的写操作仍被阻止。', + cleanup: '此前部分操作的收尾情况尚未确认。可以继续新任务，修改前请核对当前状态。', host: '系统命令', } -export function executionLabels(language: BuddyLocale) { - return language === 'en-US' ? english : chinese +export function executionLabels(language: BuddyLocale, errorCode?: string | null) { + const labels = language === 'en-US' ? english : chinese + return errorCode === 'EVENT_LOG_CORRUPTED' + ? { ...labels, cleanup: labels.incompleteHistory, cleanupBadge: labels.incompleteHistoryBadge, toolCount: labels.recordedToolCount } + : labels } diff --git a/apps/buddy/src/modules/tasks/widgets/workspace/__tests__/desktopChangeRecoveryNotice.spec.ts b/apps/buddy/src/modules/tasks/widgets/workspace/__tests__/desktopChangeRecoveryNotice.spec.ts index d54423db7..a1ac95b72 100644 --- a/apps/buddy/src/modules/tasks/widgets/workspace/__tests__/desktopChangeRecoveryNotice.spec.ts +++ b/apps/buddy/src/modules/tasks/widgets/workspace/__tests__/desktopChangeRecoveryNotice.spec.ts @@ -58,17 +58,21 @@ async function review(root: Element) { } describe('change recovery confirmation', () => { - it('keeps a fixed user-reviewed selection when additional blockers arrive', async () => { + it('keeps a fixed reviewed selection when additional historical warnings arrive', async () => { const f = mount([item('first'), item('running', 'active')]) const dialog = await review(f.root) - expect(dialog.textContent).toContain('仅对以下 1 项恢复修改') + expect(f.root.textContent).toContain('这些历史记录不会阻止新操作') + expect(f.root.textContent).toContain('重新读取文件并核对外部状态') + expect(dialog.textContent).toContain('仅标记以下 1 项为已核对') expect(dialog.textContent).toContain('Task first') expect(dialog.textContent).not.toContain('Task running') - expect(dialog.textContent).toContain('接受风险不代表已确认此前的操作结束') + expect(dialog.textContent).toContain('标记已核对不代表已确认此前的操作结束') + expect(dialog.textContent).toContain('不会重新运行旧操作') + expect(dialog.textContent).toContain('历史记录会保留') f.items.value = [...f.items.value, item('new')] await nextTick() expect(dialog.textContent).not.toContain('Task new') - button(dialog, '接受风险并恢复修改').click() + button(dialog, '标记已核对并接受风险').click() await nextTick() expect(f.decisions).toEqual([{ conversationId: 'current', decision: 'waived', items: [{ invocationId: 'first', revision: 'revision-1' }] }]) expect(f.items.value.map(item => item.invocationId)).toEqual(['running', 'new']) @@ -80,9 +84,9 @@ describe('change recovery confirmation', () => { const dialog = await review(f.root) f.items.value = [{ ...item('first'), revision: 'revision-2', status: 'active' }] await nextTick() - expect(button(dialog, '接受风险并恢复修改').disabled).toBe(true) + expect(button(dialog, '标记已核对并接受风险').disabled).toBe(true) expect(dialog.textContent).toContain('待处理事项已变化') - button(dialog, '接受风险并恢复修改').click() + button(dialog, '标记已核对并接受风险').click() await nextTick() expect(f.decisions).toEqual([]) expect(f.items.value[0]?.status).toBe('active')