From 0253ef2319407de26d4ac1f83f52b0823c19fc8b Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 03:47:17 +0800 Subject: [PATCH 01/12] fix(quota): close lost execution attempts without replacement delivery Signed-off-by: lusendong.6789 Signed-off-by: lusendong.6789 Signed-off-by: lusendong.6789 Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 21 + .../monorepo-distribution-split-v0.zh-CN.md | 15 + docs/reference/protocols/turn-envelope-v0.md | 43 ++ .../project_lifecycle_refresh_state.py | 25 + loopx/cli_commands/quota_failure_report.py | 34 +- loopx/cli_runtime.py | 7 + .../control_plane/effect_runtime_handlers.ts | 1 + .../goals/first_party_host_runtime.ts | 10 +- .../quota/execution_loss_closeout.ts | 301 ++++++++ .../quota/execution_preparation.py | 105 ++- loopx/control_plane/quota/settlement_phase.ts | 2 + .../quota/settlement_readback.ts | 37 + .../work_items/execution_preparation.ts | 141 ++++ loopx/quota.py | 5 +- ...roject_registry_io_classifications_v1.json | 2 + loopx/state_refresh.py | 6 + .../test_execution_loss_closeout_cli.py | 712 ++++++++++++++++++ .../test_execution_loss_closeout_recovery.py | 270 +++++++ .../execution_loss_closeout.test.ts | 86 +++ .../quota_settlement_readback.test.ts | 3 + 20 files changed, 1813 insertions(+), 13 deletions(-) create mode 100644 loopx/control_plane/quota/execution_loss_closeout.ts create mode 100644 tests/control_plane/test_execution_loss_closeout_cli.py create mode 100644 tests/control_plane/test_execution_loss_closeout_recovery.py create mode 100644 tests/control_plane_ts/execution_loss_closeout.test.ts diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 8edc633f3d..3e1b7e1299 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4788,3 +4788,24 @@ view shows its independently checked result in the same conversation. - **Boundary:** this is a source wiring repair. Installed Core and composition qualification, interrupted original-Task recovery, complete context/closure, actual models, costs, full R1/M3 and C1 cutover remain separate acceptance. + +### 2026-11-20 — Close a lost execution attempt without replacement delivery + +- **Gap:** a replacement task lease cannot authorize the original prepared + Turn's business writeback. Repeating the same work guard can therefore leave + the original receipt open without a legal recovery operation. +- **Delta:** the explicit administrative closeout reuses the TS settlement, + Goal lifetime and File/SQLite provider owners. It preserves the original + identity and prepared proof, qualifies a distinct current control writer, + and records only blocked no-spend evidence. Its immutable Run recovers + missing tails within the same store identity; replay requires a fresh control + proof. Ordinary writeback and pure historical readback keep their boundaries. +- **Source evidence:** CLI tests cover both local providers, stale/foreign + proofs, ordinary parity and zero completion/debit. Recovery tests inject + interruption after JSON, Markdown, index and receipt durability. SourceSession + coverage uses typed-owner history plus the administrative CLI; its generic + Todo/quota entry remains unavailable. +- **Boundary:** this is an R1 recovery proposal aligned with C1's single TS + owner. Installed Core, C1 composition, the original ten requirements and full + R1/M3 regression remain independent acceptance; earlier failures and budgets + retain their status. diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index debd267ef7..a9db6de9f9 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3698,3 +3698,18 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 - **边界:**本次是源码接线修复。Installed Core 与组合资格、被中断原 Task 的 恢复、完整 context/closure、真实模型、成本、完整 R1/M3 及 C1 cutover 仍是 独立验收。 + +### 2026-11-20 — 收尾失去的执行尝试,不借用替代执行交付 + +- **缺口:**替代 task lease 不能授权原 prepared Turn 的业务写回。重复同一个 + work guard 会让原收据持续开放,却没有合法的恢复操作。 +- **增量:**显式行政收尾复用 TS settlement、Goal lifetime 与 File/SQLite + provider owner。保留原 identity/prepared proof,独立核验当前 control writer, + 仅记录 blocked 免扣额证据。不可变 Run 在同一 store identity 内恢复缺失尾部; + 重放仍须 fresh control proof。普通写回与纯历史回读保留各自边界。 +- **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 + parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt + 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, + 其通用 Todo/quota 入口仍不可用。 +- **边界:**这是与 C1 单一 TS owner 对齐的 R1 恢复提案。Installed Core、C1 + 组合、原十项要求与完整 R1/M3 回归仍须独立验收;保留既有失败与预算的状态。 diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 10198cc56e..6876889a51 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -61,6 +61,49 @@ the original `settlement_owed` recovery instead of a second debit. Contract composition failure preserves the decided receipt and available original capture, reports failure, and grants no work or automatic retry. +`refresh-state --execution-loss-closeout` explicitly closes an original +prepared Todo Turn after its execution generation was replaced. Supply the +same Goal, Agent, Todo and Turn, typed `blocked` / `outcome_gap` / +`semantic_closeout` evidence, and the current control writer's exact lease key +and version. The TypeScript owner checks original admission and current +canonical Todo/lease under the real File/SQLite provider fence, retaining the +original prepared proof. It appends only an administrative blocked no-spend Run +and refresh receipt. Current control ownership grants no original business +execution. Vision, replan, Next Action, Explore, usage, external delivery, +completion, lease mutation and ordinary refresh hooks/sync are unavailable. +Direct `refresh_state_run` callers set `sync_global=False`; an ordinary sync +request, including that API's default, is rejected before any write. +The deterministic immutable Run is the recovery intent for missing index or +receipt appends within the same authority-store identity. Explicit replay still +requires a valid current control-writer proof; pure historical settlement readback +does not. Conflicting or damaged history refuses recovery. +Repeating `quota spend-slot` on the settled identity returns its TS-qualified +no-spend progress and appends no debit. +Ordinary same-generation writeback retains its existing fence. Exact SourceSession +ownership stays with its original GoalRef. Non-activated legacy projects keep +their existing stable runtime/Goal alias ownership without an incarnation or +ABA guarantee; mixed/malformed source ownership never falls back to legacy. +Identity conflicts return pure original settlement readback, without another +work guard. Omitting the flag retains the ordinary writer and recovery rules. + +`refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 +执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / +semantic_closeout 证据及当前 control writer 的确切租约 key/version。TS owner 在真实 +File/SQLite provider fence 内核验原准入与当前 canonical Todo/lease,只追加免扣额 +blocked Run 和 refresh 收据,保留原 prepared proof;当前证明不成为原业务执行证明。 +该路径不执行 Vision、replan、Next Action、Explore、usage、外部投递、completion、 +租约 mutation 或普通 refresh hooks/sync。 +直接调用 `refresh_state_run` 时须设置 `sync_global=False`;普通 sync 请求及该 API +的默认值在任何写入前被拒绝。 +确定路径的不可变 Run 是同一 authority-store +identity 内缺失 index/receipt append 的恢复 intent。显式重放仍须有效的当前 control +writer proof;纯历史结算回读不依赖当前租约。历史冲突或损坏均拒绝。 +在已结算 identity 上重复 `quota spend-slot`,返回 TS 核验的免扣额进度,不追加扣额。 +原 generation 可用时沿普通 fence。 +SourceSession 沿原 GoalRef;未激活项目保留既有稳定 runtime/Goal alias 语义,不新增 +incarnation 或 ABA 保证,混合及非法 ownership 不回退。identity conflict 返回原结算 +的纯 readback,不再运行 work guard。省略该 flag 保持普通 writer 与 recovery 规则。 + Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. User task time requirements remain task context. Caller-private deadlines diff --git a/loopx/cli_commands/project_lifecycle_refresh_state.py b/loopx/cli_commands/project_lifecycle_refresh_state.py index 9a7e5135b8..03dda6d1d9 100644 --- a/loopx/cli_commands/project_lifecycle_refresh_state.py +++ b/loopx/cli_commands/project_lifecycle_refresh_state.py @@ -207,6 +207,8 @@ def register_refresh_state_command( refresh_state_parser.add_argument("--goal-instance-id", help=argparse.SUPPRESS) refresh_state_parser.add_argument("--task-lease-idempotency-key", help=argparse.SUPPRESS) refresh_state_parser.add_argument("--task-lease-expected-version", type=int, help=argparse.SUPPRESS) + refresh_state_parser.add_argument("--execution-loss-closeout", action="store_true", + help="Close only the original prepared Turn with verified blocked no-spend evidence after execution generation loss.") refresh_state_parser.add_argument("--completion-todo-id", help=argparse.SUPPRESS) refresh_state_parser.add_argument("--completion-turn-key", help=argparse.SUPPRESS) refresh_state_parser.add_argument( @@ -454,6 +456,29 @@ def handle_refresh_state_command( if args.command != "refresh-state": return None fmt = output_format(args) + if getattr(args, "execution_loss_closeout", False): + # This path must precede ordinary file/Explore/Vision input hooks and + # all outcome refinement, post-writeback, sink and spend composition. + from ..control_plane.goals.first_party_host_admission import capture_first_party_host_goal_ref + from ..control_plane.goals.goal_ref_validation import exact_goal_ref + from .project_lifecycle_inputs import inline_progress_observation + try: + ref = (exact_goal_ref(args.goal_id, args.goal_instance_id) if args.goal_instance_id + else capture_first_party_host_goal_ref(registry_path=registry_path, goal_id=args.goal_id)) + payload = refresh_state_run(registry_path=registry_path, runtime_root_override=args.runtime_root, + goal_id=args.goal_id, project=None, state_file=None, classification=args.classification, + recommended_action=None, agent_id=args.agent_id, todo_id=args.todo_id, + turn_instance_id=args.turn_instance_id, delivery_outcome=args.delivery_outcome, + delivery_boundary=args.delivery_boundary, progress_observation=inline_progress_observation(args), + task_lease_proof={"idempotency_key": args.task_lease_idempotency_key, + "expected_version": args.task_lease_expected_version}, goal_ref=ref, dry_run=bool(args.dry_run), + sync_global=False, + execution_loss_closeout=True, execution_loss_requested_effects=vars(args)) + except Exception as exc: + payload = {"ok": False, "goal_id": args.goal_id, "appended": False, "error": str(exc), + **({"error_code": exc.code} if isinstance(getattr(exc, "code", None), str) else {})} + print_payload(payload, fmt, render_state_refresh_markdown) + return 0 if payload.get("ok") is True else 1 agent_vision_packet: dict[str, object] | None = None progress_observation: dict[str, object] | None = None merge_agent_vision_patch = False diff --git a/loopx/cli_commands/quota_failure_report.py b/loopx/cli_commands/quota_failure_report.py index 1de65a1b0b..ed1bd3352d 100644 --- a/loopx/cli_commands/quota_failure_report.py +++ b/loopx/cli_commands/quota_failure_report.py @@ -173,13 +173,37 @@ def quota_failure_payload( "status": "heartbeat_receipt_identity_conflict", "effective_action": EffectiveAction.QUOTA_SKIP.value, "recommended_action": ( - "Rerun quota should-run with the same --turn-instance-id and without " - "--todo-id or --replan-obligation-id to read the bound Turn's recovery " - "contract. Settle verified work or record its verified blocked closeout; " - "do not replace the committed binding, repair receipt storage, or invent " - "a new Turn id. Independent work belongs to the next real host Turn." + "Read original_settlement_readback below; it reads the original receipt without work requalification. " + "If original execution is lost, use explicit refresh-state --execution-loss-closeout only for verified " + "blocked no-spend evidence with the exact current control-writer proof. Keep the original Goal, Agent, " + "Todo and Turn. A current replacement lease is not original execution; do not replace receipts, " + "repeat business work or invent a Turn id." ), }) + try: + from ..control_plane.goals.first_party_host_admission import capture_first_party_host_goal_ref + from ..control_plane.goals.goal_ref_validation import exact_goal_ref + from ..control_plane.projects.registry_codec import load_project_registry + from ..control_plane.quota.settlement import read_heartbeat_settlement, settlement_result_payload + from ..paths import resolve_runtime_root + instance = getattr(args, "goal_instance_id", None) + ref = (exact_goal_ref(args.goal_id, instance) if instance else + capture_first_party_host_goal_ref(registry_path=registry_path, goal_id=args.goal_id)) + root = resolve_runtime_root(load_project_registry(registry_path), runtime_root_arg, registry_path=registry_path) + original = read_heartbeat_settlement(root, goal_id=args.goal_id, agent_id=getattr(args, "agent_id", None), + todo_id=None, turn_instance_id=getattr(args, "turn_instance_id", None), resolve_original_binding=True, + registry_path=registry_path, goal_ref=ref) + if original is None: + raise ValueError("original receipt readback is unavailable") + payload["original_settlement_readback"] = { + "settlement_identity": original.identity.value.as_dict() if original.identity.value is not None else None, + "identity": settlement_result_payload(original.identity), + "settlement": settlement_result_payload(original.settlement), + "settlement_progress": original.progress, "replay_phase": original.replay_phase, + "prepared_lease": original.prepared_lease, + } + except (OSError, ValueError, RuntimeError, TypeError) as exc: + payload["original_settlement_readback"] = {"ok": False, "error": str(exc)} if isinstance(error, CloseoutQueryUnavailableError): payload.update({ "status": error.diagnostic_code, diff --git a/loopx/cli_runtime.py b/loopx/cli_runtime.py index e70f2fb937..b37258dd21 100644 --- a/loopx/cli_runtime.py +++ b/loopx/cli_runtime.py @@ -655,6 +655,13 @@ def _dispatch_common_command( if args.command in PROJECT_LIFECYCLE_COMMANDS: from .cli_commands.project_lifecycle import handle_project_lifecycle_command from .cli_rollout import append_cli_rollout_event + if args.command == "refresh-state" and getattr(args, "execution_loss_closeout", False) is True: + # The explicit administrative owner validates SourceSession itself. + # Ordinary hook discovery reads the generic registry before dispatch. + return handle_project_lifecycle_command( + args, registry_path=registry_path, print_payload=print_payload, + output_format=output_format, append_cli_rollout_event=append_cli_rollout_event, + ) from .capabilities.periodic_report.post_writeback_hook import ( build_periodic_report_post_writeback_projection, periodic_report_post_writeback_hooks_for_goal, diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 6fbe7ec4ab..c6f170b5b5 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -830,6 +830,7 @@ export function createEffectRuntimeHandlers( ["turn.host_todo_completion.evaluate", lazyHandler(() => import("./turn_driver/host_todo_completion.ts"), ({evaluateHostTodoCompletion}) => evaluateHostTodoCompletion)], ["work_item.interaction_execution.project", lazyHandler(() => import("./work_items/interaction_contract.ts"), ({projectInteractionExecution}) => projectInteractionExecution)], ["work_item.execution_preparation.project", lazyHandler(() => import("./work_items/execution_preparation.ts"), ({projectExecutionPreparation}) => projectExecutionPreparation)], + ["quota.execution_loss_closeout.commit", lazyHandler(() => import("./quota/execution_loss_closeout.ts"), ({commitExecutionLossCloseout}) => commitExecutionLossCloseout)], ["work_item.interaction_reads.project", lazyHandler(() => import("./quota/turn_envelope.ts"), ({projectInteractionReadsAndGuidance}) => projectInteractionReadsAndGuidance)], ["work_item.blocked_wait_observation.project", lazyHandler(() => import("./work_items/interaction_contract.ts"), ({projectBlockedSuccessorWaitObservation}) => projectBlockedSuccessorWaitObservation)], ["work_item.context.plan", lazyHandler(() => import("./work_items/interaction_contract.ts"), ({planInteractionWorkContext}) => planInteractionWorkContext)], diff --git a/loopx/control_plane/goals/first_party_host_runtime.ts b/loopx/control_plane/goals/first_party_host_runtime.ts index e2b655ccad..8a5baa2baf 100644 --- a/loopx/control_plane/goals/first_party_host_runtime.ts +++ b/loopx/control_plane/goals/first_party_host_runtime.ts @@ -140,6 +140,12 @@ function registryWitness(value: unknown): {path: string; sha256: string} | null ? {path: witness.path, sha256: witness.sha256} : null; } +/** Legacy registration follows the existing active-* Goal frontier vocabulary. */ +export function legacyGoalRegistrationIsActive(value: unknown): boolean { + const registration = typeof value === "string" ? value.trim().toLowerCase() : ""; + return registration === "active" || registration.startsWith("active-"); +} + /** Launch rechecks the registry. An already dispatched Turn checks its running authority. */ function legacyRuntime(facts: JsonObject): FirstPartyHostRuntimeDecision { const current = jsonObject(facts.authority); @@ -148,9 +154,7 @@ function legacyRuntime(facts: JsonObject): FirstPartyHostRuntimeDecision { || current.goal_id !== facts.goal_id) return {kind: "reject", code: "goal_authority_unavailable"}; // Legacy registration accepts the active-* variants used by the existing // Goal frontier. Source-session registration remains exact-only below. - const registration = typeof current.registration_status === "string" - ? current.registration_status.trim().toLowerCase() : ""; - if (registration !== "active" && !registration.startsWith("active-")) { + if (!legacyGoalRegistrationIsActive(current.registration_status)) { return {kind: "reject", code: "goal_not_registered"}; } if (current.activation_state === "stopped") return {kind: "reject", code: "goal_stopped"}; diff --git a/loopx/control_plane/quota/execution_loss_closeout.ts b/loopx/control_plane/quota/execution_loss_closeout.ts new file mode 100644 index 0000000000..048d012960 --- /dev/null +++ b/loopx/control_plane/quota/execution_loss_closeout.ts @@ -0,0 +1,301 @@ +/** Explicit administrative closeout. The immutable Run is its durable intent; + * original admission and the current provider head stay fenced through append. */ +import {createHash, randomUUID} from "node:crypto"; +import {closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync} from "node:fs"; +import {dirname, join, resolve} from "node:path"; +import {isDeepStrictEqual} from "node:util"; +import {settlementIdentity, settlementIdentityPayload, type JsonObject} from "../effect_program.ts"; +import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {appendJsonLineSync, claimFileMutationLock, mutationLockOwner, releaseFileMutationLockClaim, + type FileMutationLockClaim} from "../effect_runtime_io.ts"; +import {jsonObject, requireJsonObject, requireNonEmptyString} from "../runtime_decode.ts"; +import {canonicalAuthoritySha256} from "../coordination/authority_store_codec.ts"; +import {indexCoordinationProjection} from "../coordination/coordination_projection.ts"; +import {FileAuthorityStore} from "../coordination/file_authority_store.ts"; +import {SqliteAuthorityStore} from "../coordination/sqlite_authority_store.ts"; +import {openRuntimeAuthorityStore, requireLocalAuthorityRuntimeRoot} from "../coordination/local_authority_provider.ts"; +import {requireShadowPrimaryWriteAllowed, shadowMaintenanceLockPath} from "../coordination/shadow_management.ts"; +import {decodeTaskLeaseAuthority} from "../work_items/task_lease_acquire.ts"; +import {legacyGoalRegistrationIsActive} from "../goals/first_party_host_runtime.ts"; +import {EXECUTION_LOSS_QUALIFICATION_SCHEMA, isExecutionLossCloseout, qualifyReceiptBoundExecutionLossCloseout} + from "../work_items/execution_preparation.ts"; +import {goalRolloutEventLogPath} from "../rollout_receipt_log.ts"; +import {parseQuotaAccountingOwner, quotaGoalRef, requireCurrentQuotaAccountingOwner, + requireQuotaOwnerProjection, withBorrowedQuotaAccountingOwner} from "./source_admission.ts"; +import {QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, readQuotaSettlementForAdmittedOwnerFromSnapshot, + readQuotaSettlementSnapshot} from "./settlement_readback.ts"; + +export const EXECUTION_LOSS_COMMIT_SCHEMA = "quota_execution_loss_closeout_commit_v0"; +export const EXECUTION_LOSS_COMMIT_RESULT_SCHEMA = "quota_execution_loss_closeout_result_v0"; + +function reject(message: string, code = "refresh_task_lease_unavailable"): never { + throw new EffectRuntimeRequestError(`execution-loss closeout ${message}`, code); +} +const digest = (bytes: Uint8Array): string => createHash("sha256").update(bytes).digest("hex"); +function optionalBytes(path: string): Buffer | null { + try {return readFileSync(path);} + catch (error) {if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error;} +} +const bytes = (path: string): Buffer => optionalBytes(path) ?? Buffer.alloc(0); +function rows(path: string): JsonObject[] { + const content = bytes(path); + if (content.length && content[content.length - 1] !== 10) + reject("has an incomplete history tail; retain the immutable Run for inspection", "execution_loss_commit_unknown"); + return content.toString("utf8").split(/\r?\n/u).filter(line => line.trim()).map(line => { + try {return requireJsonObject(JSON.parse(line), "closeout history row");} + catch {reject("has malformed history", "execution_loss_commit_unknown");} + }); +} +function immutableText(path: string, content: string): void { + const existing = optionalBytes(path); + if (existing !== null) { + if (existing.toString("utf8") !== content) reject("immutable artifact differs", "execution_loss_commit_unknown"); + return; + } + mkdirSync(dirname(path), {recursive: true, mode: 0o700}); + const temporary = `${path}.tmp-${randomUUID()}`; + try { + const fd = openSync(temporary, "wx", 0o600); + try {writeFileSync(fd, content, "utf8"); fsyncSync(fd);} finally {closeSync(fd);} + renameSync(temporary, path); + if (process.platform !== "win32") { + const directory = openSync(dirname(path), "r"); + try {fsyncSync(directory);} finally {closeSync(directory);} + } + } finally {try {unlinkSync(temporary);} catch (error) {if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;}} +} + +/** No alternate writer hooks or semantic defaults enter this closed intent. */ +export function decodeExecutionLossCloseoutIntent(value: unknown): JsonObject { + const intent = requireJsonObject(value, "administrative closeout intent"); + if (Object.keys(intent).sort().join("|") !== + ["delivery_outcome", "delivery_boundary", "progress_observation"].sort().join("|") + || intent.delivery_outcome !== "outcome_gap" || intent.delivery_boundary !== "semantic_closeout") + reject("permits only explicit blocked no-spend intent"); + return {...intent, progress_observation: requireJsonObject(intent.progress_observation, "blocked observation")}; +} + +function rejectRequestedEffects(value: unknown): void { + const effects = requireJsonObject(value, "requested administrative effects"); + const route = ["command", "format", "subcommand_format", "registry", "runtime_root", "goal_id", "goal_instance_id", + "project", "state_file", "classification", "execution_loss_closeout", "agent_id", "todo_id", "turn_instance_id", + "delivery_outcome", "delivery_boundary", "delivery_batch_scale", "progress_result_class", "progress_work_item_id", + "progress_blocker_id", "progress_evidence_ids", "task_lease_idempotency_key", "task_lease_expected_version", + "no_global_sync", "suppress_external_sinks", "quiet", "thin", "verbose"]; + for (const [field, value] of Object.entries(effects)) { + if (field === "delivery_batch_scale" && value !== null && value !== undefined && value !== "single_surface") + reject("permits only single-surface administrative closeout"); + if (!route.includes(field) && value !== null && value !== undefined && value !== false + && !(Array.isArray(value) && value.length === 0)) reject(`does not permit ${field}`); + } +} + +export async function commitExecutionLossCloseout(value: unknown): Promise { + const request = requireJsonObject(value, "administrative closeout request"); + if (request.schema_version !== EXECUTION_LOSS_COMMIT_SCHEMA || request.execution_loss_closeout !== true) + reject("requires its explicit typed operation"); + const permitted = ["schema_version", "execution_loss_closeout", "runtime_root", "identity", "goal_ref", + "source_admission", "registry_source", "locks", "intent", "control_writer_proof", "observed_at", "requested_effects"]; + if (Object.keys(request).some(field => !permitted.includes(field))) reject("has unsupported requested effects"); + rejectRequestedEffects(request.requested_effects ?? {}); + const root = requireLocalAuthorityRuntimeRoot(request.runtime_root); + const binding = requireJsonObject(request.identity, "original identity"); + const identity = settlementIdentity({goal_id: requireNonEmptyString(binding.goal_id, "goal_id"), + agent_id: requireNonEmptyString(binding.agent_id, "agent_id"), todo_id: requireNonEmptyString(binding.todo_id, "todo_id"), + turn_instance_id: requireNonEmptyString(binding.turn_instance_id, "turn_instance_id")}); + if (!isDeepStrictEqual(binding, settlementIdentityPayload(identity))) reject("identity is not canonical"); + const intent = decodeExecutionLossCloseoutIntent(request.intent); + const owner = parseQuotaAccountingOwner({goalRefValue: request.goal_ref, + sourceAdmissionValue: request.source_admission, runtimeRoot: root, goalId: identity.goal_id}); + if (owner.kind === "alias" && (Object.hasOwn(request, "goal_ref") || Object.hasOwn(request, "source_admission"))) + reject("legacy ownership requires absent exact-source fields"); + const registrySource = requireJsonObject(request.registry_source, "registry source"); + const registryPath = resolve(requireNonEmptyString(registrySource.path, "registry path")); + const lifetimeSources = registrySource.lifetime_sources ?? []; + if (!Array.isArray(lifetimeSources) || lifetimeSources.length > 1 + || (owner.kind === "exact_source" && lifetimeSources.length)) reject("legacy source lifetime is unsupported"); + const upstream = lifetimeSources.map(source => requireJsonObject(source, "legacy source lifetime")); + if (owner.kind === "exact_source" && registryPath !== owner.admission.registryPath) + reject("registry differs from the original source admission"); + const indexPath = join(root, "goals", identity.goal_id, "runs", "index.jsonl"); + const eventPath = goalRolloutEventLogPath(root, identity.goal_id); + const artifact = join(dirname(indexPath), `execution-loss-${createHash("sha256").update(identity.effect_id).digest("hex")}`); + const jsonPath = `${artifact}.json`, markdownPath = `${artifact}.md`; + const targets = [shadowMaintenanceLockPath(root, identity.goal_id), eventPath, + ...(owner.kind === "alias" ? [indexPath, registryPath, + ...upstream.map(source => requireNonEmptyString(source.path, "legacy source path"))] : [])] + .map(path => resolve(path)); + const requestDigest = canonicalAuthoritySha256({identity: binding, goal_ref: quotaGoalRef(owner), intent}); + const claims: FileMutationLockClaim[] = []; + try { + if (!Array.isArray(request.locks) || request.locks.length !== targets.length) + reject("requires the existing writer/lifetime lock handoff"); + for (const [index, value] of request.locks.entries()) { + const witness = requireJsonObject(value, "writer lock witness"); + if (resolve(String(witness.target)) !== targets[index]) reject("writer lock target differs"); + const token = requireNonEmptyString(witness.token, "lock token"); + const claim = await claimFileMutationLock(targets[index], token); + if (!claim) reject("writer lock handoff expired"); + claims.push(claim); + const current = await mutationLockOwner(targets[index]); + if (current?.token !== token || current.pid !== witness.pid) reject("writer lock owner changed"); + } + return await withBorrowedQuotaAccountingOwner(owner, async () => { + const registryBytes = readFileSync(registryPath); + if (digest(registryBytes) !== registrySource.sha256) reject("current registry changed"); + // The existing registry codec validated this transport observation. Bind + // it to complete bytes; exact-source absence cannot become a legacy alias. + const document = JSON.parse(registryBytes.toString("utf8")); + const registry = requireJsonObject(registrySource.payload, "decoded registry source"); + const rawPayload = Array.isArray(document) && document.length === 2 ? document[1] : document; + if (!isDeepStrictEqual(registry, rawPayload) + || (registry.profile_id === "source_session_v1") !== (owner.kind === "exact_source")) + reject("registry profile and receipt ownership differ"); + const goals = registry.goals; + if (!Array.isArray(goals)) reject("registry Goal authority is unavailable"); + const matches = goals.filter(goal => jsonObject(goal)?.id === identity.goal_id).map(goal => requireJsonObject(goal, "Goal")); + const goal = matches[0]; + if (matches.length !== 1 || !goal || (owner.kind === "alias" + ? !legacyGoalRegistrationIsActive(goal.status ?? "active") : goal.status !== "active") + || Object.hasOwn(goal, "activation_state") + || (goal.activation != null && (jsonObject(goal.activation)?.schema_version != null + && jsonObject(goal.activation)?.schema_version !== "loopx_goal_activation_v1" + || jsonObject(goal.activation)?.state !== "active")) + || (owner.kind === "exact_source" && goal.goal_instance_id !== owner.goalRef.goalInstanceId.value)) + reject("current Goal is stopped, unavailable or recreated"); + const sourceRoute = goal.source_registry; + const expectedUpstream = owner.kind === "alias" && typeof sourceRoute === "string" && sourceRoute.trim() + ? resolve(dirname(registryPath), sourceRoute.trim()) : null; + if ((sourceRoute != null && sourceRoute !== "" && typeof sourceRoute !== "string") + || (expectedUpstream === null ? upstream.length !== 0 + : upstream.length !== 1 || resolve(String(upstream[0].path)) !== expectedUpstream)) + reject("current legacy source route differs from its lifetime admission"); + for (const source of upstream) { + const sourceBytes = readFileSync(String(source.path)); + const sourcePayload = requireJsonObject(source.payload, "legacy source registry"); + const raw = JSON.parse(sourceBytes.toString("utf8")); + if (digest(sourceBytes) !== source.sha256 || !isDeepStrictEqual(sourcePayload, Array.isArray(raw) ? raw[1] : raw) + || sourcePayload.profile_id === "source_session_v1") reject("legacy source ownership changed"); + const sourceGoals = Array.isArray(sourcePayload.goals) ? sourcePayload.goals.map(jsonObject) : []; + const sourceGoal = sourceGoals.filter(row => row?.id === identity.goal_id); + if (sourceGoal.length !== 1 || !legacyGoalRegistrationIsActive(sourceGoal[0]?.status ?? "active") + || Object.hasOwn(sourceGoal[0]!, "activation_state") + || (sourceGoal[0]?.activation != null && jsonObject(sourceGoal[0].activation)?.state !== "active")) + reject("legacy source Goal is unavailable or stopped"); + } + const authority = decodeTaskLeaseAuthority({handoff_mode: "hard_lease", todos: [], + registered_agent_candidates: [jsonObject(goal.coordination)?.registered_agents, goal.registered_agents, + jsonObject(goal.spawn_policy)?.registered_agents], + source_receipts: [{source_id: "registry", path: registryPath, state: "file", sha256: digest(registryBytes)}]}); + rows(indexPath); rows(eventPath); + const indexDigest = digest(bytes(indexPath)), eventDigest = digest(bytes(eventPath)); + const snapshot = await readQuotaSettlementSnapshot(root, identity.goal_id); + const readback = readQuotaSettlementForAdmittedOwnerFromSnapshot({ + schema_version: QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, runtime_root: root, + goal_id: identity.goal_id, agent_id: identity.agent_id, todo_id: identity.todo_id, + turn_instance_id: identity.turn_instance_id, infer_turn_instance_id: false, allow_unbound_binding: false, + }, owner, snapshot); + const receipt = requireJsonObject(readback.heartbeat_receipt, "original heartbeat receipt"); + requireQuotaOwnerProjection(owner, receipt.goal_ref, "original heartbeat receipt"); + if (jsonObject(readback.identity)?.failure != null) reject("original settlement identity could not be verified"); + const existingBytes = optionalBytes(jsonPath); + let existing: JsonObject | null = null; + if (existingBytes !== null) { + try {existing = requireJsonObject(JSON.parse(existingBytes.toString("utf8")), "immutable closeout Run");} + catch {reject("immutable Run is empty or malformed", "execution_loss_commit_unknown");} + } + if (existing && jsonObject(existing.execution_loss_commit)?.request_sha256 !== requestDigest) + reject("intent differs from the original immutable Run", "committed_writeback_payload_conflict"); + const prior = jsonObject(readback.writeback_run); + if (prior && (!existing || !isDeepStrictEqual(prior, existing))) + reject("cannot replace the original writeback", "committed_writeback_payload_conflict"); + if (readback.spend_run != null || readback.spend_event != null || readback.completion_event != null) + reject("cannot erase business completion or a committed debit"); + if (!existing && (readback.replay_phase !== "open" || jsonObject(readback.progress)?.state !== "writeback_required")) + reject("requires the original unclosed first writeback"); + await requireShadowPrimaryWriteAllowed(root, identity.goal_id); + const store = await openRuntimeAuthorityStore(root, identity.goal_id, {}); + if (!(store instanceof FileAuthorityStore) && !(store instanceof SqliteAuthorityStore)) + reject("supports only the existing File and SQLite provider fences"); + const result = await store.withCheckpointHead((head, providerIdentity) => { + // No await from current canonical head through all durable writes. + if (digest(readFileSync(registryPath)) !== registrySource.sha256 + || digest(bytes(indexPath)) !== indexDigest || digest(bytes(eventPath)) !== eventDigest) + reject("sources changed before administrative commit"); + if (upstream.some(source => digest(readFileSync(String(source.path))) !== source.sha256)) + reject("legacy source changed before administrative commit"); + requireCurrentQuotaAccountingOwner(owner); + const projection = indexCoordinationProjection(head.head, identity.goal_id); + if (head.head.handoff_mode !== "hard_lease") reject("requires original hard-lease authority"); + const qualification = qualifyReceiptBoundExecutionLossCloseout({ + schema_version: EXECUTION_LOSS_QUALIFICATION_SCHEMA, identity: binding, + ...(owner.kind === "exact_source" ? {goal_ref: quotaGoalRef(owner)} : {}), receipt, + current_todo: projection.todos.get(identity.todo_id!), current_lease: projection.leases.get(identity.todo_id!), + registered_agents: authority.registered_agents, control_writer_proof: request.control_writer_proof, + progress_observation: intent.progress_observation, observed_at: request.observed_at, + provider_identity: providerIdentity, provider_revision: head.provider_revision, + }); + const buildRun = (qualified: JsonObject, observedAt: unknown): JsonObject => { + const event = {schema_version: "loopx_rollout_event_v0", + event_id: createHash("sha256").update(`${identity.effect_id}#execution-loss-closeout`).digest("hex"), + event_kind: "refresh_state", goal_id: identity.goal_id, agent_id: identity.agent_id, + todo_id: identity.todo_id, run_id: identity.turn_instance_id, status: "appended", + recorded_at: observedAt, ...(owner.kind === "exact_source" ? {goal_ref: quotaGoalRef(owner)} : {}), + details: {settlement_effect_id: identity.effect_id, execution_loss_closeout: qualified}}; + return {generated_at: observedAt, goal_id: identity.goal_id, agent_id: identity.agent_id, + todo_id: identity.todo_id, turn_instance_id: identity.turn_instance_id, settlement_identity: binding, + ...(owner.kind === "exact_source" ? {goal_ref: quotaGoalRef(owner)} : {}), + classification: "execution_loss_closeout", delivery_outcome: "outcome_gap", delivery_batch_scale: "single_surface", + delivery_boundary: "semantic_closeout", progress_observation: intent.progress_observation, + blocked_retry: qualified, execution_loss_closeout: qualified, + json_path: jsonPath, markdown_path: markdownPath, + execution_loss_commit: {schema_version: EXECUTION_LOSS_COMMIT_SCHEMA, request_sha256: requestDigest, + original_receipt_sha256: canonicalAuthoritySha256(receipt), event}}; + }; + let record = existing; + if (!record) { + record = buildRun(qualification, request.observed_at); + immutableText(jsonPath, JSON.stringify(record, null, 2) + "\n"); + } else { + const frozen = jsonObject(record.execution_loss_closeout); + if (!frozen || !isExecutionLossCloseout(frozen, identity.todo_id) + || frozen.effect_id !== identity.effect_id || frozen.observed_at !== record.generated_at + || frozen.original_receipt_event_id !== receipt.event_id + || !isDeepStrictEqual(frozen.original_prepared, qualification.original_prepared) + || jsonObject(frozen.current_control_writer)?.owner !== identity.agent_id + || jsonObject(frozen.current_control_writer)?.store_identity !== providerIdentity + || frozen.observation_fingerprint !== jsonObject(intent.progress_observation)?.fingerprint + || !isDeepStrictEqual(record, buildRun(frozen, record.generated_at))) + reject("immutable Run does not match original administrative intent", "execution_loss_commit_unknown"); + } + const event = requireJsonObject(requireJsonObject(record.execution_loss_commit, "closeout intent").event, "closeout receipt"); + if (event.event_kind !== "refresh_state" || event.run_id !== identity.turn_instance_id + || jsonObject(event.details)?.settlement_effect_id !== identity.effect_id + || !isDeepStrictEqual(jsonObject(event.details)?.execution_loss_closeout, record.execution_loss_closeout)) + reject("immutable refresh receipt is invalid", "execution_loss_commit_unknown"); + immutableText(markdownPath, `# Administrative execution-loss closeout\n\nTurn: ${identity.turn_instance_id}\nNo delivery, completion or quota spend.\n`); + const indexed = rows(indexPath).filter(row => jsonObject(row.settlement_identity)?.effect_id === identity.effect_id); + if (indexed.length > 1 || (indexed.length === 1 && !isDeepStrictEqual(indexed[0], record))) + reject("indexed writeback differs", "execution_loss_commit_unknown"); + const recorded = rows(eventPath).filter(row => row.event_id === event.event_id); + if (recorded.length > 1 || (recorded.length === 1 && !isDeepStrictEqual(recorded[0], event))) + reject("refresh receipt differs", "execution_loss_commit_unknown"); + if (!indexed.length) appendJsonLineSync(indexPath, record); + if (!recorded.length) appendJsonLineSync(eventPath, event); + return {record, replayed: indexed.length === 1 && recorded.length === 1}; + }); + const finalSnapshot = await readQuotaSettlementSnapshot(root, identity.goal_id); + const final = readQuotaSettlementForAdmittedOwnerFromSnapshot({schema_version: QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, + runtime_root: root, goal_id: identity.goal_id, agent_id: identity.agent_id, todo_id: identity.todo_id, + turn_instance_id: identity.turn_instance_id, infer_turn_instance_id: false, allow_unbound_binding: false}, owner, finalSnapshot); + if (final.replay_phase !== "settled") reject("durable no-spend readback is incomplete", "execution_loss_commit_unknown"); + return {schema_version: EXECUTION_LOSS_COMMIT_RESULT_SCHEMA, ok: true, mode: "execution_loss_closeout", + classification: "execution_loss_closeout", appended: result.replayed !== true, idempotent_replay: result.replayed === true, + dry_run: false, runtime_root: root, goal_id: identity.goal_id, settlement_identity: binding, + ...(owner.kind === "exact_source" ? {goal_ref: quotaGoalRef(owner)} : {}), + execution_loss_closeout: jsonObject(result.record)?.execution_loss_closeout, + json_path: jsonPath, markdown_path: markdownPath, settlement_progress: final.progress, + replay_phase: final.replay_phase, settlement_result: jsonObject(final.settlement)?.payload}; + }); + } finally {for (const claim of claims.reverse()) await releaseFileMutationLockClaim(claim);} +} diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 4fb044f3e6..f8bf45c22b 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -3,7 +3,7 @@ import hashlib import shlex from collections.abc import Iterator, Mapping -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from pathlib import Path from typing import TYPE_CHECKING, Any @@ -144,6 +144,109 @@ class RefreshTaskLeaseUnavailable(ValueError): code = "refresh_task_lease_unavailable" +def commit_execution_loss_closeout_from_refresh(arguments: Mapping[str, Any]) -> dict[str, Any]: + """Map the early refresh entry to its existing typed administrative owner.""" + # Preserve original CLI requests independently of direct arguments. Only + # these explicit input fields cross the transport; caller locals never do. + effects = { + **dict(arguments["execution_loss_requested_effects"] or {}), + **{f"direct_{field}": arguments[field] for field in ( + "recommended_action", "next_action", "next_action_basis", "replan_obligation_id", + "autonomous_replan_recorded", "repair_delta_kinds", "agent_vision_packet", + "merge_agent_vision_patch", "vision_unchanged_reason", "checkpoint_read_context_id", + "explore_result", "completion_todo_id", "completion_turn_key", "usage_measurement", + "dry_run", "external_delivery", "goal_acceptance_preview", "agent_lane", "progress_scope", "sync_global", + )}, + **{f"direct_{field}": str(arguments[field]) if arguments[field] is not None else None + for field in ("project", "state_file", "delivery_workspace_path")}, + "direct_usage_codex_session": str(arguments["usage_codex_session"]) if arguments["usage_codex_session"] else None, + "direct_delivery_batch_scale": (arguments["delivery_batch_scale"] + if arguments["delivery_batch_scale"] not in (None, "single_surface") else None), + } + return commit_execution_loss_closeout( + registry_path=arguments["registry_path"], runtime_root_override=arguments["runtime_root_override"], + goal_id=arguments["goal_id"], agent_id=arguments["agent_id"] or "", todo_id=arguments["todo_id"] or "", + turn_instance_id=arguments["turn_instance_id"] or "", goal_ref=arguments["goal_ref"], + requested_proof=arguments["task_lease_proof"], progress_observation=arguments["progress_observation"], + delivery_outcome=arguments["delivery_outcome"], delivery_boundary=arguments["delivery_boundary"], + requested_effects=effects, + ) + + +def commit_execution_loss_closeout( + *, registry_path: Path, runtime_root_override: str | None, goal_id: str, agent_id: str, + todo_id: str, turn_instance_id: str, goal_ref: Mapping[str, Any] | None, + requested_proof: Mapping[str, Any] | None, progress_observation: Mapping[str, Any] | None, + delivery_outcome: str | None, delivery_boundary: str | None, + requested_effects: Mapping[str, Any], +) -> dict[str, Any]: + """Transport one explicit administrative intent; TS owns every verdict/write.""" + from ...file_lock import cross_runtime_lock_witness + from ...paths import resolve_runtime_root + from ..effect_runtime import EffectRuntimeRejected + from ..goals.first_party_host_admission import FirstPartyHostGoalAdmission + from ..projects.registry_codec import load_project_registry + from ..runtime.time import now_local_iso + from ..work_items.progress_observation import normalize_progress_observation + from .accounting_admission import quota_accounting_admission + from .settlement import SettlementIdentity + + registry_path = registry_path.expanduser().resolve() + registry = load_project_registry(registry_path) + root = resolve_runtime_root(registry, runtime_root_override, registry_path=registry_path) + identity = SettlementIdentity(goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, + turn_instance_id=turn_instance_id).as_dict() + progress = (normalize_progress_observation(progress_observation, work_item_id=todo_id) + if progress_observation is not None else None) + admission = FirstPartyHostGoalAdmission.for_plan(registry_path=registry_path, + goal_id=goal_id, planned_goal_ref=goal_ref, agent_id=agent_id) + with quota_accounting_admission(runtime_root=root, registry_path=registry_path, + goal_id=goal_id, goal_ref=goal_ref, operation="execution-loss-closeout", + handoff_legacy_index=True) as source_admission: + lifetime = (nullcontext() if source_admission is not None + else admission._legacy_lifetime(operation="execution-loss-closeout", launch=False)) + with lifetime, ExitStack() as locks: + targets = [shadow_maintenance_lock_target(root, goal_id), + root / "goals" / goal_id / "rollout-event-log.jsonl"] + for target in targets: + # Existing quota requalification can own the rollout lock + # before asking for index/source; do not wait in reverse order. + locks.enter_context(exclusive_cross_runtime_file_lock(target, + timeout_seconds=0, operation="execution-loss-closeout")) + raw = registry_path.read_bytes() + witnesses = [cross_runtime_lock_witness(target) for target in targets] + lifetime_sources = [] + if source_admission is None: + witnesses.append(cross_runtime_lock_witness(root / "goals" / goal_id / "runs" / "index.jsonl")) + witnesses.append(cross_runtime_lock_witness(registry_path)) + upstream = admission.legacy_source_admission + while upstream is not None: + source_path = upstream.registry_path + source_bytes = source_path.read_bytes() + lifetime_sources.append({"path": str(source_path), "sha256": hashlib.sha256(source_bytes).hexdigest(), + "payload": load_project_registry(source_path)}) + witnesses.append(cross_runtime_lock_witness(source_path)) + upstream = upstream.legacy_source_admission + request = {"schema_version": "quota_execution_loss_closeout_commit_v0", + "execution_loss_closeout": True, "runtime_root": str(root), "identity": identity, + **({"goal_ref": dict(goal_ref)} if goal_ref is not None else {}), + **({"source_admission": source_admission} if source_admission is not None else {}), + "registry_source": {"path": str(registry_path), "sha256": hashlib.sha256(raw).hexdigest(), + "payload": load_project_registry(registry_path), "lifetime_sources": lifetime_sources}, "locks": witnesses, + "intent": {"delivery_outcome": delivery_outcome, "delivery_boundary": delivery_boundary, + "progress_observation": progress}, "control_writer_proof": dict(requested_proof) if requested_proof else None, + "requested_effects": dict(requested_effects), "observed_at": now_local_iso()} + try: + result = effect_runtime_result("quota.execution_loss_closeout.commit", request) + except EffectRuntimeRejected as exc: + error = RefreshTaskLeaseUnavailable(str(exc)) + error.code = exc.diagnostic_code + raise error from exc + if not isinstance(result, dict) or result.get("schema_version") != "quota_execution_loss_closeout_result_v0": + raise RuntimeError("typed execution-loss closeout returned an invalid result") + return result + + def prepared_lease_receipt_fields( proof: Mapping[str, Any], *, lease_epoch: int | None = None, ) -> dict[str, Any]: diff --git a/loopx/control_plane/quota/settlement_phase.ts b/loopx/control_plane/quota/settlement_phase.ts index 4836ddf165..8527a46a45 100644 --- a/loopx/control_plane/quota/settlement_phase.ts +++ b/loopx/control_plane/quota/settlement_phase.ts @@ -4,6 +4,7 @@ import { parseTodoTimestampMicros } from "../runtime_timestamp.ts"; import { isCanonicalTodoId } from "../todos/identity.ts"; import { blockedResumeInstant, isBlockedLifecycleWait, isCausalBlockedWait, isMonitorUnavailableWait } from "./blocked_wait.ts"; import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts"; +import {isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; function retryDelayIsBounded(seconds: number): boolean { return Number.isFinite(seconds) && seconds >= 60 && seconds <= 30 * 60; @@ -13,6 +14,7 @@ function retryDelayIsBounded(seconds: number): boolean { * retry/causal wait, blocked lifecycle or a Monitor unavailable attempt. * A lifecycle/Monitor closeout creates no retry clock or execution grant. */ export function isBoundedBlockedRetry(value: unknown, todoId: string | null): boolean { + if (isExecutionLossCloseout(value, todoId)) return true; if (isCausalBlockedWait(value, todoId)) return true; if (isBlockedLifecycleWait(value, todoId)) return true; if (isMonitorUnavailableWait(value, todoId)) return true; diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 9f046b5009..8cc96ea898 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -49,6 +49,9 @@ import { type ReceiptBoundReplayPhase, } from "./settlement_phase.ts"; import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts"; +import {isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; +import {isDeepStrictEqual} from "node:util"; +import {canonicalAuthoritySha256} from "../coordination/authority_store_codec.ts"; import { decodeRefreshRetry, isMaterialMonitorPoll, @@ -86,6 +89,7 @@ import { } from "../work_items/replan_semantics.ts"; import { parseQuotaAccountingOwner, + quotaGoalRef, quotaOwnerOwnsProjection, withBorrowedQuotaAccountingOwner, withQuotaAccountingOwner, @@ -209,6 +213,7 @@ function settlementProgress( quota_spend_source: source, ...(noSpendKind ? { closeout_kind: noSpendKind, + no_spend: true, } : {}), }; } @@ -1479,6 +1484,38 @@ function readQuotaSettlementFromRequest( } const completionEvent = findStepEvent(events, identity, "todo_complete"); const supersedeEvent = findStepEvent(events, identity, "todo_supersede"); + if (writebackRun?.execution_loss_closeout !== undefined) { + const closeout = jsonObject(writebackRun.execution_loss_closeout); + const commit = jsonObject(writebackRun.execution_loss_commit); + const permittedRunFields = ["generated_at", "goal_id", "agent_id", "todo_id", "turn_instance_id", + "settlement_identity", "classification", "delivery_outcome", "delivery_batch_scale", "delivery_boundary", + "progress_observation", "blocked_retry", "execution_loss_closeout", "json_path", "markdown_path", + "execution_loss_commit", ...(request.owner.kind === "exact_source" ? ["goal_ref"] : [])]; + if (!closeout || !isExecutionLossCloseout(closeout, identity.todo_id) + || Object.keys(writebackRun).sort().join("|") !== permittedRunFields.sort().join("|") + || writebackRun.classification !== "execution_loss_closeout" + || writebackRun.delivery_outcome !== "outcome_gap" || writebackRun.delivery_boundary !== "semantic_closeout" + || writebackRun.delivery_batch_scale !== "single_surface" || writebackRun.generated_at !== closeout.observed_at + || closeout.effect_id !== identity.effect_id + || closeout.original_receipt_event_id !== heartbeatReceipt.event_id + || !isDeepStrictEqual(closeout.original_prepared, preparedLease) + || jsonObject(closeout.current_control_writer)?.owner !== identity.agent_id + || closeout.observation_fingerprint !== jsonObject(writebackRun.progress_observation)?.fingerprint + || !isDeepStrictEqual(writebackRun.blocked_retry, closeout) + || !commit || Object.keys(commit).sort().join("|") !== ["schema_version", "request_sha256", + "original_receipt_sha256", "event"].sort().join("|") + || commit.schema_version !== "quota_execution_loss_closeout_commit_v0" + || commit.original_receipt_sha256 !== canonicalAuthoritySha256(heartbeatReceipt) + || commit.request_sha256 !== canonicalAuthoritySha256({identity: settlementIdentityPayload(identity), + goal_ref: quotaGoalRef(request.owner), intent: {delivery_outcome: "outcome_gap", delivery_boundary: "semantic_closeout", + progress_observation: writebackRun.progress_observation}}) + || !isDeepStrictEqual(jsonObject(commit.event)?.details, + {settlement_effect_id: identity.effect_id, execution_loss_closeout: closeout}) + || (writebackEvent !== null && !isDeepStrictEqual(writebackEvent, commit.event))) { + throw new EffectRuntimeRequestError("administrative execution-loss receipt differs from original admission", + "malformed_settlement_state"); + } + } const closeoutStarted = [writebackRun, writebackEvent, spendRun, spendEvent, completionEvent, supersedeEvent] .some((receipt) => receipt !== null); diff --git a/loopx/control_plane/work_items/execution_preparation.ts b/loopx/control_plane/work_items/execution_preparation.ts index a942b66431..2703f1e454 100644 --- a/loopx/control_plane/work_items/execution_preparation.ts +++ b/loopx/control_plane/work_items/execution_preparation.ts @@ -3,6 +3,19 @@ import {HANDOFF_MODE_RETIRED_REASON, retiredHandoffModeRejection} from "../coord import type {JsonObject} from "../effect_program.ts"; import {leaseEpoch} from "./task_lease_record.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; +import {createHash} from "node:crypto"; +import {isDeepStrictEqual} from "node:util"; +import {settlementIdentity, settlementIdentityPayload} from "../effect_program.ts"; +import {parseExactGoalRef} from "../goals/goal_instance_identity.ts"; +import {canonicalTaskLease, canonicalLeaseTodoFact} from "../coordination/task_lease_state.ts"; +import {decodeTaskLeaseProof} from "../coordination/task_lease_proof.ts"; +import {heartbeatPreparedLease, heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent} + from "../quota/heartbeat_receipt_identity.ts"; +import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import {projectProgressDelta} from "./progress_delta.ts"; +import {leaseRepositoryRejection} from "./task_lease_repository.ts"; +import {leaseOwnerRejection} from "./task_lease_eligibility.ts"; import { optionalNonEmptyString, requireBoolean, @@ -16,9 +29,137 @@ export const EXECUTION_PREPARATION_FACTS_SCHEMA_VERSION = "execution_preparation_facts_v0"; export const EXECUTION_PREPARATION_SCHEMA_VERSION = "execution_preparation_v0"; +export const EXECUTION_LOSS_CLOSEOUT_SCHEMA = "quota_execution_loss_closeout_v0"; +export const EXECUTION_LOSS_QUALIFICATION_SCHEMA = "execution_loss_closeout_qualification_v0"; const LEASE_STATUSES = ["active", "released", "unknown"] as const; +function rejectExecutionLoss(message: string): never { + throw new EffectRuntimeRequestError(`execution-loss closeout ${message}`, "refresh_task_lease_unavailable"); +} + +function executionLossClock(value: unknown): bigint { + if (typeof value !== "string") rejectExecutionLoss("requires an exact aware timestamp"); + const time = parseTodoTimestampMicros(value, true); + if (time === null) rejectExecutionLoss("has an invalid or ambiguous timestamp"); + return time; +} + +/** Pure qualification consumed only by the fenced administrative commit owner. + * A current lease proves control-writer ownership, never original execution. */ +export function qualifyReceiptBoundExecutionLossCloseout(value: unknown): JsonObject { + const request = requireJsonObject(value, "execution-loss qualification"); + if (request.schema_version !== EXECUTION_LOSS_QUALIFICATION_SCHEMA) + rejectExecutionLoss("qualification schema is unsupported"); + const binding = requireJsonObject(request.identity, "execution-loss identity"); + const identity = settlementIdentity({goal_id: requireNonEmptyString(binding.goal_id, "goal_id"), + agent_id: requireNonEmptyString(binding.agent_id, "agent_id"), todo_id: requireNonEmptyString(binding.todo_id, "todo_id"), + turn_instance_id: requireNonEmptyString(binding.turn_instance_id, "turn_instance_id")}); + if (identity.binding_kind !== "todo" || !identity.todo_id + || !isDeepStrictEqual(settlementIdentityPayload(identity), binding)) + rejectExecutionLoss("requires the exact original Todo Turn identity"); + const receipt = requireJsonObject(request.receipt, "original heartbeat receipt"); + if (request.goal_ref === undefined) { + if (Object.hasOwn(receipt, "goal_ref")) rejectExecutionLoss("cannot mix legacy and exact Goal ownership"); + } else { + const current = parseExactGoalRef(request.goal_ref), original = parseExactGoalRef(receipt.goal_ref); + if (current.kind !== "parsed" || original.kind !== "parsed" + || current.value.goalId.value !== identity.goal_id + || !isDeepStrictEqual(request.goal_ref, receipt.goal_ref)) + rejectExecutionLoss("requires the original exact Goal lifetime"); + } + const details = heartbeatReceiptDetails(receipt), fact = heartbeatReceiptFactFromEvent(receipt); + const originalBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); + const prepared = heartbeatPreparedLease(details); + if (receipt.schema_version !== "loopx_rollout_event_v0" || receipt.event_kind !== "quota_should_run" + || receipt.goal_id !== identity.goal_id || receipt.agent_id !== identity.agent_id + || receipt.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id + || originalBinding?.binding_kind !== "todo" || originalBinding.binding_id !== identity.todo_id + || fact.settlement_effect_id !== identity.effect_id || fact.closeout_required !== true + || !prepared || prepared.prepared_lease_epoch === undefined + || (Object.hasOwn(details, "prepared_lease_required") && details.prepared_lease_required !== true) + || (details.semantic_replan_obligation_id != null && details.semantic_replan_obligation_id !== "") + || nativeChildReportAdmission(details, receipt.status, identity.effect_id, "open", false) + .report_permission !== "new_operation") + rejectExecutionLoss("requires complete original prepared work admission"); + const todo = requireJsonObject(request.current_todo, "current Todo"); + const registered = request.registered_agents; + if (!Array.isArray(registered) || registered.some(agent => typeof agent !== "string") + || !registered.includes(identity.agent_id) || todo.todo_id !== identity.todo_id + || todo.role !== "agent" || todo.task_class !== "advancement_task" + || todo.archive_state !== "active" || todo.status !== "open" + || todo.claimed_by !== identity.agent_id || todo.done === true) + rejectExecutionLoss("requires the current registered control writer and unfinished Todo"); + if (leaseOwnerRejection(canonicalLeaseTodoFact(todo), identity.agent_id, registered as string[]) !== null) + rejectExecutionLoss("current control writer is not eligible for this Todo"); + const lease = canonicalTaskLease(requireJsonObject(request.current_lease, "current task lease"), + identity.goal_id, identity.todo_id); + const proof = decodeTaskLeaseProof(request.control_writer_proof); + if (!proof || leaseRepositoryRejection(todo, lease) !== null + || lease.owner !== identity.agent_id || lease.status !== "active" + || typeof lease.version !== "number" || !Number.isSafeInteger(lease.version) + || typeof lease.lease_epoch !== "number" || !Number.isSafeInteger(lease.lease_epoch) + || proof.idempotency_key !== lease.idempotency_key || proof.expected_version !== lease.version) + rejectExecutionLoss("requires the exact current control-writer proof"); + const now = executionLossClock(request.observed_at), admittedAt = executionLossClock(receipt.recorded_at); + const acquired = executionLossClock(lease.acquired_at), updated = executionLossClock(lease.updated_at); + const expires = executionLossClock(lease.expires_at); + if (admittedAt > acquired || acquired > updated || updated > now || expires <= now || expires <= updated) + rejectExecutionLoss("current control-writer clock is stale or ambiguous"); + const currentHash = createHash("sha256").update(proof.idempotency_key).digest("hex"); + if (lease.lease_epoch <= prepared.prepared_lease_epoch || lease.version <= prepared.prepared_lease_version + || currentHash === prepared.prepared_lease_key_sha256) + rejectExecutionLoss("cannot replace an available or ambiguous original generation"); + const progress = requireJsonObject(request.progress_observation, "blocked observation"); + if (progress.result_class !== "blocked" || progress.work_item_id !== identity.todo_id + || typeof progress.blocker_id !== "string" || !Array.isArray(progress.evidence_ids) + || progress.evidence_ids.length === 0 + || Object.keys(progress).some(field => !["schema_version", "result_class", "work_item_id", + "blocker_id", "evidence_ids", "fingerprint"].includes(field))) + rejectExecutionLoss("requires only verified blocked observation and evidence"); + projectProgressDelta({current: progress, baseline: null, window: []}); + return {schema_version: EXECUTION_LOSS_CLOSEOUT_SCHEMA, effect_id: identity.effect_id, + todo_id: identity.todo_id, source: "turn_settlement", observed_at: request.observed_at, + original_receipt_event_id: requireNonEmptyString(receipt.event_id, "original receipt event id"), + original_prepared: {...prepared}, current_control_writer: { + lease_key_sha256: currentHash, version: lease.version, lease_epoch: lease.lease_epoch, + owner: lease.owner, store_identity: requireNonEmptyString(request.provider_identity, "provider identity"), + provider_revision: requireNonEmptyString(request.provider_revision, "provider revision")}, + observation_fingerprint: progress.fingerprint}; +} + +/** Historical readback validates the frozen qualification, not today's lease. */ +export function isExecutionLossCloseout(value: unknown, todoId: string | null): boolean { + const closeout = typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonObject : null; + if (!closeout || closeout.schema_version !== EXECUTION_LOSS_CLOSEOUT_SCHEMA || !todoId + || closeout.todo_id !== todoId || closeout.source !== "turn_settlement" + || Object.keys(closeout).sort().join("|") !== ["schema_version", "effect_id", "todo_id", "source", + "observed_at", "original_receipt_event_id", "original_prepared", "current_control_writer", + "observation_fingerprint"].sort().join("|")) return false; + try { + const original = requireJsonObject(closeout.original_prepared, "original preparation"); + const prepared = heartbeatPreparedLease(original); + const writer = requireJsonObject(closeout.current_control_writer, "control-writer witness"); + executionLossClock(closeout.observed_at); + return !!prepared && prepared.prepared_lease_epoch !== undefined + && isDeepStrictEqual(original, prepared) + && Object.keys(writer).sort().join("|") === ["lease_key_sha256", "version", "lease_epoch", "owner", + "store_identity", "provider_revision"].sort().join("|") + && typeof closeout.effect_id === "string" && !!closeout.effect_id + && /^[a-f0-9]{64}$/u.test(String(writer.lease_key_sha256)) + && writer.lease_key_sha256 !== prepared.prepared_lease_key_sha256 + && typeof writer.version === "number" && Number.isSafeInteger(writer.version) + && writer.version > prepared.prepared_lease_version + && typeof writer.lease_epoch === "number" && Number.isSafeInteger(writer.lease_epoch) + && writer.lease_epoch > prepared.prepared_lease_epoch + && typeof closeout.original_receipt_event_id === "string" && !!closeout.original_receipt_event_id + && typeof writer.owner === "string" && !!writer.owner + && typeof writer.store_identity === "string" && !!writer.store_identity + && typeof writer.provider_revision === "string" && !!writer.provider_revision + && /^[a-f0-9]{16}$/u.test(String(closeout.observation_fingerprint)); + } catch {return false;} +} + function leaseFact(value: unknown, label: string): JsonObject { const lease = requireJsonObject(value, label); const status = requireStringLiteral(lease.status, LEASE_STATUSES, `${label}.status`); diff --git a/loopx/quota.py b/loopx/quota.py index e3477dc065..022d8913cb 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1387,10 +1387,7 @@ def spend_quota_slot( ) spent_result = settlement_readback.settlement if spent_result.failure is None: - no_spend = ( - _as_dict(settlement_readback.progress).get("closeout_kind") - == "capability_duty_retired_no_spend" - ) + no_spend = _as_dict(settlement_readback.progress).get("no_spend") is True return _spend_slot_reply( ok=True, execute=execute, goal_id=safe_goal_id, goal_ref=goal_ref, identity=identity, settlement_result=spent_result, idempotent_replay=True, diff --git a/loopx/semantics/project_registry_io_classifications_v1.json b/loopx/semantics/project_registry_io_classifications_v1.json index 3ab5f90fc5..32f7b2331b 100644 --- a/loopx/semantics/project_registry_io_classifications_v1.json +++ b/loopx/semantics/project_registry_io_classifications_v1.json @@ -5,6 +5,8 @@ "loopx/control_plane/goals/deletion_service.py::._restore_locked_registry::direct_json_write:atomic_write_json#1": "global_registry_io", "loopx/control_plane/goals/deletion_service.py::._write_deletion::direct_json_read:read_json#1": "global_registry_io", "loopx/control_plane/goals/deletion_service.py::._write_locked_registry::direct_json_write:atomic_write_json#1": "global_registry_io", + "loopx/control_plane/quota/execution_loss_closeout.ts::::direct_json_read:readFileSync#1": "codec_internal", + "loopx/control_plane/quota/execution_loss_closeout.ts::::direct_json_read:readFileSync#2": "codec_internal", "loopx/control_plane/scheduler/heartbeat_commit.ts::readCurrentGoalInstance::direct_json_read:readFile#1": "codec_internal", "loopx/state_migration.py::.legacy_registry_goal_ids::direct_json_read:read_json_object#1": "legacy_registry_source", "loopx/state_migration.py::.migrate_legacy_state::direct_json_read:read_json_object#1": "legacy_registry_source" diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index 28d6e10900..943c3dba88 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -759,7 +759,13 @@ def refresh_state_run( goal_ref: dict[str, str] | None = None, goal_acceptance_preview: dict[str, Any] | None = None, task_lease_proof: dict[str, Any] | None = None, + execution_loss_closeout: bool = False, + execution_loss_requested_effects: Mapping[str, object] | None = None, ) -> dict[str, Any]: + if execution_loss_closeout: + from .control_plane.quota.execution_preparation import commit_execution_loss_closeout_from_refresh + # Explicit administrative entry bypasses every ordinary refresh hook. + return commit_execution_loss_closeout_from_refresh(locals()) from .control_plane.todos.provider_projection import recover_refresh_todo_projection safe_goal_id = validate_goal_id_path_segment(goal_id) diff --git a/tests/control_plane/test_execution_loss_closeout_cli.py b/tests/control_plane/test_execution_loss_closeout_cli.py new file mode 100644 index 0000000000..3860701228 --- /dev/null +++ b/tests/control_plane/test_execution_loss_closeout_cli.py @@ -0,0 +1,712 @@ +"""A lost execution generation can close only its original Turn, without spend.""" +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +import json +from pathlib import Path +import subprocess +import sys + +import pytest +import loopx +from loopx.agent_registry import registered_agent_ids_for_goal + +from canonical_authority_fixture import initialize_canonical_source_fixture, isolate_sqlite_runtime +from codex_checkpoint_fixture import read_cli_policy_payload +from loopx.cli import build_parser, main as cli_main +from loopx.cli_commands.project_lifecycle_inputs import inline_progress_observation +from loopx.control_plane.coordination import local_authority +from loopx.control_plane.effect_runtime import effect_runtime_result +from loopx.control_plane.goals.source_session_registration import ( + FreshSourceSessionRegistration, + configure_source_creation_goal, + register_fresh_source_session_project, +) +from loopx.control_plane.quota import settlement +from loopx.control_plane.projects.registry_codec import load_project_registry, ProjectRegistryProtocolError +from loopx.control_plane.quota.effect_program import SettlementIdentity +from loopx.control_plane.quota.execution_preparation import prepared_lease_receipt_fields +from loopx.control_plane.quota.heartbeat_receipt import ( + HeartbeatReceiptIdentityConflictError, + ensure_turn_heartbeat_settlement_receipt, + requalify_bound_heartbeat_receipt, + retain_pending_heartbeat_action_selection, + upgrade_identityless_heartbeat_receipt, +) +from loopx.control_plane.runtime import time as runtime_time +from loopx.control_plane.todos.completion_validation_projection import ( + completion_validation_declaration, + project_completion_validation_authority, +) +from loopx.control_plane.todos.completion_validation_store import ( + persist_completion_validation_declaration, + prepare_completion_validation_declaration, +) +from loopx.control_plane.work_items import task_lease +from loopx.control_plane.work_items.progress_observation import normalize_progress_observation +from loopx.rollout_event_log import append_rollout_event, build_rollout_event, rollout_event_log_path +from tests.test_attached_session_goal_instance import _register +from tests.control_plane.test_explore_result_writeback import ATTACHMENT +from tests.control_plane.test_quota_settlement_cli import _write_fixture as write_legacy_fixture + + +GOAL = "execution-loss-closeout" +OWNER = "original-worker" +PEER = "replacement-worker" +TURN = "original-prepared-turn" + + +def _registered_agents(context): + goal = next(goal for goal in load_project_registry(context["registry"])["goals"] + if goal["id"] == context["goal_id"]) + return registered_agent_ids_for_goal(goal) + + +def _source_transaction(context, method, schema, *, operation_id, **intent): + registry_digest = sha256(context["registry"].read_bytes()).hexdigest() + result = effect_runtime_result(method, { + "schema_version": schema, "runtime_root": str(context["runtime"].resolve()), + "goal_id": context["goal_id"], "operation_id": operation_id, + "expected_provider_revision": _canonical(context)["provider_revision"], + "expected_registry_sha256": registry_digest, + "registry_source": {"path": str(context["registry"].resolve()), "sha256": registry_digest}, + "registered_agents": _registered_agents(context), "dry_run": False, + "observed_at": runtime_time.now_local_iso(), **intent, + }) + assert result["status"] == "applied", result + assert result["decision_read_from_provider"] is True + assert result["legacy_fallback_used"] is False + return result + + +def _source_create_todo(context, validation_argv): + operation = "execution-loss-task" + todo_id = "todo_" + sha256(operation.encode()).hexdigest()[:24] + fields = {"schema_version": "todo_domain_record_v0", "todo_id": todo_id, + "role": "agent", "text": "Validate the original task before declaring completion.", + "status": "open", "done": False, "archive_state": "active", + "task_class": "advancement_task", "action_kind": "validate", + "continuation_policy": "same_agent_non_delivery", "claimed_by": OWNER, + "validation_command_argv": validation_argv} + declaration = completion_validation_declaration(fields) + assert declaration is not None + prepare_completion_validation_declaration(runtime_root=context["runtime"], + goal_id=context["goal_id"], declaration=declaration) + result = _source_transaction(context, "coordination.local_authority.todo_create", + "loopx_local_coordination_todo_create_request_v3", operation_id=operation, + actor_agent_id=OWNER, todo=project_completion_validation_authority(fields)) + assert result["todo_id"] == todo_id + assert result["todo"]["completion_validation_required"] is True + persist_completion_validation_declaration(runtime_root=context["runtime"], + goal_id=context["goal_id"], todo_id=todo_id, declaration=declaration) + return todo_id + + +def _claim(context, owner, operation): + if context["source_profile"]: + return _source_transaction(context, "coordination.local_authority.todo_claim", + "loopx_local_coordination_todo_claim_request_v3", operation_id=operation, + todo_id=context["todo"], role="agent", actor_agent_id=owner, claimed_by=owner, + manage_execution=True, lease_request=None) + code, claimed = context["cli"]("todo", "claim", "--goal-id", context["goal_id"], + "--todo-id", context["todo"], "--agent-id", owner, "--claimed-by", owner) + assert code == 0, claimed + return claimed + + +def _source_reservation(context): + # SourceSession's generic work producer is intentionally unavailable. Model + # stored reservation history through the existing append/binding owners; + # TS alone derives work_admission and closeout_required below. + fields = {"ok": True, "should_run": False, "must_attempt_work": False, + "delivery_allowed": False, "quiet_noop_allowed": True, + "prepared_lease_required": True, "semantic_replan_obligation_id": "", + "stall_observation": "not_applicable"} + reserved = append_rollout_event(rollout_event_log_path(context["runtime"], context["goal_id"]), + build_rollout_event(goal_id=context["goal_id"], goal_ref=context["goal_ref"], + event_kind="quota_should_run", agent_id=OWNER, run_id=TURN, + status="turn_reservation", details=fields)) + retained, appended = retain_pending_heartbeat_action_selection(context["runtime"], + goal_id=context["goal_id"], goal_ref=context["goal_ref"], agent_id=OWNER, + turn_instance_id=TURN, todo_id=context["todo"], reason="claim_required") + assert appended is True and retained["event_id"] != reserved["event_id"] + identity = SettlementIdentity(goal_id=context["goal_id"], agent_id=OWNER, + todo_id=context["todo"], turn_instance_id=TURN) + bound, appended = upgrade_identityless_heartbeat_receipt(context["runtime"], + goal_id=context["goal_id"], goal_ref=context["goal_ref"], agent_id=OWNER, + turn_instance_id=TURN, todo_id=context["todo"], settlement_effect_id=identity.effect_id, + status="turn_reservation", summary="Retain the original Todo while execution is prepared.", details=fields) + assert appended is True and bound["details"]["settlement_receipt_revision"] == "identity_upgrade" + ensured = ensure_turn_heartbeat_settlement_receipt(context["runtime"], identity, + semantic_replan_guard_scoped=True, semantic_replan_obligation_id=None, + prepared_lease_required=True, goal_ref=context["goal_ref"]) + assert ensured == bound + context["reservation_details"] = {key: value for key, value in bound["details"].items() + if key not in {"settlement_receipt_revision", "closeout_required"}} + + +def _source_requalify(context, lease): + fields = {**context["reservation_details"], "ok": True, "should_run": True, + "must_attempt_work": True, "delivery_allowed": True, "quiet_noop_allowed": False, + "selected_todo_id": context["todo"]} + if lease is not None: + fields.update(prepared_lease_receipt_fields({"idempotency_key": lease["idempotency_key"], + "expected_version": lease["version"]}, lease_epoch=lease["lease_epoch"])) + return requalify_bound_heartbeat_receipt(context["runtime"], + goal_id=context["goal_id"], goal_ref=context["goal_ref"], agent_id=OWNER, + turn_instance_id=TURN, todo_id=context["todo"], replan_obligation_id=None, + status="normal_run", details=fields, registry_path=context["registry"]) + + +def _fixture(tmp_path, monkeypatch, capsys, provider, *, prepared=True, + replace=True, replacement_owner=OWNER, source_profile=True): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime = tmp_path / "project", tmp_path / "runtime" + registry = project / ".loopx" / "registry.json" + state = project / "GOAL.md" + if source_profile: + project.mkdir() + goal = {"id": GOAL, "project_id": "project", "title": "Recover original Turn", + "status": "active", "repo": str(project), "state_file": state.name} + configure_source_creation_goal(goal, [OWNER, PEER], provider, "register-execution-loss-goal") + created = register_fresh_source_session_project(FreshSourceSessionRegistration( + registry_path=registry, runtime_root=runtime, + operation_id="register-execution-loss-goal", project_id="project", goal_id=GOAL, + objective="Retain the original work while closing its interrupted Turn.", + non_goals=["Do not complete the task or credit delivery."], + acceptance=["Original Turn recovery preserves the current work owner."], + unknowns=[], next_effect="Inspect the interrupted execution.", + stop_condition="Stop when the original execution authority is unavailable.", + project_record={"id": "project", "kind": "work", "path": str(project)}, + goal_record=goal, state_file=state, + )) + assert created["ok"] is True, created + goal_ref, goal_id = created["goal_ref"], GOAL + else: + project, runtime, registry = write_legacy_fixture(tmp_path) + configuration = json.loads(registry.read_text()) + goal = configuration["goals"][0] + goal["coordination"]["registered_agents"] = [OWNER, PEER] + registry.write_text(json.dumps(configuration)) + goal_id, goal_ref = goal["id"], None + state = project / goal["state_file"] + state.write_text(state.read_text().split("## Agent Todo")[0] + "## Agent Todo\n\n") + initialize_canonical_source_fixture( + runtime, goal_id, state_path=state, provider=provider, handoff_mode="hard_lease", + ) + + def cli(*argv): + # This callable always exercises the real public parser and dispatcher. + with monkeypatch.context() as context: + context.chdir(project) + code = cli_main([ + "--registry", str(registry), "--runtime-root", str(runtime), + "--format", "json", *argv, + ]) + return code, read_cli_policy_payload(capsys.readouterr().out, returncode=code) + + context = {"cli": cli, "project": project, "runtime": runtime, "registry": registry, + "state": state, "goal_id": goal_id, "goal_ref": goal_ref, "source_profile": source_profile} + validation_argv = [sys.executable, "-c", + "from pathlib import Path; Path('completion-validator-ran').write_text('ran')"] + if source_profile: + todo = _source_create_todo(context, validation_argv) + else: + code, added = cli("todo", "add", "--goal-id", goal_id, "--role", "agent", + "--text", "Validate the original task before declaring completion.", + "--action-kind", "validate", "--continuation-policy", "same_agent_non_delivery", + "--claimed-by", OWNER, "--operation-id", "execution-loss-task", "--execute", + "--validation-command-json", json.dumps(validation_argv)) + assert code == 0, json.dumps(added, ensure_ascii=False) + todo = added["todo_id"] + context["todo"] = todo + if source_profile: + _source_reservation(context) + if not prepared: + if source_profile: + pending, appended = _source_requalify(context, None) + assert appended is True and "prepared_lease_key_sha256" not in pending["details"] + else: + code, pending = cli(*_guard_args(context)) + assert code == 0 and pending["execution_preparation"]["state"] == "claim_required", pending + claim = _claim(context, OWNER, "execution-loss-original-claim") + original_lease = _lease(context) + context["original_lease"] = original_lease + if prepared: + if source_profile: + admitted, appended = _source_requalify(context, original_lease) + assert appended is True + assert admitted["details"]["settlement_receipt_revision"] == "work_admission" + assert admitted["details"]["closeout_required"] is True + assert admitted["details"]["prepared_lease_epoch"] == original_lease["lease_epoch"] + else: + code, admitted = cli(*_guard_args(context)) + assert code == 0 and admitted["normal_delivery_allowed"] is True, admitted + assert admitted["heartbeat_receipt"]["closeout_required"] is True + assert admitted["execution_preparation"]["lease_epoch"] == original_lease["lease_epoch"] + context["admitted"] = admitted + context["receipt_lines"] = _receipt_lines(context) + assert context["receipt_lines"], "the original receipt must be production-generated" + if replace: + after_expiry = datetime.fromisoformat( + original_lease["expires_at"].replace("Z", "+00:00"), + ).replace(microsecond=0) + timedelta(seconds=2) + + class AfterExpiryDatetime(datetime): + @classmethod + def now(cls, tz=None): + return (after_expiry.astimezone(tz) if tz + else after_expiry.astimezone().replace(tzinfo=None)) + + # Registered transport clock seams, never edits to persisted lease bytes. + monkeypatch.setattr(runtime_time, "datetime", AfterExpiryDatetime) + monkeypatch.setattr(local_authority, "now_local", lambda: after_expiry.isoformat()) + monkeypatch.setattr(task_lease, "now_utc", lambda: after_expiry.astimezone(timezone.utc)) + assert runtime_time.now_utc() == after_expiry.astimezone(timezone.utc) + replacement = _claim(context, replacement_owner, "execution-loss-replacement-claim") + assert replacement["claim_operation_id"] != claim["claim_operation_id"] + current = _lease(context) + assert current["owner"] == replacement_owner + assert current["lease_epoch"] > original_lease["lease_epoch"] + assert current["version"] > original_lease["version"] + assert current["idempotency_key"] != original_lease["idempotency_key"] + context["current_lease"] = _lease(context) + assert _receipt_lines(context) == context["receipt_lines"] + return context + + +def _binding(context): + return ["--goal-id", context["goal_id"], + *(["--goal-instance-id", context["goal_ref"]["goal_instance_id"]] if context["goal_ref"] else []), + "--agent-id", OWNER, "--todo-id", context["todo"], "--turn-instance-id", TURN] + + +def _guard_args(context): + return ["quota", "should-run", "--codex-app", "--context-delivery", "off", *_binding(context)] + + +def _proof(lease): + return ["--task-lease-idempotency-key", lease["idempotency_key"], + "--task-lease-expected-version", str(lease["version"])] + + +def _closeout_args(context, *, lease=None, execution_loss=True): + return ["refresh-state", *_binding(context), + *(["--execution-loss-closeout"] if execution_loss else []), + "--classification", "execution_loss_closeout", "--delivery-batch-scale", "single_surface", + "--delivery-outcome", "outcome_gap", "--delivery-boundary", "semantic_closeout", + "--progress-result-class", "blocked", "--progress-work-item-id", context["todo"], + "--progress-blocker-id", "original-execution-lost", + "--progress-evidence-id", "evidence:original-lease-expired", + *(["--vision-unchanged-reason", "The original work remains open; close only its interrupted Turn."] + if not execution_loss else []), + "--no-global-sync", "--suppress-external-sinks", + *_proof(lease or context["current_lease"])] + + +def _lease(context): + if context["source_profile"]: + inspected = effect_runtime_result("task_lease.inspect.native", { + "schema_version": "loopx_task_lease_inspect_request_v0", "source": "canonical", + "runtime_root": str(context["runtime"].resolve()), "goal_id": context["goal_id"], + "todo_id": context["todo"], "authority": {"handoff_mode": "hard_lease", "todos": [], + "registered_agent_candidates": [_registered_agents(context)], + "source_receipts": [{"source_id": "registry", "path": str(context["registry"].resolve()), + "state": "file", "sha256": sha256(context["registry"].read_bytes()).hexdigest()}]}, + }) + assert inspected["ok"] is True and inspected["source_authority"] in {"file_v0", "sqlite_v0"}, inspected + assert inspected["provider_revision"] == _canonical(context)["provider_revision"] + assert inspected["legacy_fallback_used"] is False + assert isinstance(inspected["lease"], dict) + return inspected["lease"] + code, inspected = context["cli"]( + "task-lease", "inspect", "--goal-id", context["goal_id"], "--todo-id", context["todo"], + ) + assert code == 0, inspected + assert isinstance(inspected["lease"], dict) + return inspected["lease"] + + +def _readback(context): + return settlement.read_heartbeat_settlement( + context["runtime"], goal_id=context["goal_id"], agent_id=OWNER, todo_id=context["todo"], + turn_instance_id=TURN, registry_path=context["registry"], goal_ref=context["goal_ref"], + ) + + +def _receipt_lines(context): + path = rollout_event_log_path(context["runtime"], context["goal_id"]) + if not path.exists(): + return [] + return [line for line in path.read_bytes().splitlines(keepends=True) + if (event := json.loads(line)).get("event_kind") == "quota_should_run" + and event.get("run_id") == TURN + and (event.get("goal_ref") == context["goal_ref"] if context["goal_ref"] else "goal_ref" not in event)] + + +def _runs(context): + directory = context["runtime"] / "goals" / context["goal_id"] / "runs" + return {path.name: path.read_bytes() for path in directory.glob("*.json")} + + +def _canonical(context): + payload = local_authority.read_canonical_todos_if_promoted( + runtime_root=context["runtime"], goal_id=context["goal_id"], include_leases=True, + ) + assert payload is not None + return {key: payload[key] for key in ("store_identity", "provider_revision", "todos", "leases")} + + +def _unchanged_state(context): + log = rollout_event_log_path(context["runtime"], context["goal_id"]) + return (_canonical(context), context["state"].read_bytes(), _runs(context), log.read_bytes()) + + +def _assert_no_completion_or_debit(context): + if context["source_profile"]: + todo = next(row for row in _canonical(context)["todos"] if row["todo_id"] == context["todo"]) + else: + code, listed = context["cli"]("todo", "list", "--goal-id", context["goal_id"], "--todo-id", context["todo"]) + assert code == 0, listed + todo = listed["todo"] + assert todo["status"] == "open" and todo.get("completion_validation_required") is True + assert not (context["project"] / "completion-validator-ran").exists() + index = context["runtime"] / "goals" / context["goal_id"] / "runs" / "index.jsonl" + records = [json.loads(line) for line in index.read_text().splitlines()] if index.exists() else [] + assert not any(record.get("classification") == "quota_slot_spent" for record in records) + + +def _assert_original_prepared_generation(context, readback): + original, current = context["original_lease"], context["current_lease"] + assert readback.prepared_lease == { + "prepared_lease_key_sha256": sha256(original["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": original["version"], + "prepared_lease_epoch": original["lease_epoch"], + } + if current["lease_epoch"] != original["lease_epoch"]: + assert readback.prepared_lease["prepared_lease_key_sha256"] != sha256( + current["idempotency_key"].encode(), + ).hexdigest() + + +def _reject_without_effect(context, argv, *, expected_code=None): + before = _unchanged_state(context) + code, rejected = context["cli"](*argv) + assert code != 0 and rejected["ok"] is False, rejected + if expected_code is not None: + assert rejected.get("error_code") == expected_code, json.dumps(rejected, ensure_ascii=False) + assert _unchanged_state(context) == before + _assert_no_completion_or_debit(context) + return rejected + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_direct_refresh_sparse_effect_map_cannot_hide_business_requests( + tmp_path, monkeypatch, capsys, provider, +): + from loopx.state_refresh import refresh_state_run + + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False) + args = build_parser().parse_args(_closeout_args(context)) + request = { + "registry_path": context["registry"], "runtime_root_override": str(context["runtime"]), + "goal_id": context["goal_id"], "project": None, "state_file": None, + "classification": "execution_loss_closeout", "recommended_action": None, + "agent_id": OWNER, "todo_id": context["todo"], "turn_instance_id": TURN, + "delivery_outcome": "outcome_gap", "delivery_boundary": "semantic_closeout", + "progress_observation": normalize_progress_observation(inline_progress_observation(args)), + "task_lease_proof": {"idempotency_key": context["current_lease"]["idempotency_key"], + "expected_version": context["current_lease"]["version"]}, + "execution_loss_closeout": True, "execution_loss_requested_effects": {}, "dry_run": False, + "sync_global": False, + } + before = _unchanged_state(context) + for ordinary in ( + {"recommended_action": "Continue business work."}, + {"next_action": "Continue business work."}, + {"agent_vision_packet": {"state": "active"}}, + {"vision_unchanged_reason": "Record a normal checkpoint."}, + {"explore_result": ATTACHMENT}, + {"completion_todo_id": context["todo"]}, + {"usage_measurement": {"input_tokens": 1}}, + {"usage_codex_session": tmp_path / "private-session.jsonl"}, + {"project": context["project"]}, + {"state_file": context["state"]}, + {"delivery_workspace_path": tmp_path / "business-workspace"}, + {"external_delivery": {"suppress": True}}, + {"sync_global": True}, + ): + with pytest.raises(ValueError) as rejected: + refresh_state_run(**{**request, **ordinary}) + assert getattr(rejected.value, "code", None) == "refresh_task_lease_unavailable", ordinary + assert _unchanged_state(context) == before + _assert_no_completion_or_debit(context) + default_sync = dict(request) + default_sync.pop("sync_global") + with pytest.raises(ValueError) as rejected: + refresh_state_run(**default_sync) + assert getattr(rejected.value, "code", None) == "refresh_task_lease_unavailable" + assert _unchanged_state(context) == before + closed = refresh_state_run(**request) + assert closed["appended"] is True and closed["settlement_progress"]["state"] == "settled" + _assert_no_completion_or_debit(context) + + +def _replace_argument(argv, flag, value): + result = list(argv) + result[result.index(flag) + 1] = value + return result + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("source_profile", [True, False], ids=["source-session", "legacy-alias"]) +def test_current_control_writer_closes_lost_original_turn_without_business_authority( + tmp_path, monkeypatch, capsys, provider, source_profile, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=source_profile) + original = _readback(context) + assert original is not None and original.identity.failure is None + assert not original.writeback.receipts and not original.spend.receipts + _assert_original_prepared_generation(context, original) + if source_profile: + before_requalification = _unchanged_state(context) + with pytest.raises(HeartbeatReceiptIdentityConflictError, + match="work requalification must preserve this Turn's original task lease generation"): + _source_requalify(context, context["current_lease"]) + assert _unchanged_state(context) == before_requalification + _assert_no_completion_or_debit(context) + else: + rejected = _reject_without_effect(context, _guard_args(context), + expected_code="heartbeat_receipt_identity_conflict") + pure = rejected["original_settlement_readback"] + assert pure["settlement_identity"] == original.identity.value.as_dict() + assert pure["settlement_identity"]["todo_id"] == context["todo"] + assert pure["settlement_identity"]["turn_instance_id"] == TURN + assert pure["prepared_lease"] == original.prepared_lease + assert pure["settlement_progress"]["state"] == "writeback_required" + assert "refresh-state --execution-loss-closeout" in rejected["recommended_action"] + assert "Rerun quota should-run" not in rejected["recommended_action"] + # Valid blocked intent reaches the original first-writeback fence; + # neither the expired proof nor its replacement may pass that gate. + for lease in (context["original_lease"], context["current_lease"]): + _reject_without_effect(context, _closeout_args(context, lease=lease, execution_loss=False), + expected_code="refresh_task_lease_unavailable") + before_authority, before_state, before_runs = ( + _canonical(context), context["state"].read_bytes(), _runs(context), + ) + argv = _closeout_args(context) + code, closed = context["cli"](*argv) + assert code == 0 and closed["appended"] is True, closed + assert closed["settlement_identity"] == original.identity.value.as_dict() + assert closed["settlement_progress"]["state"] == "settled" + assert closed["settlement_progress"]["closeout_kind"] == "typed_blocked_writeback_no_spend" + assert "settlement_owed" not in closed + assert _canonical(context) == before_authority + assert context["state"].read_bytes() == before_state + assert _receipt_lines(context) == context["receipt_lines"] + after_runs = _runs(context) + assert {key: after_runs[key] for key in before_runs} == before_runs + new_names = after_runs.keys() - before_runs.keys() + assert len(new_names) == 1 + run = json.loads(after_runs[new_names.pop()]) + assert run["progress_observation"]["result_class"] == "blocked" + assert run["delivery_outcome"] == "outcome_gap" + assert run["execution_loss_closeout"]["schema_version"] == "quota_execution_loss_closeout_v0" + qualification = run["execution_loss_closeout"] + assert qualification["original_prepared"] == original.prepared_lease + current = context["current_lease"] + assert qualification["current_control_writer"] == { + "lease_key_sha256": sha256(current["idempotency_key"].encode()).hexdigest(), + "version": current["version"], "lease_epoch": current["lease_epoch"], "owner": OWNER, + "store_identity": before_authority["store_identity"], + "provider_revision": before_authority["provider_revision"], + } + for forbidden in ("agent_vision", "autonomous_replan_ack", "explore_result", "usage"): + assert forbidden not in run + settled = _readback(context) + assert settled.replay_phase == settlement.ReceiptBoundReplayPhase.SETTLED + assert settled.prepared_lease == original.prepared_lease + _assert_original_prepared_generation(context, settled) + assert settled.heartbeat_receipt == original.heartbeat_receipt + assert settled.completion_event is None + assert _lease(context) == context["current_lease"] + _assert_no_completion_or_debit(context) + before_replay = _unchanged_state(context) + code, replay = context["cli"](*argv) + assert code == 0 and replay["idempotent_replay"] is True and replay["appended"] is False, replay + assert replay["settlement_result"] == closed["settlement_result"] + assert _unchanged_state(context) == before_replay + if not source_profile: + code, spent = context["cli"]( + "quota", "spend-slot", *_binding(context), "--slots", "1", "--source", "heartbeat", "--execute", + ) + assert code == 0 and spent["ok"] is True and spent["appended"] is False, spent + assert spent["settlement_progress"]["closeout_kind"] == "typed_blocked_writeback_no_spend" + assert spent["settlement_progress"]["no_spend"] is True + assert spent["reason"] == "exact Turn closeout replayed without a quota debit" + assert _unchanged_state(context) == before_replay + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_original_turn_rejects_stale_missing_or_foreign_current_writer_proof( + tmp_path, monkeypatch, capsys, provider, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider) + argv = _closeout_args(context) + no_proof = argv[:-4] + for candidate, expected in ( + (_closeout_args(context, lease=context["original_lease"]), "refresh_task_lease_unavailable"), + # The shared proof decoder rejects an absent wire before lease matching. + (no_proof, "invalid_request"), + (_replace_argument(argv, "--task-lease-expected-version", str(context["current_lease"]["version"] - 1)), + "refresh_task_lease_unavailable"), + (_replace_argument(argv, "--task-lease-idempotency-key", "foreign-control-writer"), + "refresh_task_lease_unavailable"), + ): + _reject_without_effect(context, candidate, expected_code=expected) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("source_profile", [True, False], ids=["source-session", "legacy-alias"]) +def test_replacement_owner_cannot_close_another_agents_original_turn( + tmp_path, monkeypatch, capsys, provider, source_profile, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, + replacement_owner=PEER, source_profile=source_profile) + assert context["current_lease"]["owner"] == PEER + _reject_without_effect(context, _closeout_args(context), expected_code="refresh_task_lease_unavailable") + foreign_binding = _replace_argument(_closeout_args(context), "--agent-id", PEER) + _reject_without_effect(context, foreign_binding) + + +def test_closeout_rejects_wrong_identity_and_extra_work_intent(tmp_path, monkeypatch, capsys): + context = _fixture(tmp_path, monkeypatch, capsys, "file") + argv = _closeout_args(context) + other_source = tmp_path / "other-source" + other_source.mkdir() + _other_registry, _other_runtime, _other_store, other_instance = _register(other_source) + assert other_instance != context["goal_ref"]["goal_instance_id"] + # One provider is enough for CLI identity/intent transport; both storage owners + # independently qualify the control-writer and missing-readiness cases above/below. + identities = ( + ("--goal-id", "another-goal"), ("--goal-instance-id", other_instance), + ("--agent-id", PEER), ("--todo-id", "todo_another"), ("--turn-instance-id", "another-turn"), + ) + for flag, value in identities: + _reject_without_effect(context, _replace_argument(argv, flag, value)) + for extra in ( + ["--next-action", "Continue replacement business work."], + ["--explore-result-json", json.dumps(ATTACHMENT)], + ["--progress-surface-id", "replacement-business-surface"], + ["--progress-hypothesis-id", "replacement-business-hypothesis"], + ["--completion-todo-id", context["todo"]], + ["--vision-state", "active", "--vision-summary", "Change the original work route."], + ): + _reject_without_effect(context, [*argv, *extra]) + progress = _replace_argument(argv, "--progress-result-class", "advanced") + _reject_without_effect(context, progress) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_current_original_generation_keeps_ordinary_blocked_closeout_semantics( + tmp_path, monkeypatch, capsys, provider, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, replace=False, source_profile=False) + assert context["current_lease"] == context["original_lease"] + _reject_without_effect(context, _closeout_args(context), expected_code="refresh_task_lease_unavailable") + before = _canonical(context) + before_runs = _runs(context) + code, closed = context["cli"](*_closeout_args(context, execution_loss=False)) + assert code == 0, closed + assert "quota_execution_loss_closeout_v0" not in json.dumps(closed) + after_runs = _runs(context) + new_names = after_runs.keys() - before_runs.keys() + assert len(new_names) == 1 + run = json.loads(after_runs[new_names.pop()]) + assert "execution_loss_closeout" not in run + assert run["blocked_retry"]["schema_version"] == "quota_blocked_retry_v0" + assert _canonical(context) == before + assert _lease(context) == context["original_lease"] + assert _receipt_lines(context) == context["receipt_lines"] + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_source_profile_keeps_ordinary_public_refresh_restriction(tmp_path, monkeypatch, capsys, provider): + context = _fixture(tmp_path, monkeypatch, capsys, provider, replace=False) + before = _unchanged_state(context) + with pytest.raises(ProjectRegistryProtocolError, + match="^generic registry read rejects lifecycle-only profile source_session_v1; " + "use project lifecycle commands$"): + context["cli"](*_closeout_args(context, execution_loss=False)) + capsys.readouterr() + assert _unchanged_state(context) == before + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_unprepared_original_receipt_cannot_certify_execution_loss(tmp_path, monkeypatch, capsys, provider): + context = _fixture(tmp_path, monkeypatch, capsys, provider, prepared=False) + readback = _readback(context) + assert readback is not None and readback.prepared_lease_required is True + assert readback.prepared_lease is None + assert not readback.writeback.receipts and not readback.spend.receipts + _reject_without_effect(context, _closeout_args(context), expected_code="refresh_task_lease_unavailable") + + +def test_malformed_original_prepared_receipt_or_mixed_profile_cannot_certify_execution_loss( + tmp_path, monkeypatch, capsys, +): + context = _fixture(tmp_path, monkeypatch, capsys, "sqlite") + before = _unchanged_state(context) + readback = _readback(context) + source = _canonical(context) + receipt = json.loads(context["receipt_lines"][-1]) + assert receipt == readback.heartbeat_receipt + args = build_parser().parse_args(_closeout_args(context)) + pure_input = { + "schema_version": "execution_loss_closeout_qualification_v0", + "receipt": receipt, + "current_todo": next(todo for todo in source["todos"] if todo["todo_id"] == context["todo"]), + "current_lease": context["current_lease"], "goal_ref": context["goal_ref"], + "identity": readback.identity.value.as_dict(), + "control_writer_proof": {"idempotency_key": context["current_lease"]["idempotency_key"], + "expected_version": context["current_lease"]["version"]}, + "progress_observation": normalize_progress_observation(inline_progress_observation(args)), + "registered_agents": _registered_agents(context), + "observed_at": runtime_time.now_local_iso(timespec="microseconds"), + "provider_identity": source["store_identity"], "provider_revision": source["provider_revision"], + } + malformed = deepcopy(pure_input) + original_epoch = malformed["receipt"]["details"]["prepared_lease_epoch"] + malformed["receipt"]["details"]["prepared_lease_epoch"] = 0 + restored = deepcopy(malformed) + restored["receipt"]["details"]["prepared_lease_epoch"] = original_epoch + assert restored == pure_input + mixed_profile = deepcopy(pure_input) + mixed_profile.pop("goal_ref") + module = Path(loopx.__file__).resolve().parent / "control_plane" / "work_items" / "execution_preparation.ts" + script = ( + "import {qualifyReceiptBoundExecutionLossCloseout as qualify} from " + + json.dumps(module.as_uri()) + ";" + "let input='';for await(const part of process.stdin)input+=part;" + "const variants=JSON.parse(input);const results=variants.map(request=>{" + "try{return {status:'qualified',qualification:qualify(request)};}" + "catch(error){return {status:'rejected',diagnostic_code:error.code};}});" + "process.stdout.write(JSON.stringify(results));" + ) + child = subprocess.run( + ["node", "--no-warnings", "--experimental-strip-types", "--input-type=module", "-e", script], + input=json.dumps([pure_input, malformed, mixed_profile]), capture_output=True, text=True, + check=False, timeout=45, + ) + assert child.returncode == 0, child.stdout + child.stderr + accepted, rejected, mixed_rejected = json.loads(child.stdout) + assert accepted["status"] == "qualified", accepted + assert accepted["qualification"]["schema_version"] == "quota_execution_loss_closeout_v0" + assert rejected == {"status": "rejected", "diagnostic_code": "malformed_settlement_state"} + assert mixed_rejected == {"status": "rejected", "diagnostic_code": "refresh_task_lease_unavailable"} + assert _unchanged_state(context) == before diff --git a/tests/control_plane/test_execution_loss_closeout_recovery.py b/tests/control_plane/test_execution_loss_closeout_recovery.py new file mode 100644 index 0000000000..f169263360 --- /dev/null +++ b/tests/control_plane/test_execution_loss_closeout_recovery.py @@ -0,0 +1,270 @@ +"""Durable administrative prefixes recover without changing original execution.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +import json +from pathlib import Path +import subprocess + +import loopx +import pytest + +from loopx.control_plane.quota import execution_preparation, settlement +from loopx.control_plane.coordination import local_authority +from loopx.control_plane.runtime import time as runtime_time +from loopx.control_plane.work_items import task_lease +from tests.control_plane.test_execution_loss_closeout_cli import ( + OWNER, + PEER, + _assert_no_completion_or_debit, + _canonical, + _closeout_args, + _fixture, + _lease, + _readback, + _receipt_lines, + _reject_without_effect, + _runs, + _unchanged_state, +) + + +def _immutable_run(context, previous_runs): + runs = _runs(context) + names = runs.keys() - previous_runs.keys() + assert len(names) == 1 + return json.loads(runs[names.pop()]) + + +def _assert_markdown(context, record, result): + json_path, markdown_path = Path(record["json_path"]), Path(record["markdown_path"]) + assert json_path.parent == context["runtime"] / "goals" / context["goal_id"] / "runs" + assert markdown_path == json_path.with_suffix(".md") + assert result["json_path"] == str(json_path) and result["markdown_path"] == str(markdown_path) + assert json_path.is_file() and markdown_path.is_file() + assert markdown_path.read_text() == ( + "# Administrative execution-loss closeout\n\n" + f"Turn: {record['turn_instance_id']}\nNo delivery, completion or quota spend.\n" + ) + + +def _peer_claim_after_expiry(context, monkeypatch): + previous = _lease(context) + after_expiry = datetime.fromisoformat(previous["expires_at"].replace("Z", "+00:00")) + after_expiry = after_expiry.replace(microsecond=0) + timedelta(seconds=2) + + class AfterExpiryDatetime(datetime): + @classmethod + def now(cls, tz=None): + return (after_expiry.astimezone(tz) if tz + else after_expiry.astimezone().replace(tzinfo=None)) + + monkeypatch.setattr(runtime_time, "datetime", AfterExpiryDatetime) + monkeypatch.setattr(local_authority, "now_local", lambda: after_expiry.isoformat()) + monkeypatch.setattr(task_lease, "now_utc", lambda: after_expiry.astimezone(timezone.utc)) + assert runtime_time.now_utc() == after_expiry.astimezone(timezone.utc) + code, claimed = context["cli"]("todo", "claim", "--goal-id", context["goal_id"], + "--todo-id", context["todo"], "--agent-id", PEER, "--claimed-by", PEER) + assert code == 0, claimed + lease = _lease(context) + assert lease["owner"] == PEER and lease["lease_epoch"] > previous["lease_epoch"] + assert lease["version"] > previous["version"] and lease["idempotency_key"] != previous["idempotency_key"] + return lease + + +def _interrupted_owner(monkeypatch, context, fault): + """Inject at the real owner IO boundary; canonical stores and locks remain real.""" + original = execution_preparation.effect_runtime_result + module = Path(loopx.__file__).resolve().parent / "control_plane" + script = """ +import fs from 'node:fs'; +import {syncBuiltinESMExports} from 'node:module'; +import {dirname} from 'node:path'; +let raw='';for await(const part of process.stdin)raw+=part; +const {request,fault,moduleRoot,marker}=JSON.parse(raw); +const index=request.runtime_root+'/goals/'+request.identity.goal_id+'/runs/index.jsonl'; +const event=request.runtime_root+'/goals/'+request.identity.goal_id+'/rollout-event-log.jsonl'; +const open=fs.openSync,close=fs.closeSync,write=fs.writeFileSync,sync=fs.fsyncSync,rename=fs.renameSync; +const paths=new Map(),dirty=new Set();let published=null; +fs.openSync=(path,...args)=>{const fd=open(path,...args);paths.set(fd,String(path));return fd;}; +fs.closeSync=fd=>{paths.delete(fd);dirty.delete(fd);return close(fd);}; +fs.writeFileSync=(target,data,...args)=>{ + const result=write(target,data,...args); + if(typeof target==='number'&&String(data).includes('execution_loss_closeout'))dirty.add(target); + return result; +}; +fs.renameSync=(source,target)=>{ + const result=rename(source,target); + if(String(target).includes('/execution-loss-'))published=String(target); + return result; +}; +fs.fsyncSync=fd=>{ + const result=sync(fd),path=paths.get(fd); + const hit=(fault==='json'&&published?.endsWith('.json')&&path===dirname(published)) + ||(fault==='markdown'&&published?.endsWith('.md')&&path===dirname(published)) + ||(fault==='index'&&path===index&&dirty.has(fd)) + ||(fault==='event'&&path===event&&dirty.has(fd)); + if(hit){write(marker,fault);throw new Error('injected administrative interruption: '+fault);} + return result; +}; +syncBuiltinESMExports(); +if(fault==='store_identity'){ + for(const [path,name] of [['coordination/file_authority_store.ts','FileAuthorityStore'], + ['coordination/sqlite_authority_store.ts','SqliteAuthorityStore']]){ + const Type=(await import(moduleRoot+'/'+path))[name],checkpoint=Type.prototype.withCheckpointHead; + Type.prototype.withCheckpointHead=function(save){ + return checkpoint.call(this,(head,identity)=>save(head,identity+':replacement')); + }; + } +} +const {commitExecutionLossCloseout}=await import(moduleRoot+'/quota/execution_loss_closeout.ts'); +try{process.stdout.write(JSON.stringify({ok:true,result:await commitExecutionLossCloseout(request)}));} +catch(error){process.stdout.write(JSON.stringify({ok:false,error:String(error),code:error.code}));} +""" + + def invoke(method, params, **kwargs): + if method != "quota.execution_loss_closeout.commit": + return original(method, params, **kwargs) + marker = context["project"] / "administrative-fault-observed" + child = subprocess.run( + ["node", "--no-warnings", "--experimental-sqlite", "--experimental-strip-types", + "--input-type=module", "-e", script], + input=json.dumps({"request": params, "fault": fault, "moduleRoot": module.as_uri(), + "marker": str(marker)}), + text=True, capture_output=True, timeout=45, check=False, + ) + assert child.returncode == 0, child.stdout + child.stderr + result = json.loads(child.stdout) + if fault != "store_identity": + assert marker.read_text() == fault, result + assert result["ok"] is False, result + if fault == "store_identity": + assert result["code"] == "execution_loss_commit_unknown", result + raise ValueError(result["error"]) + + monkeypatch.setattr(execution_preparation, "effect_runtime_result", invoke) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("fault", ["json", "markdown", "index", "event"]) +def test_real_durable_prefix_recovery_keeps_original_admission_and_no_spend( + tmp_path, monkeypatch, capsys, provider, fault, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False) + original = _readback(context) + authority, state, receipts = _canonical(context), context["state"].read_bytes(), _receipt_lines(context) + runs_before = _runs(context) + argv = _closeout_args(context) + with monkeypatch.context() as injected: + _interrupted_owner(injected, context, fault) + code, failed = context["cli"](*argv) + assert code != 0 and failed["ok"] is False, failed + assert _canonical(context) == authority + assert context["state"].read_bytes() == state + assert _receipt_lines(context) == receipts + record = _immutable_run(context, runs_before) + frozen = record["execution_loss_closeout"] + assert frozen["original_prepared"] == original.prepared_lease + current = _lease(context) + assert current["owner"] == OWNER + assert frozen["current_control_writer"] == { + "owner": OWNER, "lease_key_sha256": sha256(current["idempotency_key"].encode()).hexdigest(), + "version": current["version"], "lease_epoch": current["lease_epoch"], + "store_identity": authority["store_identity"], "provider_revision": authority["provider_revision"], + } + immutable = _runs(context) + code, renewal = context["cli"]("task-lease", "renew", "--goal-id", context["goal_id"], + "--todo-id", context["todo"], "--owner", OWNER, "--idempotency-key", current["idempotency_key"], + "--expected-version", str(current["version"]), "--ttl-seconds", "3600") + assert code == 0 and renewal["ok"] is True, renewal + renewed = _lease(context) + assert renewed["owner"] == current["owner"] == OWNER + assert renewed["idempotency_key"] == current["idempotency_key"] + assert renewed["lease_epoch"] == current["lease_epoch"] + assert renewed["version"] == current["version"] + 1 + assert datetime.fromisoformat(renewed["expires_at"].replace("Z", "+00:00")) > datetime.fromisoformat( + current["expires_at"].replace("Z", "+00:00")) + renewed_authority = _canonical(context) + assert renewed_authority["store_identity"] == authority["store_identity"] + assert renewed_authority["provider_revision"] != authority["provider_revision"] + assert _receipt_lines(context) == receipts and _runs(context) == immutable + _reject_without_effect(context, argv, expected_code="refresh_task_lease_unavailable") + renewed_argv = _closeout_args(context, lease=renewed) + code, recovered = context["cli"](*renewed_argv) + assert code == 0 and recovered["settlement_progress"]["state"] == "settled", recovered + assert recovered["idempotent_replay"] is (fault == "event") + assert recovered["execution_loss_closeout"] == frozen + _assert_markdown(context, record, recovered) + assert _runs(context) == immutable + settled = _readback(context) + assert settled.replay_phase == settlement.ReceiptBoundReplayPhase.SETTLED + assert settled.prepared_lease == original.prepared_lease + assert _canonical(context) == renewed_authority and _lease(context) == renewed + assert _receipt_lines(context) == receipts and context["state"].read_bytes() == state + _assert_no_completion_or_debit(context) + before_replay = _unchanged_state(context) + code, replay = context["cli"](*renewed_argv) + assert code == 0 and replay["idempotent_replay"] is True, replay + assert _unchanged_state(context) == before_replay + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_frozen_administrative_intent_rejects_changed_store_identity( + tmp_path, monkeypatch, capsys, provider, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False) + argv = _closeout_args(context) + original, runs_before = _readback(context), _runs(context) + authority, receipts = _canonical(context), _receipt_lines(context) + with monkeypatch.context() as injected: + _interrupted_owner(injected, context, "json") + code, interrupted = context["cli"](*argv) + assert code != 0 and interrupted["ok"] is False, interrupted + record = _immutable_run(context, runs_before) + assert record["execution_loss_closeout"]["original_prepared"] == original.prepared_lease + assert not Path(record["markdown_path"]).exists() + before = _unchanged_state(context) + with monkeypatch.context() as injected: + _interrupted_owner(injected, context, "store_identity") + code, rejected = context["cli"](*argv) + assert code != 0 and rejected["ok"] is False, rejected + assert _unchanged_state(context) == before + code, recovered = context["cli"](*argv) + assert code == 0 and recovered["idempotent_replay"] is False, recovered + assert recovered["settlement_progress"]["state"] == "settled" + assert recovered["execution_loss_closeout"] == record["execution_loss_closeout"] + assert _runs(context) == before[2] + _assert_markdown(context, record, recovered) + assert _canonical(context) == authority and _receipt_lines(context) == receipts + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_peer_takeover_after_durable_prefix_cannot_recover_original_owner_intent( + tmp_path, monkeypatch, capsys, provider, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False) + original, runs_before = _readback(context), _runs(context) + receipts = _receipt_lines(context) + with monkeypatch.context() as injected: + _interrupted_owner(injected, context, "json") + code, interrupted = context["cli"](*_closeout_args(context)) + assert code != 0 and interrupted["ok"] is False, interrupted + record = _immutable_run(context, runs_before) + immutable = _runs(context) + peer_lease = _peer_claim_after_expiry(context, monkeypatch) + authority = _canonical(context) + todo = next(todo for todo in authority["todos"] if todo["todo_id"] == context["todo"]) + assert todo["claimed_by"] == PEER + assert _receipt_lines(context) == receipts and _runs(context) == immutable + _reject_without_effect(context, _closeout_args(context, lease=peer_lease), + expected_code="refresh_task_lease_unavailable") + assert _canonical(context) == authority and _lease(context) == peer_lease + assert _runs(context) == immutable and _receipt_lines(context) == receipts + assert not Path(record["markdown_path"]).exists() + readback = _readback(context) + assert readback.prepared_lease == original.prepared_lease + assert readback.heartbeat_receipt == original.heartbeat_receipt + assert not readback.writeback.receipts and not readback.spend.receipts + _assert_no_completion_or_debit(context) diff --git a/tests/control_plane_ts/execution_loss_closeout.test.ts b/tests/control_plane_ts/execution_loss_closeout.test.ts new file mode 100644 index 0000000000..c145293dac --- /dev/null +++ b/tests/control_plane_ts/execution_loss_closeout.test.ts @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {createHash} from "node:crypto"; +import {qualifyReceiptBoundExecutionLossCloseout, isExecutionLossCloseout, EXECUTION_LOSS_QUALIFICATION_SCHEMA} + from "../../loopx/control_plane/work_items/execution_preparation.ts"; +import {decodeExecutionLossCloseoutIntent} from "../../loopx/control_plane/quota/execution_loss_closeout.ts"; +import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; +import {settlementIdentity, settlementIdentityPayload} from "../../loopx/control_plane/effect_program.ts"; + +function fixture(): JsonObject { + const identity = settlementIdentityPayload(settlementIdentity({goal_id: "goal", agent_id: "owner", + todo_id: "todo_loss", turn_instance_id: "original-turn"})); + const ref = {goal_id: "goal", goal_instance_id: "ginst_" + "a".repeat(32)}; + return {schema_version: EXECUTION_LOSS_QUALIFICATION_SCHEMA, identity, goal_ref: ref, + receipt: {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", event_id: "original-receipt", + goal_id: "goal", agent_id: "owner", run_id: "original-turn", status: "normal_run", goal_ref: ref, + recorded_at: "2026-01-01T00:00:00Z", details: {todo_id: "todo_loss", settlement_effect_id: identity.effect_id, + closeout_required: true, prepared_lease_key_sha256: createHash("sha256").update("original-key").digest("hex"), + prepared_lease_version: 119, prepared_lease_epoch: 50, ok: true, should_run: true, + must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false}}, + current_todo: {todo_id: "todo_loss", role: "agent", task_class: "advancement_task", archive_state: "active", + status: "open", claimed_by: "owner"}, current_lease: {schema_version: "task_lease_v0", goal_id: "goal", + todo_id: "todo_loss", owner: "owner", status: "active", idempotency_key: "current-key", version: 121, + lease_epoch: 51, acquired_at: "2026-01-01T01:00:00Z", updated_at: "2026-01-01T01:01:00Z", + expires_at: "2026-01-01T02:00:00Z"}, control_writer_proof: {idempotency_key: "current-key", expected_version: 121}, + registered_agents: ["owner", "peer"], provider_identity: "file:" + "a".repeat(32), provider_revision: "revision-1", + observed_at: "2026-01-01T01:02:00Z", progress_observation: {schema_version: "typed_progress_observation_v0", + result_class: "blocked", work_item_id: "todo_loss", blocker_id: "execution-lost", evidence_ids: ["evidence:loss"], + fingerprint: "a".repeat(16)}}; +} + +test("execution-loss control proof stays distinct from original execution and preserves inputs", () => { + for (const legacy of [false, true]) { + const request = fixture(); + if (legacy) {delete request.goal_ref; delete (request.receipt as JsonObject).goal_ref;} + const before = structuredClone(request); + const closeout = qualifyReceiptBoundExecutionLossCloseout(request); + assert.equal(closeout.schema_version, "quota_execution_loss_closeout_v0"); + assert.deepEqual(closeout.original_prepared, { + prepared_lease_key_sha256: createHash("sha256").update("original-key").digest("hex"), + prepared_lease_version: 119, prepared_lease_epoch: 50}); + assert.equal((closeout.current_control_writer as JsonObject).version, 121); + assert.equal((closeout.current_control_writer as JsonObject).lease_epoch, 51); + assert.equal(isExecutionLossCloseout(closeout, "todo_loss"), true); + assert.equal(isExecutionLossCloseout({...closeout, completion: true}, "todo_loss"), false); + assert.equal(isExecutionLossCloseout({...closeout, + current_control_writer: {...closeout.current_control_writer as JsonObject, idempotency_key: "current-key"}}, "todo_loss"), false); + assert.equal(JSON.stringify(closeout).includes("current-key"), false); + assert.deepEqual(request, before); + } +}); + +test("execution-loss qualification refuses stale, foreign, ambiguous and incomplete authority", () => { + const mutations: Array<(request: JsonObject) => void> = [ + r => {(r.receipt as JsonObject).goal_ref = {goal_id: "goal", goal_instance_id: "ginst_" + "b".repeat(32)};}, + r => {delete r.goal_ref;}, + r => {((r.receipt as JsonObject).details as JsonObject).prepared_lease_epoch = 0;}, + r => {delete ((r.receipt as JsonObject).details as JsonObject).prepared_lease_key_sha256;}, + r => {(r.current_lease as JsonObject).lease_epoch = 50;}, + r => {(r.current_lease as JsonObject).owner = "peer";}, + r => {(r.control_writer_proof as JsonObject).expected_version = 120;}, + r => {r.observed_at = "2026-01-01T01:02:00";}, + r => {r.observed_at = "2026-01-01T02:00:00Z";}, + r => {r.registered_agents = ["peer"];}, + r => {(r.current_todo as JsonObject).excluded_agents = ["owner"];}, + r => {(r.current_todo as JsonObject).task_repository = "changed"; + (r.current_lease as JsonObject).write_repository = "original";}, + r => {((r.receipt as JsonObject).details as JsonObject).replan_obligation_id = "replan-" + "a".repeat(16);}, + r => {(r.progress_observation as JsonObject).result_class = "advanced";}, + r => {(r.progress_observation as JsonObject).coverage_complete = true;}, + ]; + for (const mutate of mutations) { + const request = fixture(); mutate(request); + assert.throws(() => qualifyReceiptBoundExecutionLossCloseout(request)); + } +}); + +test("explicit administrative intent cannot carry ordinary business effects", () => { + const request = fixture(); + const intent = {delivery_outcome: "outcome_gap", delivery_boundary: "semantic_closeout", + progress_observation: request.progress_observation}; + assert.deepEqual(decodeExecutionLossCloseoutIntent(intent), intent); + for (const extra of ["next_action", "vision", "completion_todo_id", "explore_result", "usage", "lease_mutation"]) + assert.throws(() => decodeExecutionLossCloseoutIntent({...intent, [extra]: true})); + assert.throws(() => decodeExecutionLossCloseoutIntent({...intent, delivery_outcome: "outcome_progress"})); +}); diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index da5f0aae0f..5bf3d8139f 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -169,6 +169,7 @@ test("capability retirement verifies its exact receipt and never erases a commit await writeFile(index, JSON.stringify(base) + "\n"); const accepted = await read(); assert.equal((accepted.progress as any).closeout_kind, "capability_duty_retired_no_spend"); + assert.equal((accepted.progress as any).no_spend, true); assert.equal(accepted.replay_phase, "settled"); assert.equal((accepted.spend as any).payload.ok, false); assert.equal((accepted.terminal_closeout as any).payload.ok, false); @@ -189,6 +190,7 @@ test("capability retirement verifies its exact receipt and never erases a commit const spent = await read(); assert.equal((spent.progress as any).state, "settled"); assert.equal((spent.progress as any).closeout_kind, undefined); + assert.equal((spent.progress as any).no_spend, undefined); assert.equal((spent.spend as any).payload.ok, true); } finally {await rm(root, {recursive: true, force: true});} }); @@ -228,6 +230,7 @@ test("a service-qualified blocked replan closes only its exact Turn without debi const accepted = await read(); assert.equal(accepted.replay_phase, "settled"); assert.equal((accepted.progress as any).closeout_kind, "typed_blocked_writeback_no_spend"); + assert.equal((accepted.progress as any).no_spend, true); assert.equal((accepted.spend as any).payload.ok, false); assert.equal((accepted.terminal_closeout as any).payload.ok, false); const history = async (runs: unknown[]) => (await readQuotaSettlement({ From a59b5e71715a5df026bed1f13169b92a89fc9b7c Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 04:58:36 +0800 Subject: [PATCH 02/12] fix(quota): require complete proof for execution-loss readback Reject marker-free administrative retry payloads before no-spend settlement; retain File/SQLite and TypeScript regression coverage. Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../quota/settlement_readback.ts | 6 +++-- .../test_execution_loss_closeout_cli.py | 25 +++++++++++++++++++ .../quota_settlement_readback.test.ts | 21 ++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 8cc96ea898..ebcfe6d570 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -49,7 +49,7 @@ import { type ReceiptBoundReplayPhase, } from "./settlement_phase.ts"; import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts"; -import {isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; +import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; import {isDeepStrictEqual} from "node:util"; import {canonicalAuthoritySha256} from "../coordination/authority_store_codec.ts"; import { @@ -1484,7 +1484,9 @@ function readQuotaSettlementFromRequest( } const completionEvent = findStepEvent(events, identity, "todo_complete"); const supersedeEvent = findStepEvent(events, identity, "todo_supersede"); - if (writebackRun?.execution_loss_closeout !== undefined) { + if (writebackRun !== null && (writebackRun.execution_loss_closeout !== undefined + || writebackRun.execution_loss_commit !== undefined + || jsonObject(writebackRun.blocked_retry)?.schema_version === EXECUTION_LOSS_CLOSEOUT_SCHEMA)) { const closeout = jsonObject(writebackRun.execution_loss_closeout); const commit = jsonObject(writebackRun.execution_loss_commit); const permittedRunFields = ["generated_at", "goal_id", "agent_id", "todo_id", "turn_instance_id", diff --git a/tests/control_plane/test_execution_loss_closeout_cli.py b/tests/control_plane/test_execution_loss_closeout_cli.py index 3860701228..a97c4b8640 100644 --- a/tests/control_plane/test_execution_loss_closeout_cli.py +++ b/tests/control_plane/test_execution_loss_closeout_cli.py @@ -610,6 +610,31 @@ def test_closeout_rejects_wrong_identity_and_extra_work_intent(tmp_path, monkeyp _reject_without_effect(context, progress) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("removed", ["execution_loss_closeout", "execution_loss_commit", "both"]) +def test_administrative_subtype_without_complete_proof_cannot_settle( + tmp_path, monkeypatch, capsys, provider, removed, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False) + code, closed = context["cli"](*_closeout_args(context)) + assert code == 0 and closed["settlement_progress"]["state"] == "settled", closed + index = context["runtime"] / "goals" / context["goal_id"] / "runs" / "index.jsonl" + records = [json.loads(line) for line in index.read_text().splitlines()] + record = next(row for row in records if "execution_loss_closeout" in row) + for field in ("execution_loss_closeout", "execution_loss_commit"): + if removed in (field, "both"): + del record[field] + # Keep the matching refresh event and the shaped blocked_retry. Neither + # may certify no-spend when the administrative qualification is absent. + Path(record["json_path"]).write_text(json.dumps(record) + "\n") + index.write_text("".join(json.dumps(row) + "\n" for row in records)) + before = _unchanged_state(context) + with pytest.raises(ValueError, match="administrative execution-loss receipt differs from original admission"): + _readback(context) + assert _unchanged_state(context) == before + _assert_no_completion_or_debit(context) + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_current_original_generation_keeps_ordinary_blocked_closeout_semantics( tmp_path, monkeypatch, capsys, provider, diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index 5bf3d8139f..c2592471e9 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -281,6 +281,27 @@ test("a service-qualified blocked replan closes only its exact Turn without debi } finally {await rm(root, {recursive: true, force: true});} }); +test("execution-loss blocked retry without administrative proof cannot certify no-spend", async () => { + const progress = {schema_version: "typed_progress_observation_v0", work_item_id: todoId, + result_class: "blocked", blocker_id: "execution-lost", evidence_ids: ["evidence:loss"], + fingerprint: "a".repeat(16)}; + const closeout = {schema_version: "quota_execution_loss_closeout_v0", effect_id: identity.effect_id, + todo_id: todoId, source: "turn_settlement", observed_at: "2026-01-01T01:02:00Z", + original_receipt_event_id: "event-guard", original_prepared: { + prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 1, prepared_lease_epoch: 1}, + current_control_writer: {lease_key_sha256: "b".repeat(64), version: 2, lease_epoch: 2, + owner: agentId, store_identity: "file:original", provider_revision: "revision-2"}, + observation_fingerprint: progress.fingerprint}; + const root = await fixture({writeback: true, writebackOutcome: "outcome_gap", + progressObservation: progress, blockedRetry: closeout}); + try { + await assert.rejects(readQuotaSettlement(request(root)), { + code: "malformed_settlement_state", + message: "administrative execution-loss receipt differs from original admission", + }); + } finally {await rm(root, {recursive: true, force: true});} +}); + async function fixture(options: { guard?: boolean; /** From 85a3d28c7e85e3233687c312e0d5be8fe071de3e Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 06:25:50 +0800 Subject: [PATCH 03/12] fix(quota): keep execution-loss authority on the lazy path Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../quota/execution_loss_closeout.ts | 109 +++++++++++++- .../quota/execution_loss_closeout_record.ts | 43 ++++++ loopx/control_plane/quota/settlement_phase.ts | 2 +- .../quota/settlement_readback.ts | 2 +- .../work_items/execution_preparation.ts | 141 ------------------ .../content_digest_single_owner.test.ts | 1 + .../execution_loss_closeout.test.ts | 5 +- 7 files changed, 156 insertions(+), 147 deletions(-) create mode 100644 loopx/control_plane/quota/execution_loss_closeout_record.ts diff --git a/loopx/control_plane/quota/execution_loss_closeout.ts b/loopx/control_plane/quota/execution_loss_closeout.ts index 048d012960..51a964ecb2 100644 --- a/loopx/control_plane/quota/execution_loss_closeout.ts +++ b/loopx/control_plane/quota/execution_loss_closeout.ts @@ -17,14 +17,119 @@ import {openRuntimeAuthorityStore, requireLocalAuthorityRuntimeRoot} from "../co import {requireShadowPrimaryWriteAllowed, shadowMaintenanceLockPath} from "../coordination/shadow_management.ts"; import {decodeTaskLeaseAuthority} from "../work_items/task_lease_acquire.ts"; import {legacyGoalRegistrationIsActive} from "../goals/first_party_host_runtime.ts"; -import {EXECUTION_LOSS_QUALIFICATION_SCHEMA, isExecutionLossCloseout, qualifyReceiptBoundExecutionLossCloseout} - from "../work_items/execution_preparation.ts"; +import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "./execution_loss_closeout_record.ts"; +import {parseExactGoalRef} from "../goals/goal_instance_identity.ts"; +import {canonicalTaskLease, canonicalLeaseTodoFact} from "../coordination/task_lease_state.ts"; +import {decodeTaskLeaseProof} from "../coordination/task_lease_proof.ts"; +import {heartbeatPreparedLease, heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent} + from "./heartbeat_receipt_identity.ts"; +import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import {projectProgressDelta} from "../work_items/progress_delta.ts"; +import {leaseRepositoryRejection} from "../work_items/task_lease_repository.ts"; +import {leaseOwnerRejection} from "../work_items/task_lease_eligibility.ts"; import {goalRolloutEventLogPath} from "../rollout_receipt_log.ts"; import {parseQuotaAccountingOwner, quotaGoalRef, requireCurrentQuotaAccountingOwner, requireQuotaOwnerProjection, withBorrowedQuotaAccountingOwner} from "./source_admission.ts"; import {QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, readQuotaSettlementForAdmittedOwnerFromSnapshot, readQuotaSettlementSnapshot} from "./settlement_readback.ts"; +export const EXECUTION_LOSS_QUALIFICATION_SCHEMA = "execution_loss_closeout_qualification_v0"; + +function rejectExecutionLoss(message: string): never { + throw new EffectRuntimeRequestError(`execution-loss closeout ${message}`, "refresh_task_lease_unavailable"); +} + +function executionLossClock(value: unknown): bigint { + if (typeof value !== "string") rejectExecutionLoss("requires an exact aware timestamp"); + const time = parseTodoTimestampMicros(value, true); + if (time === null) rejectExecutionLoss("has an invalid or ambiguous timestamp"); + return time; +} + +/** Pure qualification consumed only by the fenced administrative commit owner. + * A current lease proves control-writer ownership, never original execution. */ +export function qualifyReceiptBoundExecutionLossCloseout(value: unknown): JsonObject { + const request = requireJsonObject(value, "execution-loss qualification"); + if (request.schema_version !== EXECUTION_LOSS_QUALIFICATION_SCHEMA) + rejectExecutionLoss("qualification schema is unsupported"); + const binding = requireJsonObject(request.identity, "execution-loss identity"); + const identity = settlementIdentity({goal_id: requireNonEmptyString(binding.goal_id, "goal_id"), + agent_id: requireNonEmptyString(binding.agent_id, "agent_id"), todo_id: requireNonEmptyString(binding.todo_id, "todo_id"), + turn_instance_id: requireNonEmptyString(binding.turn_instance_id, "turn_instance_id")}); + if (identity.binding_kind !== "todo" || !identity.todo_id + || !isDeepStrictEqual(settlementIdentityPayload(identity), binding)) + rejectExecutionLoss("requires the exact original Todo Turn identity"); + const receipt = requireJsonObject(request.receipt, "original heartbeat receipt"); + if (request.goal_ref === undefined) { + if (Object.hasOwn(receipt, "goal_ref")) rejectExecutionLoss("cannot mix legacy and exact Goal ownership"); + } else { + const current = parseExactGoalRef(request.goal_ref), original = parseExactGoalRef(receipt.goal_ref); + if (current.kind !== "parsed" || original.kind !== "parsed" + || current.value.goalId.value !== identity.goal_id + || !isDeepStrictEqual(request.goal_ref, receipt.goal_ref)) + rejectExecutionLoss("requires the original exact Goal lifetime"); + } + const details = heartbeatReceiptDetails(receipt), fact = heartbeatReceiptFactFromEvent(receipt); + const originalBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); + const prepared = heartbeatPreparedLease(details); + if (receipt.schema_version !== "loopx_rollout_event_v0" || receipt.event_kind !== "quota_should_run" + || receipt.goal_id !== identity.goal_id || receipt.agent_id !== identity.agent_id + || receipt.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id + || originalBinding?.binding_kind !== "todo" || originalBinding.binding_id !== identity.todo_id + || fact.settlement_effect_id !== identity.effect_id || fact.closeout_required !== true + || !prepared || prepared.prepared_lease_epoch === undefined + || (Object.hasOwn(details, "prepared_lease_required") && details.prepared_lease_required !== true) + || (details.semantic_replan_obligation_id != null && details.semantic_replan_obligation_id !== "") + || nativeChildReportAdmission(details, receipt.status, identity.effect_id, "open", false) + .report_permission !== "new_operation") + rejectExecutionLoss("requires complete original prepared work admission"); + const todo = requireJsonObject(request.current_todo, "current Todo"); + const registered = request.registered_agents; + if (!Array.isArray(registered) || registered.some(agent => typeof agent !== "string") + || !registered.includes(identity.agent_id) || todo.todo_id !== identity.todo_id + || todo.role !== "agent" || todo.task_class !== "advancement_task" + || todo.archive_state !== "active" || todo.status !== "open" + || todo.claimed_by !== identity.agent_id || todo.done === true) + rejectExecutionLoss("requires the current registered control writer and unfinished Todo"); + if (leaseOwnerRejection(canonicalLeaseTodoFact(todo), identity.agent_id, registered as string[]) !== null) + rejectExecutionLoss("current control writer is not eligible for this Todo"); + const lease = canonicalTaskLease(requireJsonObject(request.current_lease, "current task lease"), + identity.goal_id, identity.todo_id); + const proof = decodeTaskLeaseProof(request.control_writer_proof); + if (!proof || leaseRepositoryRejection(todo, lease) !== null + || lease.owner !== identity.agent_id || lease.status !== "active" + || typeof lease.version !== "number" || !Number.isSafeInteger(lease.version) + || typeof lease.lease_epoch !== "number" || !Number.isSafeInteger(lease.lease_epoch) + || proof.idempotency_key !== lease.idempotency_key || proof.expected_version !== lease.version) + rejectExecutionLoss("requires the exact current control-writer proof"); + const now = executionLossClock(request.observed_at), admittedAt = executionLossClock(receipt.recorded_at); + const acquired = executionLossClock(lease.acquired_at), updated = executionLossClock(lease.updated_at); + const expires = executionLossClock(lease.expires_at); + if (admittedAt > acquired || acquired > updated || updated > now || expires <= now || expires <= updated) + rejectExecutionLoss("current control-writer clock is stale or ambiguous"); + const currentHash = createHash("sha256").update(proof.idempotency_key).digest("hex"); + if (lease.lease_epoch <= prepared.prepared_lease_epoch || lease.version <= prepared.prepared_lease_version + || currentHash === prepared.prepared_lease_key_sha256) + rejectExecutionLoss("cannot replace an available or ambiguous original generation"); + const progress = requireJsonObject(request.progress_observation, "blocked observation"); + if (progress.result_class !== "blocked" || progress.work_item_id !== identity.todo_id + || typeof progress.blocker_id !== "string" || !Array.isArray(progress.evidence_ids) + || progress.evidence_ids.length === 0 + || Object.keys(progress).some(field => !["schema_version", "result_class", "work_item_id", + "blocker_id", "evidence_ids", "fingerprint"].includes(field))) + rejectExecutionLoss("requires only verified blocked observation and evidence"); + projectProgressDelta({current: progress, baseline: null, window: []}); + return {schema_version: EXECUTION_LOSS_CLOSEOUT_SCHEMA, effect_id: identity.effect_id, + todo_id: identity.todo_id, source: "turn_settlement", observed_at: request.observed_at, + original_receipt_event_id: requireNonEmptyString(receipt.event_id, "original receipt event id"), + original_prepared: {...prepared}, current_control_writer: { + lease_key_sha256: currentHash, version: lease.version, lease_epoch: lease.lease_epoch, + owner: lease.owner, store_identity: requireNonEmptyString(request.provider_identity, "provider identity"), + provider_revision: requireNonEmptyString(request.provider_revision, "provider revision")}, + observation_fingerprint: progress.fingerprint}; +} + export const EXECUTION_LOSS_COMMIT_SCHEMA = "quota_execution_loss_closeout_commit_v0"; export const EXECUTION_LOSS_COMMIT_RESULT_SCHEMA = "quota_execution_loss_closeout_result_v0"; diff --git a/loopx/control_plane/quota/execution_loss_closeout_record.ts b/loopx/control_plane/quota/execution_loss_closeout_record.ts new file mode 100644 index 0000000000..4c6ed616c6 --- /dev/null +++ b/loopx/control_plane/quota/execution_loss_closeout_record.ts @@ -0,0 +1,43 @@ +/** Frozen administrative closeout record. Reading history never loads current lease authority. */ +import {isDeepStrictEqual} from "node:util"; +import type {JsonObject} from "../effect_program.ts"; +import {BARE_SHA256_PATTERN} from "../content_digest.ts"; +import {requireJsonObject} from "../runtime_decode.ts"; +import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; +import {heartbeatPreparedLease} from "./heartbeat_receipt_identity.ts"; + +export const EXECUTION_LOSS_CLOSEOUT_SCHEMA = "quota_execution_loss_closeout_v0"; + +/** Historical readback validates the frozen qualification, not today's lease. */ +export function isExecutionLossCloseout(value: unknown, todoId: string | null): boolean { + const closeout = typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonObject : null; + if (!closeout || closeout.schema_version !== EXECUTION_LOSS_CLOSEOUT_SCHEMA || !todoId + || closeout.todo_id !== todoId || closeout.source !== "turn_settlement" + || Object.keys(closeout).sort().join("|") !== ["schema_version", "effect_id", "todo_id", "source", + "observed_at", "original_receipt_event_id", "original_prepared", "current_control_writer", + "observation_fingerprint"].sort().join("|")) return false; + try { + const original = requireJsonObject(closeout.original_prepared, "original preparation"); + const prepared = heartbeatPreparedLease(original); + const writer = requireJsonObject(closeout.current_control_writer, "control-writer witness"); + if (typeof closeout.observed_at !== "string" + || parseTodoTimestampMicros(closeout.observed_at, true) === null) return false; + return !!prepared && prepared.prepared_lease_epoch !== undefined + && isDeepStrictEqual(original, prepared) + && Object.keys(writer).sort().join("|") === ["lease_key_sha256", "version", "lease_epoch", "owner", + "store_identity", "provider_revision"].sort().join("|") + && typeof closeout.effect_id === "string" && !!closeout.effect_id + && BARE_SHA256_PATTERN.test(String(writer.lease_key_sha256)) + && writer.lease_key_sha256 !== prepared.prepared_lease_key_sha256 + && typeof writer.version === "number" && Number.isSafeInteger(writer.version) + && writer.version > prepared.prepared_lease_version + && typeof writer.lease_epoch === "number" && Number.isSafeInteger(writer.lease_epoch) + && writer.lease_epoch > prepared.prepared_lease_epoch + && typeof closeout.original_receipt_event_id === "string" && !!closeout.original_receipt_event_id + && typeof writer.owner === "string" && !!writer.owner + && typeof writer.store_identity === "string" && !!writer.store_identity + && typeof writer.provider_revision === "string" && !!writer.provider_revision + && /^[a-f0-9]{16}$/u.test(String(closeout.observation_fingerprint)); + } catch {return false;} +} + diff --git a/loopx/control_plane/quota/settlement_phase.ts b/loopx/control_plane/quota/settlement_phase.ts index 8527a46a45..eb22175f2a 100644 --- a/loopx/control_plane/quota/settlement_phase.ts +++ b/loopx/control_plane/quota/settlement_phase.ts @@ -4,7 +4,7 @@ import { parseTodoTimestampMicros } from "../runtime_timestamp.ts"; import { isCanonicalTodoId } from "../todos/identity.ts"; import { blockedResumeInstant, isBlockedLifecycleWait, isCausalBlockedWait, isMonitorUnavailableWait } from "./blocked_wait.ts"; import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts"; -import {isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; +import {isExecutionLossCloseout} from "./execution_loss_closeout_record.ts"; function retryDelayIsBounded(seconds: number): boolean { return Number.isFinite(seconds) && seconds >= 60 && seconds <= 30 * 60; diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index ebcfe6d570..891b6d7f33 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -49,7 +49,7 @@ import { type ReceiptBoundReplayPhase, } from "./settlement_phase.ts"; import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts"; -import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "../work_items/execution_preparation.ts"; +import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "./execution_loss_closeout_record.ts"; import {isDeepStrictEqual} from "node:util"; import {canonicalAuthoritySha256} from "../coordination/authority_store_codec.ts"; import { diff --git a/loopx/control_plane/work_items/execution_preparation.ts b/loopx/control_plane/work_items/execution_preparation.ts index 2703f1e454..a942b66431 100644 --- a/loopx/control_plane/work_items/execution_preparation.ts +++ b/loopx/control_plane/work_items/execution_preparation.ts @@ -3,19 +3,6 @@ import {HANDOFF_MODE_RETIRED_REASON, retiredHandoffModeRejection} from "../coord import type {JsonObject} from "../effect_program.ts"; import {leaseEpoch} from "./task_lease_record.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; -import {createHash} from "node:crypto"; -import {isDeepStrictEqual} from "node:util"; -import {settlementIdentity, settlementIdentityPayload} from "../effect_program.ts"; -import {parseExactGoalRef} from "../goals/goal_instance_identity.ts"; -import {canonicalTaskLease, canonicalLeaseTodoFact} from "../coordination/task_lease_state.ts"; -import {decodeTaskLeaseProof} from "../coordination/task_lease_proof.ts"; -import {heartbeatPreparedLease, heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent} - from "../quota/heartbeat_receipt_identity.ts"; -import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; -import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; -import {projectProgressDelta} from "./progress_delta.ts"; -import {leaseRepositoryRejection} from "./task_lease_repository.ts"; -import {leaseOwnerRejection} from "./task_lease_eligibility.ts"; import { optionalNonEmptyString, requireBoolean, @@ -29,137 +16,9 @@ export const EXECUTION_PREPARATION_FACTS_SCHEMA_VERSION = "execution_preparation_facts_v0"; export const EXECUTION_PREPARATION_SCHEMA_VERSION = "execution_preparation_v0"; -export const EXECUTION_LOSS_CLOSEOUT_SCHEMA = "quota_execution_loss_closeout_v0"; -export const EXECUTION_LOSS_QUALIFICATION_SCHEMA = "execution_loss_closeout_qualification_v0"; const LEASE_STATUSES = ["active", "released", "unknown"] as const; -function rejectExecutionLoss(message: string): never { - throw new EffectRuntimeRequestError(`execution-loss closeout ${message}`, "refresh_task_lease_unavailable"); -} - -function executionLossClock(value: unknown): bigint { - if (typeof value !== "string") rejectExecutionLoss("requires an exact aware timestamp"); - const time = parseTodoTimestampMicros(value, true); - if (time === null) rejectExecutionLoss("has an invalid or ambiguous timestamp"); - return time; -} - -/** Pure qualification consumed only by the fenced administrative commit owner. - * A current lease proves control-writer ownership, never original execution. */ -export function qualifyReceiptBoundExecutionLossCloseout(value: unknown): JsonObject { - const request = requireJsonObject(value, "execution-loss qualification"); - if (request.schema_version !== EXECUTION_LOSS_QUALIFICATION_SCHEMA) - rejectExecutionLoss("qualification schema is unsupported"); - const binding = requireJsonObject(request.identity, "execution-loss identity"); - const identity = settlementIdentity({goal_id: requireNonEmptyString(binding.goal_id, "goal_id"), - agent_id: requireNonEmptyString(binding.agent_id, "agent_id"), todo_id: requireNonEmptyString(binding.todo_id, "todo_id"), - turn_instance_id: requireNonEmptyString(binding.turn_instance_id, "turn_instance_id")}); - if (identity.binding_kind !== "todo" || !identity.todo_id - || !isDeepStrictEqual(settlementIdentityPayload(identity), binding)) - rejectExecutionLoss("requires the exact original Todo Turn identity"); - const receipt = requireJsonObject(request.receipt, "original heartbeat receipt"); - if (request.goal_ref === undefined) { - if (Object.hasOwn(receipt, "goal_ref")) rejectExecutionLoss("cannot mix legacy and exact Goal ownership"); - } else { - const current = parseExactGoalRef(request.goal_ref), original = parseExactGoalRef(receipt.goal_ref); - if (current.kind !== "parsed" || original.kind !== "parsed" - || current.value.goalId.value !== identity.goal_id - || !isDeepStrictEqual(request.goal_ref, receipt.goal_ref)) - rejectExecutionLoss("requires the original exact Goal lifetime"); - } - const details = heartbeatReceiptDetails(receipt), fact = heartbeatReceiptFactFromEvent(receipt); - const originalBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); - const prepared = heartbeatPreparedLease(details); - if (receipt.schema_version !== "loopx_rollout_event_v0" || receipt.event_kind !== "quota_should_run" - || receipt.goal_id !== identity.goal_id || receipt.agent_id !== identity.agent_id - || receipt.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id - || originalBinding?.binding_kind !== "todo" || originalBinding.binding_id !== identity.todo_id - || fact.settlement_effect_id !== identity.effect_id || fact.closeout_required !== true - || !prepared || prepared.prepared_lease_epoch === undefined - || (Object.hasOwn(details, "prepared_lease_required") && details.prepared_lease_required !== true) - || (details.semantic_replan_obligation_id != null && details.semantic_replan_obligation_id !== "") - || nativeChildReportAdmission(details, receipt.status, identity.effect_id, "open", false) - .report_permission !== "new_operation") - rejectExecutionLoss("requires complete original prepared work admission"); - const todo = requireJsonObject(request.current_todo, "current Todo"); - const registered = request.registered_agents; - if (!Array.isArray(registered) || registered.some(agent => typeof agent !== "string") - || !registered.includes(identity.agent_id) || todo.todo_id !== identity.todo_id - || todo.role !== "agent" || todo.task_class !== "advancement_task" - || todo.archive_state !== "active" || todo.status !== "open" - || todo.claimed_by !== identity.agent_id || todo.done === true) - rejectExecutionLoss("requires the current registered control writer and unfinished Todo"); - if (leaseOwnerRejection(canonicalLeaseTodoFact(todo), identity.agent_id, registered as string[]) !== null) - rejectExecutionLoss("current control writer is not eligible for this Todo"); - const lease = canonicalTaskLease(requireJsonObject(request.current_lease, "current task lease"), - identity.goal_id, identity.todo_id); - const proof = decodeTaskLeaseProof(request.control_writer_proof); - if (!proof || leaseRepositoryRejection(todo, lease) !== null - || lease.owner !== identity.agent_id || lease.status !== "active" - || typeof lease.version !== "number" || !Number.isSafeInteger(lease.version) - || typeof lease.lease_epoch !== "number" || !Number.isSafeInteger(lease.lease_epoch) - || proof.idempotency_key !== lease.idempotency_key || proof.expected_version !== lease.version) - rejectExecutionLoss("requires the exact current control-writer proof"); - const now = executionLossClock(request.observed_at), admittedAt = executionLossClock(receipt.recorded_at); - const acquired = executionLossClock(lease.acquired_at), updated = executionLossClock(lease.updated_at); - const expires = executionLossClock(lease.expires_at); - if (admittedAt > acquired || acquired > updated || updated > now || expires <= now || expires <= updated) - rejectExecutionLoss("current control-writer clock is stale or ambiguous"); - const currentHash = createHash("sha256").update(proof.idempotency_key).digest("hex"); - if (lease.lease_epoch <= prepared.prepared_lease_epoch || lease.version <= prepared.prepared_lease_version - || currentHash === prepared.prepared_lease_key_sha256) - rejectExecutionLoss("cannot replace an available or ambiguous original generation"); - const progress = requireJsonObject(request.progress_observation, "blocked observation"); - if (progress.result_class !== "blocked" || progress.work_item_id !== identity.todo_id - || typeof progress.blocker_id !== "string" || !Array.isArray(progress.evidence_ids) - || progress.evidence_ids.length === 0 - || Object.keys(progress).some(field => !["schema_version", "result_class", "work_item_id", - "blocker_id", "evidence_ids", "fingerprint"].includes(field))) - rejectExecutionLoss("requires only verified blocked observation and evidence"); - projectProgressDelta({current: progress, baseline: null, window: []}); - return {schema_version: EXECUTION_LOSS_CLOSEOUT_SCHEMA, effect_id: identity.effect_id, - todo_id: identity.todo_id, source: "turn_settlement", observed_at: request.observed_at, - original_receipt_event_id: requireNonEmptyString(receipt.event_id, "original receipt event id"), - original_prepared: {...prepared}, current_control_writer: { - lease_key_sha256: currentHash, version: lease.version, lease_epoch: lease.lease_epoch, - owner: lease.owner, store_identity: requireNonEmptyString(request.provider_identity, "provider identity"), - provider_revision: requireNonEmptyString(request.provider_revision, "provider revision")}, - observation_fingerprint: progress.fingerprint}; -} - -/** Historical readback validates the frozen qualification, not today's lease. */ -export function isExecutionLossCloseout(value: unknown, todoId: string | null): boolean { - const closeout = typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonObject : null; - if (!closeout || closeout.schema_version !== EXECUTION_LOSS_CLOSEOUT_SCHEMA || !todoId - || closeout.todo_id !== todoId || closeout.source !== "turn_settlement" - || Object.keys(closeout).sort().join("|") !== ["schema_version", "effect_id", "todo_id", "source", - "observed_at", "original_receipt_event_id", "original_prepared", "current_control_writer", - "observation_fingerprint"].sort().join("|")) return false; - try { - const original = requireJsonObject(closeout.original_prepared, "original preparation"); - const prepared = heartbeatPreparedLease(original); - const writer = requireJsonObject(closeout.current_control_writer, "control-writer witness"); - executionLossClock(closeout.observed_at); - return !!prepared && prepared.prepared_lease_epoch !== undefined - && isDeepStrictEqual(original, prepared) - && Object.keys(writer).sort().join("|") === ["lease_key_sha256", "version", "lease_epoch", "owner", - "store_identity", "provider_revision"].sort().join("|") - && typeof closeout.effect_id === "string" && !!closeout.effect_id - && /^[a-f0-9]{64}$/u.test(String(writer.lease_key_sha256)) - && writer.lease_key_sha256 !== prepared.prepared_lease_key_sha256 - && typeof writer.version === "number" && Number.isSafeInteger(writer.version) - && writer.version > prepared.prepared_lease_version - && typeof writer.lease_epoch === "number" && Number.isSafeInteger(writer.lease_epoch) - && writer.lease_epoch > prepared.prepared_lease_epoch - && typeof closeout.original_receipt_event_id === "string" && !!closeout.original_receipt_event_id - && typeof writer.owner === "string" && !!writer.owner - && typeof writer.store_identity === "string" && !!writer.store_identity - && typeof writer.provider_revision === "string" && !!writer.provider_revision - && /^[a-f0-9]{16}$/u.test(String(closeout.observation_fingerprint)); - } catch {return false;} -} - function leaseFact(value: unknown, label: string): JsonObject { const lease = requireJsonObject(value, label); const status = requireStringLiteral(lease.status, LEASE_STATUSES, `${label}.status`); diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index eccc73e8ae..1c418417c9 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -162,6 +162,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/presentation/action_review_plan.ts", "control_plane/presentation/goal_source_links.ts", "control_plane/presentation/report_review.ts", + "control_plane/quota/execution_loss_closeout_record.ts", "control_plane/quota/heartbeat_receipt_identity.ts", "control_plane/quota/refresh_external_delivery.ts", "control_plane/quota/settlement_readback.ts", diff --git a/tests/control_plane_ts/execution_loss_closeout.test.ts b/tests/control_plane_ts/execution_loss_closeout.test.ts index c145293dac..ed4627cefe 100644 --- a/tests/control_plane_ts/execution_loss_closeout.test.ts +++ b/tests/control_plane_ts/execution_loss_closeout.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; import {createHash} from "node:crypto"; -import {qualifyReceiptBoundExecutionLossCloseout, isExecutionLossCloseout, EXECUTION_LOSS_QUALIFICATION_SCHEMA} - from "../../loopx/control_plane/work_items/execution_preparation.ts"; +import {qualifyReceiptBoundExecutionLossCloseout, EXECUTION_LOSS_QUALIFICATION_SCHEMA} + from "../../loopx/control_plane/quota/execution_loss_closeout.ts"; +import {isExecutionLossCloseout} from "../../loopx/control_plane/quota/execution_loss_closeout_record.ts"; import {decodeExecutionLossCloseoutIntent} from "../../loopx/control_plane/quota/execution_loss_closeout.ts"; import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {settlementIdentity, settlementIdentityPayload} from "../../loopx/control_plane/effect_program.ts"; From c43a99eaa278b522bc7c064b27b7b6e2819109f1 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 06:29:11 +0800 Subject: [PATCH 04/12] test(quota): follow the relocated execution-loss qualifier Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- tests/control_plane/test_execution_loss_closeout_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/control_plane/test_execution_loss_closeout_cli.py b/tests/control_plane/test_execution_loss_closeout_cli.py index a97c4b8640..05f6171f20 100644 --- a/tests/control_plane/test_execution_loss_closeout_cli.py +++ b/tests/control_plane/test_execution_loss_closeout_cli.py @@ -713,7 +713,7 @@ def test_malformed_original_prepared_receipt_or_mixed_profile_cannot_certify_exe assert restored == pure_input mixed_profile = deepcopy(pure_input) mixed_profile.pop("goal_ref") - module = Path(loopx.__file__).resolve().parent / "control_plane" / "work_items" / "execution_preparation.ts" + module = Path(loopx.__file__).resolve().parent / "control_plane" / "quota" / "execution_loss_closeout.ts" script = ( "import {qualifyReceiptBoundExecutionLossCloseout as qualify} from " + json.dumps(module.as_uri()) + ";" From 7feb9bc8852883433deb6eeb03f3042e72a1f809 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 06:38:35 +0800 Subject: [PATCH 05/12] style(quota): normalize closeout record file ending Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- loopx/control_plane/quota/execution_loss_closeout_record.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/loopx/control_plane/quota/execution_loss_closeout_record.ts b/loopx/control_plane/quota/execution_loss_closeout_record.ts index 4c6ed616c6..baed472470 100644 --- a/loopx/control_plane/quota/execution_loss_closeout_record.ts +++ b/loopx/control_plane/quota/execution_loss_closeout_record.ts @@ -40,4 +40,3 @@ export function isExecutionLossCloseout(value: unknown, todoId: string | null): && /^[a-f0-9]{16}$/u.test(String(closeout.observation_fingerprint)); } catch {return false;} } - From 1537e6a1b171c8f185001e7a8fde84edbc156402 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 09:47:02 +0800 Subject: [PATCH 06/12] fix(quota): retain original preparation through native recovery Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 20 + .../monorepo-distribution-split-v0.zh-CN.md | 15 + docs/reference/protocols/turn-envelope-v0.md | 52 ++ .../capabilities/native_chat/steward_turn.py | 142 ++++- loopx/cli_commands/turn_run_once.py | 143 +++-- loopx/cli_commands/turn_settlement_probes.py | 5 + .../control_plane/effect_runtime_handlers.ts | 1 + .../effect_runtime_snapshot_methods.json | 1 + .../goals/first_party_host_admission.py | 8 +- .../control_plane/projects/registry_codec.py | 4 + .../quota/execution_loss_closeout.ts | 10 +- .../quota/execution_preparation.py | 123 ++-- .../control_plane/quota/heartbeat_receipt.py | 47 +- .../quota/heartbeat_receipt_identity.ts | 310 +++++++++- loopx/control_plane/quota/settlement.py | 26 + .../quota/settlement_readback.ts | 264 ++++++++- .../turn_driver/turn_journal_query.ts | 60 ++ loopx/state_refresh.py | 2 + .../test_execution_loss_closeout_cli.py | 181 ++++++ tests/control_plane/test_heartbeat_receipt.py | 15 +- .../heartbeat_resume_lease.test.ts | 545 ++++++++++++++++++ .../quota_settlement_readback.test.ts | 149 ++++- .../turn_journal_query.test.ts | 46 +- tests/test_codex_turn_completion.py | 340 ++++++++++- tests/test_loopx_turn_driver.py | 213 +++++++ ...t_steward_report_recovery_and_validator.py | 263 ++++++++- 26 files changed, 2800 insertions(+), 185 deletions(-) create mode 100644 tests/control_plane_ts/heartbeat_resume_lease.test.ts diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 3e1b7e1299..2e3912e85f 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4800,6 +4800,26 @@ view shows its independently checked result in the same conversation. and records only blocked no-spend evidence. Its immutable Run recovers missing tails within the same store identity; replay requires a fresh control proof. Ordinary writeback and pure historical readback keep their boundaries. +- **Resume authority:** later receipts and native resume do not replace the + original prepared admission. Remaining execution effects in the same generation use + ephemeral current-CAS qualification from the single TS owner without + rewriting original proof/version; a different generation's frozen report + gains no old-Turn business completion authority. Every effect rereads its owning + authority, and execution-loss no-spend closeout still + compares the original admission with the current control writer. + Alias-only legacy prepared Turn bindings retain their bounded compatibility; + they do not certify administrative work admission. Pending terminal effects + and committed replay reuse the existing journal/settlement owner: execute + only the original remaining effect, repair a proven missing receipt, or + return existing results without a new effect. After accepted durable writeback, + pending quota spend uses the existing accounting owner and journal-authorized + recovery without requesting execution authority or repeating business effects. + TS-owned cleanup retains an admitted Turn's original lease while closeout is + owed; frozen-report recovery qualifies and renews that same generation. + Selection-recovery governance probes acquire no report business lease; + existing TS admission must permit the report before its preparation acquires one. + Historical generation conflicts + reject, and validated causal links preserve the original prepared revision. - **Source evidence:** CLI tests cover both local providers, stale/foreign proofs, ordinary parity and zero completion/debit. Recovery tests inject interruption after JSON, Markdown, index and receipt durability. SourceSession diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index a9db6de9f9..59acec0b78 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3707,6 +3707,21 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 provider owner。保留原 identity/prepared proof,独立核验当前 control writer, 仅记录 blocked 免扣额证据。不可变 Run 在同一 store identity 内恢复缺失尾部; 重放仍须 fresh control proof。普通写回与纯历史回读保留各自边界。 +- **恢复权限:**原 prepared admission 不被后续收据或 native resume 替换。 + 同 generation 的 remaining execution effects 使用单一 TS owner 的 ephemeral 当前 CAS + 资格,不重写原 proof/version;不同 generation 的 frozen report 不能取得旧 + Turn 业务完成权。每个 effect 边界重新读取对应 owner 的 authority, + 失去执行的原 Turn 仍以原 admission 对比当前 control writer 做免扣额收尾。 + 仅 alias 的 legacy prepared Turn binding 保留有限兼容,不据此证明行政 work + admission。pending terminal effect 与已提交重放复用既有 journal/settlement + owner:只执行原剩余 effect、修复已证明缺失的收据,或不执行新 effect 而返回 + 原结果。durable writeback 被接受后,pending quota spend 使用既有记账 owner + 与 journal 授权的恢复,不请求 execution authority 或重跑业务 effect。 + TS-owned cleanup 在 admitted Turn 欠收尾时保留原 lease;frozen-report 恢复 + 核验并续租同一 generation。selection-recovery governance probe 不获取 + 报告业务 lease;既有 TS admission 允许报告后,才在 preparation 前获取。 + 历史 generation 冲突拒绝,已核验因果链接保留 + 原 prepared revision。 - **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 6876889a51..c47fa41e2c 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -86,6 +86,36 @@ ABA guarantee; mixed/malformed source ownership never falls back to legacy. Identity conflicts return pure original settlement readback, without another work guard. Omitting the flag retains the ordinary writer and recovery rules. +Native resume does not republish a current lease as the original prepared +proof. The positive prepared admission event remains the original source even +after later review or readiness receipts. Remaining execution effects may use a freshly +verified current CAS proof only within the same execution generation, under the +existing lease owner: the key and epoch remain equal and the version does not +decrease. Exact legacy-proof compatibility retains its existing limited scope +and provides no ABA guarantee. A different generation cannot complete the old +Turn, including a frozen report; it requires execution-loss closeout or a fresh +Turn through the existing C1 owner. Every effect boundary rereads its owning authority; +retired authority is usable only for the original Turn's proven post-completion +writeback or its verified pending terminal-receipt repair. A conditional terminal +effect after writeback and spend uses the existing journal/settlement owner and +exact effect reference; an active proof may execute that remaining original +effect, while a canonical completed Todo permits only receipt repair. Committed +replay returns the existing result without asking for a lease or executing an +effect. After durable writeback is accepted, a pending quota spend is accounting +under the existing receipt/writeback/quota owner. Its journal-authorized recovery +does not request an execution lease or repeat host, completion or writeback work; +it still freshly validates the original accounting authority. Cleanup retains +the original task lease while its admitted Turn owes closeout and releases it +only when the TS settlement owner confirms eligibility. Frozen-report recovery +qualifies and renews that same generation; a replacement lease cannot recover +the old business. Report selection recovery runs its governance probe before +acquiring a report execution lease; the existing TS admission must permit the +report before its preparation acquires that lease. The probe's evidence remains +unchanged. Conflicting prepared generations in one identity's history are rejected; +validated causal links retain the original proof even on older revisions without +a revision marker. Direct receipt publication permits exact replay and rejects any +change to already prepared fields before appending a receipt. + `refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / semantic_closeout 证据及当前 control writer 的确切租约 key/version。TS owner 在真实 @@ -104,6 +134,28 @@ SourceSession 沿原 GoalRef;未激活项目保留既有稳定 runtime/Goal al incarnation 或 ABA 保证,混合及非法 ownership 不回退。identity conflict 返回原结算 的纯 readback,不再运行 work guard。省略该 flag 保持普通 writer 与 recovery 规则。 +Native resume 不把当前租约重新发布为原 prepared proof。后续 review/readiness 收据 +不能取代最初的 positive prepared admission。既存 remaining execution effects 只能通过现有 +租约 owner 在同一 execution generation 下使用 freshly verified 当前 CAS proof: +key 与 epoch 保持相同,version 不降低。Legacy exact-proof 兼容保留原有限边界, +不提供 ABA 保证。不同 generation 不能完成旧 Turn,即便已有 frozen report; +恢复使用 execution-loss closeout 或既有 C1 owner 下的 fresh Turn。每个 effect +边界均重新读取对应 owner 的 authority;retired proof 只支持原 Turn 已证明 completion 后尚余的 +writeback 或已核验的 pending terminal receipt 修复。writeback 与 spend 后的条件 +terminal effect 复用既有 journal/settlement owner 及 exact effect ref:active 原 +generation proof 只能执行该剩余 effect;canonical Todo 已完成时只能修复收据。 +已提交重放直接返回原结果,不请求租约或执行 effect。durable writeback 已被接受后, +尚欠的 quota spend 由既有 receipt/writeback/quota owner 负责记账。journal 授权的 +恢复不请求 execution lease,也不重跑 host、completion 或 writeback;仍须 freshly +核验原记账 authority。cleanup 在原 admitted Turn 欠收尾时保留原 task lease, +仅在 TS settlement owner 确认资格后释放。frozen-report 恢复核验并续租同一 +generation;replacement lease 不能恢复旧业务。报告 selection recovery 先执行 +governance probe;既有 TS admission 允许报告后,才在报告 preparation 前获取 +业务 execution lease,保留原 probe 证据。同 identity 历史中的不同 +prepared generation 均拒绝;旧收据没有 revision 标记时,已核验的因果链接仍保留 +原 proof。Direct receipt publication 只允许 exact replay,任何已准备字段变化都 +必须在追加收据前拒绝。 + Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. User task time requirements remain task context. Caller-private deadlines diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 23e2f0a48e..8a1a9b9783 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -509,8 +509,60 @@ def _task_lease_proof(self, request: Mapping[str, Any], lease: Mapping[str, Any] def _task_execution_lease(self, request: Mapping[str, Any], turn_id: str, *, recovery: bool = False, admission_retry: bool = False, claim_args: list[str] | None = None): """Both generation and cached validation use the original Todo owner.""" + from ...control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from ...control_plane.goals.goal_ref_validation import exact_goal_ref + from ...control_plane.quota.settlement import read_heartbeat_settlement from .steward_commands import native_command + request = {**request, "actor": dict(request["actor"])} actor = request["actor"] + # The admitted actor retains its original Goal lifetime. Current + # admission can check that reference, never supply a replacement. + goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) + if actor.get("goal_instance_id") is not None else None) + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=self.controller.registry_path, goal_id=actor["goal_id"], + planned_goal_ref=goal_ref, agent_id=actor["agent_id"]) + + def require_original_goal(): + admission.require_running(operation="native_task_execution_lifetime_check") + + require_original_goal() + + def settlement_readback(): + require_original_goal() + return read_heartbeat_settlement(self.controller.coordination_runtime_root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"], registry_path=self.controller.registry_path, + goal_ref=goal_ref) + + def cleanup_readback(*, source_admission=None, fenced=False): + from ...control_plane.turn_driver.journal_store import find_loopx_turn_key_by_settlement_identity + if not fenced: + require_original_goal() + journal = find_loopx_turn_key_by_settlement_identity( + self.controller.coordination_runtime_root, goal_id=actor["goal_id"], + agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"]) + return read_heartbeat_settlement(self.controller.coordination_runtime_root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"], registry_path=self.controller.registry_path, + goal_ref=goal_ref, source_admission=source_admission, + borrow_source_admission=source_admission is not None, + native_task_lease_cleanup={"schema_version": "native_task_lease_cleanup_request_v0", + "recovery": recovery, "admission_retry": admission_retry, + "resume_turn_key_present": request.get("resume_turn_key") is not None, + "original_journal_present": journal is not None}) + + def cleanup_disposition(readback): + disposition = readback.native_task_lease_cleanup if readback is not None else None + if (not isinstance(disposition, Mapping) + or disposition.get("schema_version") != "native_task_lease_cleanup_v0" + or disposition.get("disposition") not in {"release", "retain", "unconfirmed"}): + raise RuntimeError("the native task lease cleanup disposition could not be read") + if disposition["disposition"] == "unconfirmed": + raise ValueError("the original Turn closeout could not be verified before lease release") + return disposition["disposition"] + if claim_args is not None: operation_id = f"todo-claim:native-turn:{turn_id}" claim = native_command(self.controller, [*claim_args, "--claim-operation-id", operation_id]) @@ -522,29 +574,76 @@ def _task_execution_lease(self, request: Mapping[str, Any], turn_id: str, *, rec if not isinstance(lease, dict): raise ValueError("native task preparation could not read back its original lease") else: - retained = native_command(self.controller, ["task-lease", "inspect", "--goal-id", actor["goal_id"], - "--todo-id", request["todo_id"]]).get("lease") + inspected = native_command(self.controller, ["task-lease", "inspect", "--goal-id", actor["goal_id"], + "--todo-id", request["todo_id"]]) + retained = inspected.get("lease") version = retained["version"] if retained else 0 - suffix = (".validation-recovery." + str(version) if recovery else - ".admission-retry." + turn_id if admission_retry else - ".admission-retry" if request.get("resume_turn_key") else "") - lease = native_command(self.controller, ["task-lease", "acquire", "--goal-id", actor["goal_id"], - "--todo-id", request["todo_id"], "--owner", actor["agent_id"], - "--idempotency-key", request["turn_instance_id"] + suffix, "--ttl-seconds", "3600", - "--expected-version", str(version)])["lease"] + if recovery: + from ...control_plane.quota.execution_preparation import read_turn_execution_lease_qualification + original = settlement_readback() + if original is None or original.identity.value is None: + raise ValueError("the original report settlement identity is unavailable") + qualified = read_turn_execution_lease_qualification( + runtime_root=self.controller.coordination_runtime_root, + registry_path=self.controller.registry_path, identity=original.identity.value, + requested_proof=({"idempotency_key": retained["idempotency_key"], + "expected_version": version} if retained else None), goal_ref=goal_ref) + qualification = qualified.resume_lease_qualification or {} + if qualification.get("state") != "ready": + raise ValueError("the original task execution lease is unavailable for settlement recovery") + # The shared TS owner qualifies original generation continuity. + # The canonical lifecycle owner renews that exact holder with CAS. + proof = qualification["current_task_lease_proof"] + lease = native_command(self.controller, ["task-lease", "renew", "--goal-id", actor["goal_id"], + "--todo-id", request["todo_id"], "--owner", actor["agent_id"], + "--idempotency-key", proof["idempotency_key"], "--ttl-seconds", "3600", + "--expected-version", str(proof["expected_version"])])["lease"] + else: + suffix = (".admission-retry." + turn_id if admission_retry else + ".admission-retry" if request.get("resume_turn_key") else "") + lease = native_command(self.controller, ["task-lease", "acquire", "--goal-id", actor["goal_id"], + "--todo-id", request["todo_id"], "--owner", actor["agent_id"], + "--idempotency-key", request["turn_instance_id"] + suffix, "--ttl-seconds", "3600", + "--expected-version", str(version)])["lease"] try: yield lease finally: - # Completion retires this lease. Never release a later owner's. + # Completion retires this lease. An unsettled exact Turn keeps its + # original generation available for the existing recovery owners. + # Never release a later owner's or a different Goal instance's. try: retained = native_command(self.controller, ["task-lease", "inspect", "--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]]).get("lease") if retained and retained.get("status") == "active": - proof = self._task_lease_proof(request, lease) - native_command(self.controller, ["task-lease", "release", "--goal-id", actor["goal_id"], - "--todo-id", request["todo_id"], "--owner", actor["agent_id"], - "--idempotency-key", proof["task_lease_idempotency_key"], - "--expected-version", str(proof["task_lease_expected_version"])]) + self._task_lease_proof(request, lease) + if cleanup_disposition(cleanup_readback()) == "release": + from ...control_plane.quota.accounting_admission import quota_accounting_admission + from ...file_lock import exclusive_cross_runtime_file_lock + from ...rollout_event_log import rollout_event_log_path + # Requalification owns rollout before index/Goal. Never + # wait for rollout in the inverse order: retain instead. + with quota_accounting_admission( + runtime_root=self.controller.coordination_runtime_root, + registry_path=self.controller.registry_path, goal_id=actor["goal_id"], + goal_ref=goal_ref, operation="native_task_execution_release") as source_admission: + lifetime = (nullcontext() if admission.source_profile else admission._legacy_lifetime( + operation="native_task_execution_release", launch=False, registry_lock_timeout=0)) + with lifetime: + if admission.source_profile: + # Quota already holds this original source + # guard. Reuse its pure owner check, not a + # second acquisition of the same lifetime. + admission._decision("require_current") + with exclusive_cross_runtime_file_lock( + rollout_event_log_path(self.controller.coordination_runtime_root, actor["goal_id"]), + operation="native_task_execution_release", timeout_seconds=0): + original = cleanup_readback(source_admission=source_admission, fenced=True) + if cleanup_disposition(original) == "release": + proof = self._task_lease_proof(request, lease) + native_command(self.controller, ["task-lease", "release", "--goal-id", actor["goal_id"], + "--todo-id", request["todo_id"], "--owner", actor["agent_id"], + "--idempotency-key", proof["task_lease_idempotency_key"], + "--expected-version", str(proof["task_lease_expected_version"])]) except (OSError, ValueError, RuntimeError): self.store.append_event(actor["session_id"], turn_id, kind="native_loopx.task_lease_release_unconfirmed", payload={"todo_id": request["todo_id"], "lease_epoch": lease["lease_epoch"]}) @@ -1080,7 +1179,7 @@ def execute(self, *, session_id: str, turn_id: str, adapter: Any, report_context = "" governance_work = False managed_answer = "" - managed_lease, managed_lease_request = None, None + report_lease, managed_lease, managed_lease_request = None, None, None preparation_status = None steward_actions_state: dict[str, Any] = {"context_handoff": None, "user_actions": [], "validation_error": None} @@ -1278,8 +1377,9 @@ def launch(): # The kernel records the actual app-server host and independent profile. args.host, args.session_scope = "codex-app-server", "agent" result: dict[str, Any] = {} - with ExitStack() as managed_leases, (self._task_execution_lease(request, turn_id, - admission_retry=bool(selection_recovery)) if report_work else nullcontext(None)) as report_lease: + with ExitStack() as managed_leases: + if report_work and not selection_recovery: + report_lease = managed_leases.enter_context(self._task_execution_lease(request, turn_id)) if not report_work and request.get("todo_id") and not request.get("resume_turn_key"): # The selected Task's existing lease must precede the kernel's # delivery guard. Taking it in the model callback is too late: @@ -1328,6 +1428,12 @@ def probe_governance(todo_id: str | None) -> dict[str, Any]: selection_recovery = probe governance_work = False if report_work: + if report_lease is None: + # Governance admission selects its own Turn. Acquire the + # report's business lease only after its typed continuation + # is admitted, before collecting and executing that report. + report_lease = managed_leases.enter_context(self._task_execution_lease( + request, turn_id, admission_retry=bool(selection_recovery))) # Portfolio collection can take minutes and legitimately see # registry updates. Finish it before the native driver captures # its launch witness; retain the launch fence and fresh source diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index aa1b9a0e59..77d13ed3c9 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -18,7 +18,10 @@ from ..control_plane.goals.first_party_host_admission import ( FirstPartyHostGoalAdmission, ) -from ..control_plane.quota.execution_preparation import read_active_task_lease_epoch +from ..control_plane.quota.execution_preparation import ( + RefreshTaskLeaseUnavailable, + read_turn_execution_lease_qualification, +) from ..control_plane.quota.heartbeat_receipt import ( ensure_turn_heartbeat_settlement_receipt, ) @@ -119,28 +122,11 @@ def execute_turn_run_once( expected_todo_text_sha256 = execution_context["sha256"] task_lease_proof = execution_context.get("task_lease_proof") or None task_lease_epoch = execution_context.get("task_lease_epoch") - rebound_lease_active = False - if ( - args.execute - and getattr(args, "resume_turn_key", None) + resume_execution = bool( + args.execute and getattr(args, "resume_turn_key", None) and native_completion_lease_resolver is not None and selected_todo.get("todo_id") - ): - resolved_lease = dict(native_completion_lease_resolver()) - if resolved_lease.get("task_lease_idempotency_key") is not None: - rebound_proof = { - "idempotency_key": resolved_lease["task_lease_idempotency_key"], - "expected_version": resolved_lease["task_lease_expected_version"], - } - task_lease_epoch = read_active_task_lease_epoch( - runtime_root=runtime_root, - goal_id=args.goal_id, - todo_id=str(selected_todo["todo_id"]), - agent_id=args.agent_id, - requested_proof=rebound_proof, - ) - task_lease_proof = rebound_proof - rebound_lease_active = True + ) transaction = ( payload.get("transaction") if isinstance(payload.get("transaction"), Mapping) @@ -178,6 +164,75 @@ def execute_turn_run_once( and persisted_effect_id != settlement_identity.effect_id ): raise ValueError("Turn settlement identity effect_id is inconsistent") + + def current_execution_lease( + *, remaining_completion_writeback: bool = False, + require_active: bool = False, + effect_ref: str | None = None, + ) -> dict[str, Any] | None: + requested_proof = task_lease_proof + resolver_unavailable = False + if remaining_completion_writeback: + # Completion can retire the lease before refresh. Only the TS + # owner may prove that the retired lease belongs to this Turn. + requested_proof = None + elif native_completion_lease_resolver is not None: + try: + offered = native_completion_lease_resolver() + except ValueError: + # The native resolver cannot offer an active proof after + # retirement. Its exception is no evidence of retirement. + requested_proof = None + resolver_unavailable = True + else: + resolved = dict(offered) + requested_proof = ( + {"idempotency_key": resolved["task_lease_idempotency_key"], + "expected_version": resolved["task_lease_expected_version"]} + if resolved.get("task_lease_idempotency_key") is not None else None + ) + readback = read_turn_execution_lease_qualification( + runtime_root=runtime_root, registry_path=registry_path, + identity=settlement_identity, requested_proof=requested_proof, + goal_ref=goal_ref, + provider_context={ + "turn_key": args.resume_turn_key, + "effect_ref": effect_ref, + }, + ) + qualification = readback.resume_lease_qualification or {} + if resolver_unavailable and not ( + qualification.get("state") == "ready" + and qualification.get("source") == "retired" + ): + raise RefreshTaskLeaseUnavailable("The native resolver could not offer this Turn's active proof.") + if qualification.get("state") == "not_required": + return None + if require_active and qualification.get("source") != "active": + raise RefreshTaskLeaseUnavailable("The original Turn has no active lease for Todo completion.") + proof = qualification.get("current_task_lease_proof") + if not isinstance(proof, dict): + raise RefreshTaskLeaseUnavailable("The qualified Turn has no current task lease proof.") + return dict(proof) + + if resume_execution: + inspection = inspect_loopx_turn_journal( + runtime_root, goal_id=args.goal_id, agent_id=args.agent_id, + turn_key=args.resume_turn_key, retry_failed=bool(args.retry_failed_turn), + ) + recovery = inspection.get("recovery_decision") + # Retained results and the typed continuation to accounting need + # no execution resolver. Spend still uses its fresh receipt owner. + if not isinstance(recovery, Mapping) or not ( + recovery.get("action") == "return_existing" + or ( + recovery.get("action") == "continue" + and recovery.get("can_continue") is True + and recovery.get("reinvoke_host") is False + and recovery.get("resume_from") == "quota_spend" + ) + ): + current_execution_lease() replan_guard_scoped = execution_context["replan_guard_scoped"] replan_obligation_id = ( replan_obligation_id_from_packet( @@ -264,6 +319,7 @@ def refresh_candidate( completion_todo_id: str | None = None, completion_turn_key: str | None = None, dry_run: bool = False, + execution_lease_proof: Mapping[str, Any] | None = None, ) -> dict[str, object]: # The host workspace is execution context, not state authority. state_project = None @@ -313,7 +369,11 @@ def refresh_candidate( dry_run=dry_run, sync_global=not bool(args.no_global_sync), goal_ref=goal_ref, - task_lease_proof=None if completion_todo_id is not None else task_lease_proof, + task_lease_proof=( + (dict(execution_lease_proof) if execution_lease_proof is not None else None) + if resume_execution + else None if completion_todo_id is not None else task_lease_proof + ), ) # A Todo-less native governance Turn has no business artifact/argv. # Its independent validator is the canonical semantic writeback preview, @@ -336,10 +396,12 @@ def writeback( effect_ref: str, ) -> dict[str, object]: require_effect_ref(effect_ref, SettlementStepKind.DURABLE_WRITEBACK) - if args.execute and rebound_lease_active: - bind_heartbeat_settlement_receipt() + proof = (current_execution_lease( + remaining_completion_writeback=completion_todo_id is not None, + effect_ref=effect_ref, + ) if resume_execution else None) refresh = refresh_candidate(result, completion_todo_id=completion_todo_id, - completion_turn_key=completion_turn_key) + completion_turn_key=completion_turn_key, execution_lease_proof=proof) if refresh.get("ok") and ( refresh.get("appended") or refresh.get("idempotent_replay") @@ -396,6 +458,18 @@ def todo_completion( raise ValueError( "validated_completion requires one selected todo for lifecycle writeback" ) + if resume_execution: + proof = current_execution_lease(require_active=True, effect_ref=effect_ref) + completion_lease_fields = { + "task_lease_idempotency_key": (proof or {}).get("idempotency_key"), + "task_lease_expected_version": (proof or {}).get("expected_version"), + } + else: + completion_lease_fields = (dict(native_completion_lease_resolver()) + if native_completion_lease_resolver is not None else { + "task_lease_idempotency_key": (task_lease_proof or {}).get("idempotency_key"), + "task_lease_expected_version": (task_lease_proof or {}).get("expected_version"), + }) completion = write_turn_validated_completion( registry_path=registry_path, runtime_root_arg=runtime_root_arg, @@ -411,10 +485,7 @@ def todo_completion( agent_id=args.agent_id, completion_delivery_workspace=completion_delivery_workspace, completion_validation_workspace_path=delivery_workspace_path, - **(dict(native_completion_lease_resolver()) if native_completion_lease_resolver else { - "task_lease_idempotency_key": execution_context.get("task_lease_proof", {}).get("idempotency_key"), - "task_lease_expected_version": execution_context.get("task_lease_proof", {}).get("expected_version"), - }), + **completion_lease_fields, ) if completion.get("ok") is not True: return completion @@ -477,8 +548,8 @@ def completion_writeback( effect_ref: str, ) -> dict[str, object]: require_effect_ref(effect_ref, SettlementStepKind.DURABLE_WRITEBACK) - if args.execute and rebound_lease_active: - bind_heartbeat_settlement_receipt() + if resume_execution: + current_execution_lease(effect_ref=effect_ref) prior = completion_readback() if prior is None: # A rejected attachment must not leave the Todo completed. @@ -619,6 +690,9 @@ def spend(*, effect_ref: str) -> dict[str, object]: return {**spent, "appended": True, "effect_ref": effect_ref} return spent + def qualify_terminal_closeout(effect_ref: str) -> None: + current_execution_lease(effect_ref=effect_ref) + ( completion_readback, writeback_resolver, @@ -634,6 +708,7 @@ def spend(*, effect_ref: str) -> dict[str, object]: require_effect_ref=require_effect_ref, append_settlement_event=append_settlement_event, finish_explore_delivery=finish_explore_delivery, + **({"qualify_terminal_closeout": qualify_terminal_closeout} if resume_execution else {}), ) def scheduler(_spend_payload: dict[str, object]) -> dict[str, object]: @@ -771,6 +846,8 @@ def codex_settlement_admission(): settlement_admission = codex_settlement_admission def bind_heartbeat_settlement_receipt() -> None: + if resume_execution: + current_execution_lease() replan_packet = execution_context["replan_action_packet"] capability_guard = ( replan_packet.get("capability_guard") @@ -789,8 +866,8 @@ def bind_heartbeat_settlement_receipt() -> None: else None ), goal_ref=goal_ref, - task_lease_proof=task_lease_proof, - task_lease_epoch=task_lease_epoch, + task_lease_proof=None if resume_execution else task_lease_proof, + task_lease_epoch=None if resume_execution else task_lease_epoch, ) def on_managed_start_admitted() -> None: diff --git a/loopx/cli_commands/turn_settlement_probes.py b/loopx/cli_commands/turn_settlement_probes.py index b605893b9e..b1caa31be5 100644 --- a/loopx/cli_commands/turn_settlement_probes.py +++ b/loopx/cli_commands/turn_settlement_probes.py @@ -125,6 +125,7 @@ def build_turn_settlement_resolvers( require_effect_ref: Callable[[str, SettlementStepKind], None], append_settlement_event: Callable[..., None], finish_explore_delivery: Callable[[dict[str, object]], dict[str, object]], + qualify_terminal_closeout: Callable[[str], None] | None = None, ) -> tuple[ Callable[[], dict[str, object] | None], Callable[[str], dict[str, object]], @@ -297,6 +298,8 @@ def terminal_closeout_resolver(effect_ref: str) -> dict[str, object]: effect_ref, SettlementStepKind.TERMINAL_CLOSEOUT, ) + if qualify_terminal_closeout is not None: + qualify_terminal_closeout(effect_ref) readback = read_heartbeat_settlement( runtime_root, goal_id=settlement_identity.goal_id, @@ -322,6 +325,8 @@ def terminal_closeout_resolver(effect_ref: str) -> dict[str, object]: "reason": "Todo state does not prove terminal no-followup", } if event is None: + if qualify_terminal_closeout is not None: + qualify_terminal_closeout(effect_ref) append_settlement_event( {"ok": True, "appended": True}, event_kind="todo_complete", diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index c6f170b5b5..a4c3e4a606 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -383,6 +383,7 @@ export function createEffectRuntimeHandlers( ["quota.spend.commit", lazyHandler(() => import("./quota/spend_commit.ts"), ({evaluateQuotaSpendCommit}) => evaluateQuotaSpendCommit)], ["quota.void.commit", lazyHandler(() => import("./quota/void_commit.ts"), ({evaluateQuotaVoidCommit}) => evaluateQuotaVoidCommit)], ["quota.settlement.read", lazyHandler(() => import("./quota/settlement_readback.ts"), ({readQuotaSettlement}) => readQuotaSettlement)], + ["quota.heartbeat_prepared_publication.qualify", lazyHandler(() => import("./quota/heartbeat_receipt_identity.ts"), ({qualifyHeartbeatPreparedPublication}) => qualifyHeartbeatPreparedPublication)], [ "quota.prior_host_turn_closeout.preflight", lazyHandler(() => import("./quota/unsettled_host_turn_recovery.ts"), ({preflightPriorHostTurnCloseout}) => preflightPriorHostTurnCloseout), diff --git a/loopx/control_plane/effect_runtime_snapshot_methods.json b/loopx/control_plane/effect_runtime_snapshot_methods.json index 1616ef0506..c851af4bcb 100644 --- a/loopx/control_plane/effect_runtime_snapshot_methods.json +++ b/loopx/control_plane/effect_runtime_snapshot_methods.json @@ -20,6 +20,7 @@ "work_item.task_graph.status", "work_item.replan_semantics.project", "quota.settlement.read", + "quota.heartbeat_prepared_publication.qualify", "quota.spend.commit", "quota.turn_envelope.evaluate", "turn.controller", diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index 837f656365..0189c55810 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -576,17 +576,19 @@ def require_running(self, *, operation: str) -> None: raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc @contextmanager - def _legacy_lifetime(self, *, operation: str, launch: bool) -> Iterator[None]: + def _legacy_lifetime( + self, *, operation: str, launch: bool, registry_lock_timeout: float | None = None, + ) -> Iterator[None]: with ExitStack() as scope: if self.legacy_source_admission is not None: # Activation writes source then global projection under these # same locks. A failed mirror sync must not hide a source stop. scope.enter_context(self.legacy_source_admission._legacy_lifetime( - operation=operation, launch=launch, + operation=operation, launch=launch, registry_lock_timeout=registry_lock_timeout, )) try: transaction = scope.enter_context(project_registry_transaction( - self.registry_path, operation=operation, + self.registry_path, operation=operation, timeout_seconds=registry_lock_timeout, )) except (OSError, TypeError, ValueError) as exc: raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc diff --git a/loopx/control_plane/projects/registry_codec.py b/loopx/control_plane/projects/registry_codec.py index f4a86be3af..0f36e03e83 100644 --- a/loopx/control_plane/projects/registry_codec.py +++ b/loopx/control_plane/projects/registry_codec.py @@ -491,12 +491,14 @@ def _registry_transaction( require_writer: Callable[[_ProjectRegistryDocument], None], agent_id: str | None = None, allow_retired_activation: bool = False, + timeout_seconds: float | None = None, ) -> Iterator[ProjectRegistryTransaction]: expanded = path.expanduser() with exclusive_cross_runtime_file_lock( expanded, agent_id=agent_id, operation=operation, + timeout_seconds=timeout_seconds, ): existed = expanded.exists() if existed: @@ -597,6 +599,7 @@ def project_registry_transaction( operation: str, create: Callable[[], dict[str, Any]] | None = None, agent_id: str | None = None, + timeout_seconds: float | None = None, ) -> Iterator[ProjectRegistryTransaction]: """Hold one legacy/v1 project-registry transaction.""" @@ -611,6 +614,7 @@ def project_registry_transaction( create_document=create_document, require_writer=_require_supported_writer, agent_id=agent_id, + timeout_seconds=timeout_seconds, ) as transaction: yield transaction diff --git a/loopx/control_plane/quota/execution_loss_closeout.ts b/loopx/control_plane/quota/execution_loss_closeout.ts index 51a964ecb2..cf8f96a2f3 100644 --- a/loopx/control_plane/quota/execution_loss_closeout.ts +++ b/loopx/control_plane/quota/execution_loss_closeout.ts @@ -21,8 +21,9 @@ import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "./executi import {parseExactGoalRef} from "../goals/goal_instance_identity.ts"; import {canonicalTaskLease, canonicalLeaseTodoFact} from "../coordination/task_lease_state.ts"; import {decodeTaskLeaseProof} from "../coordination/task_lease_proof.ts"; -import {heartbeatPreparedLease, heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent} +import {heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent} from "./heartbeat_receipt_identity.ts"; +import {heartbeatPreparedWorkAdmission} from "./heartbeat_receipt_identity.ts"; import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; import {parseTodoTimestampMicros} from "../runtime_timestamp.ts"; import {projectProgressDelta} from "../work_items/progress_delta.ts"; @@ -72,12 +73,12 @@ export function qualifyReceiptBoundExecutionLossCloseout(value: unknown): JsonOb } const details = heartbeatReceiptDetails(receipt), fact = heartbeatReceiptFactFromEvent(receipt); const originalBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); - const prepared = heartbeatPreparedLease(details); + const prepared = heartbeatPreparedWorkAdmission(receipt, identity); if (receipt.schema_version !== "loopx_rollout_event_v0" || receipt.event_kind !== "quota_should_run" || receipt.goal_id !== identity.goal_id || receipt.agent_id !== identity.agent_id || receipt.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id || originalBinding?.binding_kind !== "todo" || originalBinding.binding_id !== identity.todo_id - || fact.settlement_effect_id !== identity.effect_id || fact.closeout_required !== true + || (fact.settlement_effect_id !== null && fact.settlement_effect_id !== identity.effect_id) || fact.closeout_required !== true || !prepared || prepared.prepared_lease_epoch === undefined || (Object.hasOwn(details, "prepared_lease_required") && details.prepared_lease_required !== true) || (details.semantic_replan_obligation_id != null && details.semantic_replan_obligation_id !== "") @@ -300,7 +301,8 @@ export async function commitExecutionLossCloseout(value: unknown): Promise dict[str, Any]: @@ -247,25 +247,38 @@ def commit_execution_loss_closeout( return result -def read_active_task_lease_epoch( - *, - runtime_root: Path, - goal_id: str, - todo_id: str, - agent_id: str, - requested_proof: Mapping[str, Any], -) -> int | None: - """Return the verified current lease epoch for an active resumed execution lease.""" - snapshot = _read_execution_snapshot(runtime_root, goal_id, todo_id) - if snapshot is None: - raise RefreshTaskLeaseUnavailable("The selected Todo's execution authority could not be read.") - if snapshot.get("handoff_mode") != "hard_lease": - return None - current = _project_execution_facts(_execution_facts(snapshot, goal_id, todo_id, agent_id)) - if current.get("state") != "ready" or current.get("lease_proof") != dict(requested_proof): - raise RefreshTaskLeaseUnavailable("The resumed Turn's active task lease could not be verified.") - epoch = current.get("lease_epoch") - return int(epoch) if epoch is not None else None +def read_turn_execution_lease_qualification( + *, runtime_root: Path, registry_path: Path, + identity: SettlementIdentity, requested_proof: Mapping[str, Any] | None, + goal_ref: Mapping[str, Any] | None = None, + source_admission: Mapping[str, Any] | None = None, + provider_context: Mapping[str, Any] | None = None, +) -> QuotaSettlementReadback: + """Read a fresh TS verdict bound to the original prepared Turn admission. + + The candidate is only transport. The settlement owner loads canonical + authority and decides generation continuity, including retired completion. + """ + from .settlement import read_heartbeat_settlement + + readback = read_heartbeat_settlement( + runtime_root, goal_id=identity.goal_id, agent_id=identity.agent_id, + todo_id=identity.todo_id, turn_instance_id=identity.turn_instance_id, + replan_obligation_id=identity.replan_obligation_id, + registry_path=registry_path, goal_ref=goal_ref, + source_admission=source_admission, + borrow_source_admission=source_admission is not None, + resume_lease_candidate={ + "requested_proof": dict(requested_proof) if requested_proof is not None else None, + **({"provider_context": dict(provider_context)} if provider_context is not None else {}), + }, + ) + if readback is None: + raise RefreshTaskLeaseUnavailable("The original Turn's settlement could not be read.") + qualification = readback.resume_lease_qualification + if not isinstance(qualification, dict) or qualification.get("state") not in {"ready", "not_required"}: + raise RefreshTaskLeaseUnavailable("The original Turn's execution lease could not be qualified.") + return readback def prepared_lease_receipt_fields( @@ -305,6 +318,8 @@ def guard_first_writeback_execution( todo_id: str, agent_id: str, requested_proof: dict[str, Any] | None, settlement: QuotaSettlementReadback, locked_goal: dict[str, Any] | None, + goal_ref: Mapping[str, Any] | None = None, + source_admission: Mapping[str, Any] | None = None, ) -> Iterator[None]: prepared_lease = settlement.prepared_lease prepared_required = settlement.prepared_lease_required or prepared_lease is not None @@ -322,49 +337,41 @@ def guard_first_writeback_execution( operation="refresh-state-execution", )) require_shadow_primary_write_allowed(runtime_root, goal_id) - snapshot = _read_execution_snapshot(runtime_root, goal_id, todo_id) - if snapshot is None: - raise RefreshTaskLeaseUnavailable("The selected Todo's execution authority could not be read.") - if prepared_required and snapshot.get("handoff_mode") != "hard_lease": - raise RefreshTaskLeaseUnavailable("The Todo's original hard-lease authority is unavailable.") - if snapshot.get("handoff_mode") == "hard_lease": - todo_open = snapshot["todo"].get("status") == "open" - facts = _execution_facts(snapshot, goal_id, todo_id, agent_id) - current = _project_execution_facts({**facts, "include_retired_lease_for_writeback": not todo_open}) - current_proof = (current.get("lease_proof") if current.get("state") == "ready" - else current.get("retired_lease_proof")) - if todo_open or prepared_required: - # Legacy prepared receipts can prove their exact version, but cannot - # advance to a later version without a recorded generation. - lease_epoch = (current.get("lease_epoch") - if prepared_lease and "prepared_lease_epoch" in prepared_lease else None) - # Recovery can read the retired proof from the same locked - # snapshot, including a renewal within the admitted generation. - proof = current_proof if not todo_open and requested_proof is None else requested_proof - execution_bound = False - if prepared_lease is not None and proof is not None and proof == current_proof: - observed = prepared_lease_receipt_fields(proof, lease_epoch=lease_epoch) - admitted_version = prepared_lease["prepared_lease_version"] - if ( - not todo_open and lease_epoch is not None - and observed["prepared_lease_version"] > admitted_version - ): - observed["prepared_lease_version"] = admitted_version - execution_bound = prepared_lease == observed - else: + hard_lease_required = prepared_required + if prepared_required: + identity = settlement.identity.value + if identity is None: + raise RefreshTaskLeaseUnavailable("The original Turn's settlement identity is unavailable.") + qualified = read_turn_execution_lease_qualification( + runtime_root=runtime_root, registry_path=registry_path, + identity=identity, requested_proof=requested_proof, + goal_ref=goal_ref, source_admission=source_admission, + ) + if (qualified.resume_lease_qualification or {}).get("state") != "ready": + raise RefreshTaskLeaseUnavailable("The Todo's original hard-lease authority is unavailable.") + else: + snapshot = _read_execution_snapshot(runtime_root, goal_id, todo_id) + if snapshot is None: + raise RefreshTaskLeaseUnavailable("The selected Todo's execution authority could not be read.") + hard_lease_required = snapshot.get("handoff_mode") == "hard_lease" + if hard_lease_required: + todo_open = snapshot["todo"].get("status") == "open" + facts = _execution_facts(snapshot, goal_id, todo_id, agent_id) + current = _project_execution_facts({**facts, "include_retired_lease_for_writeback": not todo_open}) + current_proof = (current.get("lease_proof") if current.get("state") == "ready" + else current.get("retired_lease_proof")) # Older admitted closeouts may predate persisted lease readiness. # Their retired holder still has to match the original binding. admission = settlement.native_child_admission or {} - execution_bound = (current_proof is not None + execution_bound = (not todo_open and current_proof is not None and admission.get("report_permission") in {"new_operation", "existing_operation_only"} and (requested_proof is None or requested_proof == current_proof)) - if not execution_bound: - raise RefreshTaskLeaseUnavailable( - "The original writeback needs a current task lease recorded for this Turn. " - "Read the same Todo and Turn through its current guard before retrying." - ) - if snapshot["todo"].get("status") == "done" and prepared_required and settlement.completion_event is None: - raise RefreshTaskLeaseUnavailable("The completed Todo is missing this Turn's completion receipt.") + if not execution_bound: + raise RefreshTaskLeaseUnavailable( + "The original writeback needs a current task lease recorded for this Turn. " + "Read the same Todo and Turn through its current guard before retrying." + ) + if hard_lease_required: goal = locked_goal if goal is None: source, _ = load_recommendation_source_goal(registry_path, goal_id) diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index 364a5ac6d6..493cf438f8 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -264,19 +264,36 @@ def ensure_turn_heartbeat_settlement_receipt( raise HeartbeatReceiptIdentityConflictError( "Turn heartbeat receipt belongs to another quota spend source" ) - if before_bind is not None: - # The existing cross-runtime binding lock serializes the competing - # identity before transport-specific start admission. Its owner may - # refuse admission without writing a receipt or claiming work. - before_bind() if task_lease_proof is not None and identity.todo_id is not None: from .execution_preparation import prepared_lease_receipt_fields + from ..effect_runtime import EffectRuntimeRemoteError, effect_runtime_result - rebound_lease_fields = prepared_lease_receipt_fields( + proposed_lease_fields = prepared_lease_receipt_fields( task_lease_proof, lease_epoch=task_lease_epoch, ) + try: + publication = effect_runtime_result("quota.heartbeat_prepared_publication.qualify", { + "schema_version": "loopx_quota_heartbeat_prepared_publication_request_v0", + "existing_receipt": effective, + "original_prepared": None, + "receipt_history": matching, + "settlement_identity": identity.as_dict(), + "proposed_prepared_fields": proposed_lease_fields, + }) + except EffectRuntimeRemoteError as error: + if error.diagnostic_code == "heartbeat_receipt_identity_conflict": + raise HeartbeatReceiptIdentityConflictError(str(error)) from error + raise + if publication.get("ok") is not True or not isinstance(publication.get("prepared_fields"), Mapping): + raise RuntimeError("typed prepared publication returned an invalid result") + prepared_lease_fields = dict(publication["prepared_fields"]) else: - rebound_lease_fields = None + prepared_lease_fields = None + if before_bind is not None: + # The binding lock and typed immutable-proof check precede any + # transport-specific start admission. Resume authority is a separate + # ephemeral qualification; it never rewrites prepared fields here. + before_bind() preserved_details: dict[str, object] = {} if effective is not None: if observed is not None: @@ -287,13 +304,8 @@ def ensure_turn_heartbeat_settlement_receipt( else {} ) preserved_details = dict(effective_details) - lease_unchanged = rebound_lease_fields is None or ( - all(effective_details.get(k) == v for k, v in rebound_lease_fields.items()) - and (task_lease_epoch is not None or "prepared_lease_epoch" not in effective_details) - ) if not semantic_replan_guard_scoped: - if lease_unchanged: - return effective + return effective elif "semantic_replan_obligation_id" in effective_details: raw_existing_guard = effective_details.get( "semantic_replan_obligation_id" @@ -323,8 +335,7 @@ def ensure_turn_heartbeat_settlement_receipt( raise HeartbeatReceiptIdentityConflictError( "Turn heartbeat receipt belongs to another capability guard" ) - if lease_unchanged: - return effective + return effective details = { **preserved_details, @@ -338,10 +349,8 @@ def ensure_turn_heartbeat_settlement_receipt( } if identity.todo_id is not None and (prepared_lease_required or task_lease_proof is not None): details["prepared_lease_required"] = True - if rebound_lease_fields is not None: - details.update(rebound_lease_fields) - if task_lease_epoch is None: - details.pop("prepared_lease_epoch", None) + if prepared_lease_fields is not None: + details.update(prepared_lease_fields) if semantic_replan_guard_scoped: details["semantic_replan_obligation_id"] = ( normalized_semantic_replan_obligation_id or "" diff --git a/loopx/control_plane/quota/heartbeat_receipt_identity.ts b/loopx/control_plane/quota/heartbeat_receipt_identity.ts index 4a0cb70da2..026c50b1ff 100644 --- a/loopx/control_plane/quota/heartbeat_receipt_identity.ts +++ b/loopx/control_plane/quota/heartbeat_receipt_identity.ts @@ -10,12 +10,16 @@ * this rule, so it lives here rather than in either caller. */ import type { JsonObject } from "../effect_program.ts"; -import {executionRequestSubject, settlementIdentity, type SettlementIdentity, type ExecutionRequestSubject} from "../effect_program.ts"; +import {effectIdsMatch, executionRequestSubject, settlementIdentity, settlementIdentityPayload, type SettlementIdentity, type ExecutionRequestSubject} from "../effect_program.ts"; +import {isDeepStrictEqual} from "node:util"; +import {createHash} from "node:crypto"; +import {leaseIsActive} from "../work_items/task_lease_record.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; import { BARE_SHA256_PATTERN } from "../content_digest.ts"; -import { jsonObject } from "../runtime_decode.ts"; +import { jsonObject, requireJsonObject, requireNonEmptyString } from "../runtime_decode.ts"; import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; import type {ReceiptBoundReplayPhase} from "./settlement_phase.ts"; +import type {ProviderEffect} from "../turn_driver/settlement_provider.ts"; const TODO_ID_PATTERN = /^todo_[a-z0-9_-]{3,64}$/; const REPLAN_OBLIGATION_ID_PATTERN = /^replan-[a-f0-9]{16}$/; @@ -231,6 +235,101 @@ interface HeartbeatPreparedLease extends JsonObject { prepared_lease_epoch?: number; } +export const HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA = + "loopx_quota_heartbeat_prepared_publication_request_v0"; + +/** Publication compatibility is separate from execution readiness. A null + * proposal preserves stored history; it cannot renew or repair a proof. */ +export function qualifyHeartbeatPreparedPublication(value: unknown): JsonObject { + const prepared = (fields: JsonObject): HeartbeatPreparedLease | null => { + try {return heartbeatPreparedLease(fields);} + catch (error) { + if (error instanceof EffectRuntimeRequestError) throw receiptIdentityConflict( + "prepared receipt publication requires a complete exact proof"); + throw error; + } + }; + const request = requireJsonObject(value, "prepared receipt publication"); + if (request.schema_version !== HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA + || Object.keys(request).some(key => !["schema_version", "existing_receipt", + "original_prepared", "proposed_prepared_fields", "receipt_history", "settlement_identity"].includes(key))) { + throw receiptIdentityConflict("prepared receipt publication request is malformed"); + } + let existing = request.existing_receipt === null ? null + : requireJsonObject(request.existing_receipt, "existing heartbeat receipt"); + const originalFields = request.original_prepared === null ? null + : requireJsonObject(request.original_prepared, "original prepared proof"); + const suppliedOriginal = originalFields === null ? null : prepared(originalFields); + if (originalFields !== null && (suppliedOriginal === null || !isDeepStrictEqual(originalFields, suppliedOriginal))) { + throw receiptIdentityConflict("original prepared proof is incomplete"); + } + let original = suppliedOriginal; + const hasHistory = Object.hasOwn(request, "receipt_history"); + if (hasHistory !== Object.hasOwn(request, "settlement_identity")) { + throw receiptIdentityConflict("publication receipt history requires its exact settlement identity"); + } + if (hasHistory) { + if (!Array.isArray(request.receipt_history)) throw receiptIdentityConflict("publication receipt history is malformed"); + const binding = requireJsonObject(request.settlement_identity, "publication settlement identity"); + const identity = settlementIdentity({goal_id: requireNonEmptyString(binding.goal_id, "goal_id"), + agent_id: requireNonEmptyString(binding.agent_id, "agent_id"), + todo_id: optionalHeartbeatString(binding.todo_id), + turn_instance_id: requireNonEmptyString(binding.turn_instance_id, "turn_instance_id"), + replan_obligation_id: optionalHeartbeatString(binding.replan_obligation_id), + request_subject: executionRequestSubject(binding.request_subject)}); + if (!isDeepStrictEqual(binding, settlementIdentityPayload(identity))) { + throw receiptIdentityConflict("publication must preserve the exact settlement identity"); + } + const history = request.receipt_history.map(row => requireJsonObject(row, "heartbeat history row")) + .filter(row => row.event_kind === "quota_should_run" && row.goal_id === identity.goal_id + && row.agent_id === identity.agent_id && row.run_id === identity.turn_instance_id); + if (history.length && history.some(row => !isDeepStrictEqual(row.goal_ref ?? null, + (existing ?? history[0]!).goal_ref ?? null))) { + throw receiptIdentityConflict("publication receipt history crosses Goal instances"); + } + const effective = history.length === 0 ? null : selectEffectiveHeartbeatReceipt(identity.goal_id, + identity.agent_id, history.map(value => ({value, fact: heartbeatReceiptFactFromEvent(value)}))); + if (existing !== null && !isDeepStrictEqual(existing, effective)) { + throw receiptIdentityConflict("publication effective receipt differs from its locked history"); + } + existing = effective; + const allowLegacyPrepared = effective !== null && !Object.hasOwn(effective, "goal_ref"); + const validation = effective === null ? null + : heartbeatValidationEvent(history, effective, identity, allowLegacyPrepared); + if (effective !== null && validation === null) { + throw receiptIdentityConflict("publication prepared receipt causality is incomplete"); + } + original = validation === null ? null + : heartbeatPreparedWorkAdmission(validation, identity, allowLegacyPrepared); + if (suppliedOriginal !== null && !isDeepStrictEqual(suppliedOriginal, original)) { + throw receiptIdentityConflict("supplied original proof differs from its causal admission"); + } + } + const storedDetails = heartbeatReceiptDetails(existing); + const stored = prepared(storedDetails); + const retained: JsonObject = {...stored, + ...(Object.hasOwn(storedDetails, "prepared_lease_required") + ? {prepared_lease_required: storedDetails.prepared_lease_required} : {})}; + if (request.proposed_prepared_fields === null) return {ok: true, prepared_fields: retained}; + const proposal = requireJsonObject(request.proposed_prepared_fields, "proposed prepared fields"); + if (Object.keys(proposal).some(key => !["prepared_lease_required", "prepared_lease_key_sha256", + "prepared_lease_version", "prepared_lease_epoch"].includes(key)) + || (Object.hasOwn(proposal, "prepared_lease_required") + && typeof proposal.prepared_lease_required !== "boolean")) { + throw receiptIdentityConflict("proposed prepared fields are malformed"); + } + const proposed = prepared(proposal); + if (proposed === null) throw receiptIdentityConflict("a non-null publication proposal requires a complete proof"); + if ((!hasHistory && stored !== null && original !== null && !isDeepStrictEqual(stored, original)) + || !isDeepStrictEqual(original ?? stored ?? proposed, proposed) + || (storedDetails.prepared_lease_required === true && proposal.prepared_lease_required === false)) { + throw receiptIdentityConflict("prepared receipt publication must preserve the exact original proof"); + } + // A valid replay of the causal root does not rewrite a later stored + // readiness record. The append/binder retains that record's exact fields. + return {ok: true, prepared_fields: stored === null ? {...retained, ...proposal} : retained}; +} + export function heartbeatPreparedLease(details: JsonObject): HeartbeatPreparedLease | null { const key = details.prepared_lease_key_sha256; const version = details.prepared_lease_version; @@ -245,6 +344,213 @@ export function heartbeatPreparedLease(details: JsonObject): HeartbeatPreparedLe ...(epoch === undefined ? {} : {prepared_lease_epoch: epoch})}; } +/** Positive prepared work admission, using the existing identity and report + * admission owners. A pre-upgrade receipt may omit the derived effect id. */ +export function heartbeatPreparedWorkAdmission( + event: JsonObject, + identity: SettlementIdentity, + allowLegacyPrepared = false, +): HeartbeatPreparedLease | null { + const fact = heartbeatReceiptFactFromEvent(event); + const binding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); + const receiptDetails = heartbeatReceiptDetails(event); + // The ordinary Turn guard predates work projections. This compatibility is + // explicit at alias callers and cannot promote partial modern facts or + // supply the strict work admission required by execution-loss closeout. + const legacyPrepared = allowLegacyPrepared && !Object.hasOwn(event, "goal_ref") + && event.status === "turn_run_once" && !Object.hasOwn(receiptDetails, "closeout_required") + && receiptDetails.prepared_lease_required === true + && ["must_attempt_work", "delivery_allowed", "quiet_noop_allowed"] + .every(field => !Object.hasOwn(receiptDetails, field)); + if (event.schema_version !== "loopx_rollout_event_v0" || event.event_kind !== "quota_should_run" + || event.goal_id !== identity.goal_id || event.agent_id !== identity.agent_id + || fact.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id + || binding?.binding_kind !== "todo" || binding.binding_id !== identity.todo_id + || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id)) + || (fact.closeout_required !== true && !legacyPrepared) + || (Object.hasOwn(receiptDetails, "prepared_lease_required") && receiptDetails.prepared_lease_required !== true) + || nativeChildReportAdmission(receiptDetails, event.status, identity.effect_id, "open", false, + normalizeHeartbeatReplanObligationId(receiptDetails.semantic_replan_obligation_id)) + .report_permission !== "new_operation") return null; + return heartbeatPreparedLease(receiptDetails); +} + +/** One causal owner for binding validation and original positive preparation. + * Readiness, work and identity revisions cannot replace an earlier admission. */ +export function heartbeatValidationEvent( + events: readonly JsonObject[], + receipt: JsonObject, + identity: SettlementIdentity, + allowLegacyPrepared = false, +): JsonObject | null { + const scoped = events.filter(event => event.event_kind === "quota_should_run" + && event.goal_id === identity.goal_id && event.agent_id === identity.agent_id + && event.run_id === identity.turn_instance_id + && isDeepStrictEqual(event.goal_ref ?? null, receipt.goal_ref ?? null)); + const positions = new Map(); + for (const [index, event] of scoped.entries()) { + const indices = positions.get(event.event_id) ?? []; + indices.push(index); positions.set(event.event_id, indices); + } + const start = positions.get(receipt.event_id); + if (start?.length !== 1 || !isDeepStrictEqual(scoped[start[0]!], receipt)) return null; + const initialFact = heartbeatReceiptFactFromEvent(receipt); + const initialBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, initialFact); + if (initialBinding !== null && (initialBinding.binding_kind !== identity.binding_kind + || initialBinding.binding_id !== identity.binding_id + || (initialFact.settlement_effect_id !== null + && !effectIdsMatch(initialFact.settlement_effect_id, identity.effect_id)))) return null; + const generations = new Set(); + const positivePrepared = new Map(); + for (const [index, event] of scoped.entries()) { + const prepared = heartbeatPreparedWorkAdmission(event, identity, allowLegacyPrepared); + if (prepared === null) continue; + positivePrepared.set(index, prepared); + generations.add(JSON.stringify(prepared.prepared_lease_epoch === undefined + ? ["legacy", prepared.prepared_lease_key_sha256, prepared.prepared_lease_version] + : ["epoch", prepared.prepared_lease_key_sha256, prepared.prepared_lease_epoch])); + } + if (generations.size > 1) throw receiptIdentityConflict( + "one Turn cannot contain positive prepared admissions from different lease generations"); + let position = start[0]!; + let validation = scoped[position]!; + let preparedAdmission: JsonObject | null = null; + while (position >= 0) { + const prepared = positivePrepared.get(position); + const positive = prepared !== undefined; + if (positive) preparedAdmission = validation; + const validationDetails = heartbeatReceiptDetails(validation); + const revision = validationDetails.settlement_receipt_revision; + const legacyLineage = !Object.hasOwn(validationDetails, "settlement_receipt_revision") + && positive && Object.hasOwn(validation, "causality"); + if (revision !== "lease_readiness" && revision !== "work_admission" + && revision !== "identity_upgrade" && revision !== "review_guard_upgrade" && !legacyLineage) { + if (positive && !Object.hasOwn(validationDetails, "settlement_receipt_revision") + && !Object.hasOwn(validation, "causality")) { + let first: JsonObject | null = null; + for (const [index, priorPrepared] of positivePrepared) { + if (index >= position) break; + if (!isDeepStrictEqual(priorPrepared, prepared)) throw receiptIdentityConflict( + "an uncaused prepared receipt cannot redefine this Turn's original proof"); + first ??= scoped[index]!; + } + return first ?? validation; + } + return preparedAdmission ?? validation; + } + const lineage = jsonObject(validation.causality); + const parentId = optionalHeartbeatString(lineage?.source_event_id); + if (parentId === null || lineage?.caused_by !== parentId) return null; + const parentPositions = positions.get(parentId); + if (parentPositions?.length !== 1 || parentPositions[0]! >= position) return null; + const parent = scoped[parentPositions[0]!]!; + const fact = heartbeatReceiptFactFromEvent(parent); + const parentDetails = heartbeatReceiptDetails(parent); + if (legacyLineage) { + const parentBinding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); + if ((parentBinding !== null && (parentBinding.binding_kind !== identity.binding_kind + || parentBinding.binding_id !== identity.binding_id)) + || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id))) return null; + if (heartbeatPreparedWorkAdmission(parent, identity, allowLegacyPrepared) === null) return preparedAdmission; + } + const pendingParent = revision === "identity_upgrade" && identity.binding_kind === "todo" + && fact.todo_id === null && fact.replan_obligation_id === null && fact.request_subject == null + && fact.settlement_effect_id === null && !fact.closeout_required + && parentDetails.pending_action_selection_state === "deferred" + && parentDetails.pending_action_selection_todo_id === identity.todo_id + && normalizeHeartbeatTodoId(parentDetails.pending_action_selection_todo_id) === identity.todo_id; + if ((!pendingParent && fact.todo_id !== identity.todo_id) + || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id))) return null; + validation = parent; + position = parentPositions[0]!; + } + return null; +} + +/** Ephemeral qualification over freshly loaded canonical facts. This never + * publishes a new prepared proof or grants a replacement generation work. */ +export function qualifyHeartbeatResumeLease(input: { + originalReceipt: JsonObject; + identity: SettlementIdentity; + requestedProof: JsonObject | null; + todo: JsonObject; + lease: JsonObject; + handoffMode: unknown; + completionEvent: JsonObject | null; + replayPhase: ReceiptBoundReplayPhase; + closeoutStarted: boolean; + writebackPending: boolean; + otherTerminalStarted: boolean; + terminalProviderEffect?: ProviderEffect | null; + terminalCompletion?: JsonObject | null; + exactSource: boolean; + observedAt: Date; +}): JsonObject { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const original = heartbeatPreparedWorkAdmission(input.originalReceipt, input.identity, !input.exactSource); + if (original === null || input.handoffMode !== "hard_lease") { + reject("The original Turn's prepared work admission is unavailable."); + } + const {todo, lease, identity} = input; + if (todo.todo_id !== identity.todo_id || lease.todo_id !== identity.todo_id + || lease.owner !== identity.agent_id + || (todo.claimed_by != null && todo.claimed_by !== identity.agent_id) + || typeof lease.idempotency_key !== "string" || !lease.idempotency_key + || lease.idempotency_key !== lease.idempotency_key.trim() + || typeof lease.version !== "number" || !Number.isSafeInteger(lease.version) || lease.version < 1) { + reject("The original Turn's current canonical lease proof is unavailable."); + } + const currentProof = {idempotency_key: lease.idempotency_key, expected_version: lease.version}; + if (input.requestedProof !== null && !isDeepStrictEqual(input.requestedProof, currentProof)) { + reject("The supplied resume proof does not match the current canonical lease."); + } + const keyDigest = createHash("sha256").update(lease.idempotency_key).digest("hex"); + if (keyDigest !== original.prepared_lease_key_sha256) { + reject("A replacement lease cannot execute this Turn's original work."); + } + if (original.prepared_lease_epoch === undefined) { + // Preserve the existing legacy exact-version contract, without claiming + // incarnation/ABA protection or allowing an unrecorded renewal/rebind. + if (input.exactSource || lease.version !== original.prepared_lease_version) { + reject("This prepared receipt cannot prove a renewed lease generation."); + } + } else if (typeof lease.lease_epoch !== "number" || !Number.isSafeInteger(lease.lease_epoch) + || lease.lease_epoch !== original.prepared_lease_epoch + || lease.version < original.prepared_lease_version) { + reject("The current lease does not preserve the original prepared generation."); + } + const terminalPending = input.terminalProviderEffect?.step_kind === "terminal_closeout" + && input.terminalProviderEffect.effect_ref === `${identity.effect_id}#terminal_closeout` + && ["prepare_and_execute", "resolve_prepared", "execute_prepared"].includes(input.terminalProviderEffect.action); + const active = todo.status === "open" + && input.requestedProof !== null + && ((input.replayPhase === "open" && !input.closeoutStarted) || terminalPending) + && leaseIsActive(lease, input.observedAt); + const completion = input.completionEvent; + const retiredWriteback = input.writebackPending + && !input.otherTerminalStarted && completion !== null + && completion.event_kind === "todo_complete" && completion.goal_id === identity.goal_id + && completion.agent_id === identity.agent_id && completion.run_id === identity.turn_instance_id + && (completion.todo_id == null || completion.todo_id === identity.todo_id) + && (heartbeatReceiptDetails(completion).todo_id == null + || heartbeatReceiptDetails(completion).todo_id === identity.todo_id) + && effectIdsMatch(optionalHeartbeatString(heartbeatReceiptDetails(completion).settlement_effect_id), identity.effect_id); + const terminalCompletion = jsonObject(input.terminalCompletion?.completion); + const retiredTerminal = terminalPending && input.terminalProviderEffect?.action === "resolve_prepared" + && input.terminalCompletion?.kind === "committed" && terminalCompletion?.todo_id === identity.todo_id + && terminalCompletion.continuation === "no_followup"; + const retired = todo.status === "done" && lease.status === "released" && (retiredWriteback || retiredTerminal); + if (!active && !retired) { + reject("The original Turn requires active execution or its exact retired completion proof."); + } + return {state: "ready", original_prepared: {...original}, + current_task_lease_proof: currentProof, + lease_epoch: original.prepared_lease_epoch ?? null, + source: active ? "active" : "retired"}; +} + export function isHeartbeatLeaseReadiness(event: JsonObject): boolean { return heartbeatReceiptDetails(event).settlement_receipt_revision === "lease_readiness"; } diff --git a/loopx/control_plane/quota/settlement.py b/loopx/control_plane/quota/settlement.py index 99747d82bf..ccda951f8e 100644 --- a/loopx/control_plane/quota/settlement.py +++ b/loopx/control_plane/quota/settlement.py @@ -186,6 +186,11 @@ class QuotaSettlementReadback: admission_event_id: str | None = None prepared_lease: dict[str, Any] | None = None prepared_lease_required: bool = False + resume_lease_qualification: dict[str, Any] | None = None + original_prepared_receipt: dict[str, Any] | None = None + original_prepared_event_id: str | None = None + original_prepared_receipt_sha256: str | None = None + native_task_lease_cleanup: dict[str, Any] | None = None def render_owed_host_command(argv: list[str]) -> str: @@ -399,6 +404,10 @@ def read(admission: Mapping[str, Any] | None) -> Any: except EffectRuntimeRejected as exc: if exc.diagnostic_code == "heartbeat_receipt_identity_conflict": raise HeartbeatReceiptIdentityConflictError(str(exc)) from None + if exc.diagnostic_code == "refresh_task_lease_unavailable": + from .execution_preparation import RefreshTaskLeaseUnavailable + + raise RefreshTaskLeaseUnavailable(str(exc)) from None raise ValueError(str(exc)) from None if source_admission is not None: @@ -496,6 +505,8 @@ def read_heartbeat_settlement( source_admission: Mapping[str, Any] | None = None, borrow_source_admission: bool = False, heartbeat_reentry_guard: dict[str, Any] | None = None, + resume_lease_candidate: Mapping[str, Any] | None = None, + native_task_lease_cleanup: Mapping[str, Any] | None = None, include_record_bodies: bool = True, tolerate_malformed_history: bool = False, ) -> QuotaSettlementReadback | None: @@ -530,6 +541,16 @@ def read_heartbeat_settlement( if heartbeat_reentry_guard is not None else {} ), + **( + {"resume_lease_candidate": dict(resume_lease_candidate)} + if resume_lease_candidate is not None + else {} + ), + **( + {"native_task_lease_cleanup": dict(native_task_lease_cleanup)} + if native_task_lease_cleanup is not None + else {} + ), }, registry_path=registry_path, goal_ref=goal_ref, @@ -589,4 +610,9 @@ def read_heartbeat_settlement( admission_event_id=_optional_readback_text(payload.get("admission_event_id")), prepared_lease=_optional_readback_record(payload.get("prepared_lease")), prepared_lease_required=payload.get("prepared_lease_required") is True, + resume_lease_qualification=_optional_readback_record(payload.get("resume_lease_qualification")), + original_prepared_receipt=_optional_readback_record(payload.get("original_prepared_receipt")), + original_prepared_event_id=_optional_readback_text(payload.get("original_prepared_event_id")), + original_prepared_receipt_sha256=_optional_readback_text(payload.get("original_prepared_receipt_sha256")), + native_task_lease_cleanup=_optional_readback_record(payload.get("native_task_lease_cleanup")), ) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 891b6d7f33..2209e7156e 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -28,6 +28,7 @@ import { hasMonitorPollTransaction } from "./monitor_poll_commit.ts"; import { readQuotaSpendEventFromRun } from "./accounting_artifact_transaction.ts"; import { jsonObject, + requireBoolean, requireJsonObject, requireNonEmptyString, } from "../runtime_decode.ts"; @@ -69,7 +70,9 @@ import { selectEffectiveHeartbeatReceipt, heartbeatWorkRequalification, heartbeatPreparedLease, - isHeartbeatLeaseReadiness, + heartbeatPreparedWorkAdmission, + heartbeatValidationEvent, + qualifyHeartbeatResumeLease, type HeartbeatReceiptFact, } from "./heartbeat_receipt_identity.ts"; @@ -101,6 +104,60 @@ export const QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA = export const QUOTA_SETTLEMENT_READBACK_RESULT_SCHEMA = "loopx_quota_settlement_readback_result_v0"; export const SEMANTIC_REPLAN_GUARD_SCHEMA = "semantic_replan_guard_v0"; +export const NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA = "native_task_lease_cleanup_request_v0"; +export const NATIVE_TASK_LEASE_CLEANUP_SCHEMA = "native_task_lease_cleanup_v0"; + +interface NativeTaskLeaseCleanupRequest { + schema_version: typeof NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA; + recovery: boolean; + admission_retry: boolean; + resume_turn_key_present: boolean; + original_journal_present: boolean; +} + +function decodeNativeTaskLeaseCleanup(value: unknown): NativeTaskLeaseCleanupRequest { + const request = requireJsonObject(value, "native task lease cleanup request"); + if (request.schema_version !== NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA || + Object.keys(request).some(key => !["schema_version", "recovery", "admission_retry", + "resume_turn_key_present", "original_journal_present"].includes(key))) { + throw new EffectRuntimeRequestError("native task lease cleanup request shape mismatch"); + } + return {schema_version: NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA, + recovery: requireBoolean(request.recovery, "cleanup.recovery"), + admission_retry: requireBoolean(request.admission_retry, "cleanup.admission_retry"), + resume_turn_key_present: requireBoolean(request.resume_turn_key_present, "cleanup.resume_turn_key_present"), + original_journal_present: requireBoolean(request.original_journal_present, "cleanup.original_journal_present")}; +} + +/** Receipt cleanup eligibility belongs to the settlement owner. Host/journal + * facts never substitute for a readable admitted receipt or settle its debt. */ +export function projectNativeTaskLeaseCleanup(readback: JsonObject, value: unknown): JsonObject { + const request = decodeNativeTaskLeaseCleanup(value); + const disposition = (action: "release" | "retain" | "unconfirmed", reason: string): JsonObject => + ({schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, disposition: action, reason}); + const identity = jsonObject(jsonObject(readback.identity)?.result); + const failure = jsonObject(identity?.failure); + if (failure !== null) { + if (failure.kind === "receipt_unbound") return disposition("release", "receipt_unbound"); + if (failure.kind === "receipt_missing" && !request.recovery && !request.admission_retry + && !request.resume_turn_key_present && !request.original_journal_present) { + return disposition("release", "fresh_pre_admission_missing"); + } + return disposition("unconfirmed", "original_identity_unconfirmed"); + } + if (identity?.value == null) return disposition("unconfirmed", "original_identity_unconfirmed"); + if (readback.replay_phase === "settled") return disposition("release", "original_turn_settled"); + const receipt = jsonObject(readback.heartbeat_receipt), receiptDetails = jsonObject(receipt?.details); + if (receiptDetails === null) return disposition("unconfirmed", "original_receipt_unconfirmed"); + const revision = receiptDetails.settlement_receipt_revision, required = receiptDetails.closeout_required; + if (revision === "turn_reservation" && (!Object.hasOwn(receiptDetails, "closeout_required") || required === false)) { + return disposition("release", "original_reservation_without_closeout"); + } + if (["work_admission", "lease_readiness", "identity_upgrade"].includes(String(revision)) && required === true) { + return disposition("retain", "original_turn_owes_closeout"); + } + return disposition("unconfirmed", "original_closeout_requirement_unconfirmed"); +} // Where the guard compiles the repair for an owed prior Turn. The repair lane // depends on the bound Todo's facts, which this read model does not read, so @@ -111,6 +168,11 @@ interface OwedPriorTurn extends PriorHostTurnCloseoutCandidate { missing_receipts: string[]; } +interface ResumeLeaseCandidate { + requested_proof: JsonObject | null; + provider_context?: {turn_key: string; effect_ref: string | null}; +} + type OwedPriorTurnLookup = (agentId: string, turnInstanceId: string) => OwedPriorTurn | null; const TURN_INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; @@ -132,6 +194,8 @@ interface ReadbackRequest { tolerate_malformed_history: boolean; refresh_retry: RefreshRetryRequest | null; heartbeat_reentry_guard: JsonObject | null; + resume_lease_candidate: ResumeLeaseCandidate | null; + native_task_lease_cleanup: NativeTaskLeaseCleanupRequest | null; owner: QuotaAccountingOwner; } @@ -162,6 +226,36 @@ interface SpendTerms { source: unknown; } +function decodeResumeLeaseCandidate(value: unknown): ResumeLeaseCandidate | null { + if (value === undefined) return null; + const candidate = requireJsonObject(value, "resume_lease_candidate"); + if (Object.keys(candidate).some(key => key !== "requested_proof" && key !== "provider_context") + || !Object.hasOwn(candidate, "requested_proof")) { + throw new EffectRuntimeRequestError("resume_lease_candidate requires only requested_proof and optional provider_context"); + } + let providerContext: ResumeLeaseCandidate["provider_context"]; + if (Object.hasOwn(candidate, "provider_context")) { + const context = requireJsonObject(candidate.provider_context, "resume provider_context"); + if (Object.keys(context).length !== 2 || typeof context.turn_key !== "string" + || !context.turn_key || context.turn_key !== context.turn_key.trim() + || (context.effect_ref !== null && (typeof context.effect_ref !== "string" + || !context.effect_ref || context.effect_ref !== context.effect_ref.trim()))) { + throw new EffectRuntimeRequestError("resume provider_context requires an exact turn_key and nullable effect_ref"); + } + providerContext = {turn_key: context.turn_key, effect_ref: context.effect_ref}; + } + const contextFields = providerContext === undefined ? {} : {provider_context: providerContext}; + if (candidate.requested_proof === null) return {requested_proof: null, ...contextFields}; + const proof = requireJsonObject(candidate.requested_proof, "requested resume proof"); + if (Object.keys(proof).length !== 2 || typeof proof.idempotency_key !== "string" + || !proof.idempotency_key || proof.idempotency_key !== proof.idempotency_key.trim() + || typeof proof.expected_version !== "number" || !Number.isSafeInteger(proof.expected_version) + || proof.expected_version < 1) { + throw new EffectRuntimeRequestError("requested resume proof requires an exact key and positive safe-integer version"); + } + return {requested_proof: proof, ...contextFields}; +} + const ORIGINAL_TURN_SPEND_SLOTS = 1; type SettlementProgressState = "identity_required" | "writeback_required" | @@ -314,6 +408,26 @@ function decodeRequest( "quota_source_admission_invalid", ); } + const resumeCandidate = decodeResumeLeaseCandidate(request.resume_lease_candidate); + if (resumeCandidate !== null && (request.infer_turn_instance_id + || request.allow_unbound_binding || request.pending_closeout === true + || optionalRequestString(request.agent_id, "agent_id") === null + || optionalRequestString(request.todo_id, "todo_id") === null + || optionalRequestString(request.turn_instance_id, "turn_instance_id") === null)) { + throw new EffectRuntimeRequestError("resume lease qualification requires an exact Agent/Todo/Turn readback"); + } + const nativeCleanup = request.native_task_lease_cleanup === undefined ? null + : decodeNativeTaskLeaseCleanup(request.native_task_lease_cleanup); + if (nativeCleanup !== null && (request.infer_turn_instance_id || request.allow_unbound_binding + || request.resolve_original_binding || request.pending_closeout || request.refresh_retry != null + || request.heartbeat_reentry_guard != null || request.spend_precheck === true + || request.tolerate_malformed_history === true || resumeCandidate !== null + || !optionalRequestString(request.agent_id, "agent_id") + || !optionalRequestString(request.todo_id, "todo_id") + || !optionalRequestString(request.turn_instance_id, "turn_instance_id") + || optionalRequestString(request.replan_obligation_id, "replan_obligation_id") !== null)) { + throw new EffectRuntimeRequestError("native task lease cleanup requires only an explicit original Todo Turn read"); + } return { runtime_root: runtimeRoot, goal_id: goalId, @@ -337,6 +451,8 @@ function decodeRequest( refresh_retry: decodeRefreshRetry(request.refresh_retry), heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null : requireJsonObject(request.heartbeat_reentry_guard, "heartbeat_reentry_guard"), + resume_lease_candidate: resumeCandidate, + native_task_lease_cleanup: nativeCleanup, owner, }; } @@ -792,32 +908,6 @@ function admissionHeartbeatReceipt( : selectAdmissionHeartbeatReceipt(identity.goal_id, identity.agent_id, entries); } -function heartbeatValidationEvent( - events: readonly JsonObject[], - receipt: JsonObject, - identity: SettlementIdentity, -): JsonObject | null { - if (!isHeartbeatLeaseReadiness(receipt)) return receipt; - // Readiness appends to the same Turn; only its original positive work - // admission can serve as validation. Follow durable links toward that root. - let validation = receipt; - for (let index = events.lastIndexOf(receipt) - 1; index >= 0; index -= 1) { - const event = events[index]!; - const lineage = jsonObject(validation.causality); - const parentId = optionalString(lineage?.source_event_id); - if (parentId === null || lineage?.caused_by !== parentId) return null; - if (event.event_id !== parentId) continue; - const fact = heartbeatReceiptFactFromEvent(event); - if (event.event_kind !== "quota_should_run" || - event.goal_id !== identity.goal_id || event.agent_id !== identity.agent_id || - fact.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id || - !effectIdsMatch(fact.settlement_effect_id, identity.effect_id)) return null; - if (!isHeartbeatLeaseReadiness(event)) return event; - validation = event; - } - return null; -} - function writebackResult( identity: SettlementIdentity, run: JsonObject | null, @@ -1386,6 +1476,17 @@ function readQuotaSettlementFromRequest( snapshot: QuotaSettlementReadbackSnapshot, nameOwedPriorTurn = true, matchedSpend?: MatchedSpendRecords, +): JsonObject { + const result = readQuotaSettlementFromRequestBase(request, snapshot, nameOwedPriorTurn, matchedSpend); + if (request.native_task_lease_cleanup === null || result.found !== true) return result; + return {...result, native_task_lease_cleanup: projectNativeTaskLeaseCleanup(result, request.native_task_lease_cleanup)}; +} + +function readQuotaSettlementFromRequestBase( + request: ReadbackRequest, + snapshot: QuotaSettlementReadbackSnapshot, + nameOwedPriorTurn = true, + matchedSpend?: MatchedSpendRecords, ): JsonObject { if ( snapshot.runtimeRoot !== request.runtime_root || @@ -1464,7 +1565,11 @@ function readQuotaSettlementFromRequest( ); } const receiptDetails = details(heartbeatReceipt); - const preparedLease = heartbeatPreparedLease(receiptDetails); + const allowLegacyPrepared = request.owner.kind === "alias"; + const validationReceipt = heartbeatValidationEvent(events, heartbeatReceipt, identity, allowLegacyPrepared); + const preparedLease = validationReceipt === null ? null + : heartbeatPreparedWorkAdmission(validationReceipt, identity, allowLegacyPrepared); + const originalPreparedReceipt = preparedLease === null ? null : validationReceipt; if (Object.hasOwn(receiptDetails, "prepared_lease_required") && receiptDetails.prepared_lease_required !== true) { throw new EffectRuntimeRequestError("Turn readiness requirement is malformed", "malformed_settlement_state"); } @@ -1499,7 +1604,8 @@ function readQuotaSettlementFromRequest( || writebackRun.delivery_outcome !== "outcome_gap" || writebackRun.delivery_boundary !== "semantic_closeout" || writebackRun.delivery_batch_scale !== "single_surface" || writebackRun.generated_at !== closeout.observed_at || closeout.effect_id !== identity.effect_id - || closeout.original_receipt_event_id !== heartbeatReceipt.event_id + || originalPreparedReceipt === null + || closeout.original_receipt_event_id !== originalPreparedReceipt.event_id || !isDeepStrictEqual(closeout.original_prepared, preparedLease) || jsonObject(closeout.current_control_writer)?.owner !== identity.agent_id || closeout.observation_fingerprint !== jsonObject(writebackRun.progress_observation)?.fingerprint @@ -1507,7 +1613,7 @@ function readQuotaSettlementFromRequest( || !commit || Object.keys(commit).sort().join("|") !== ["schema_version", "request_sha256", "original_receipt_sha256", "event"].sort().join("|") || commit.schema_version !== "quota_execution_loss_closeout_commit_v0" - || commit.original_receipt_sha256 !== canonicalAuthoritySha256(heartbeatReceipt) + || commit.original_receipt_sha256 !== canonicalAuthoritySha256(originalPreparedReceipt) || commit.request_sha256 !== canonicalAuthoritySha256({identity: settlementIdentityPayload(identity), goal_ref: quotaGoalRef(request.owner), intent: {delivery_outcome: "outcome_gap", delivery_boundary: "semantic_closeout", progress_observation: writebackRun.progress_observation}}) @@ -1650,6 +1756,10 @@ function readQuotaSettlementFromRequest( ), spend_run: includeRecords ? spendRun : null, heartbeat_receipt: includeRecords ? heartbeatReceipt : null, + original_prepared_receipt: includeRecords ? originalPreparedReceipt : null, + original_prepared_event_id: originalPreparedReceipt?.event_id ?? null, + original_prepared_receipt_sha256: originalPreparedReceipt === null ? null + : canonicalAuthoritySha256(originalPreparedReceipt), ...(preparedLease === null ? {} : {prepared_lease: preparedLease}), ...(receiptDetails.prepared_lease_required === true ? {prepared_lease_required: true} : {}), admission_recorded_at: typeof admissionReceipt?.recorded_at === "string" @@ -1682,6 +1792,8 @@ export function readQuotaSettlementFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value); + if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + "resume lease qualification requires a live canonical authority read"); if (request.owner.kind !== "alias") { throw new EffectRuntimeRequestError( "exact source settlement readback requires live owner admission", @@ -1696,7 +1808,10 @@ export function readAdmittedQuotaSettlementFromSnapshot( value: unknown, snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { - return readQuotaSettlementFromRequest(decodeRequest(value), snapshot); + const request = decodeRequest(value); + if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + "resume lease qualification requires a live canonical authority read"); + return readQuotaSettlementFromRequest(request, snapshot); } /** Read under the parsed owner already admitted by the enclosing transaction. */ @@ -1705,7 +1820,88 @@ export function readQuotaSettlementForAdmittedOwnerFromSnapshot( owner: QuotaAccountingOwner, snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { - return readQuotaSettlementFromRequest(decodeRequest(value, owner), snapshot); + const request = decodeRequest(value, owner); + if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + "resume lease qualification requires a live canonical authority read"); + return readQuotaSettlementFromRequest(request, snapshot); +} + +async function readResumeLeaseQualification(request: ReadbackRequest, result: JsonObject): Promise { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const identityResult = jsonObject(jsonObject(result.identity)?.result); + const identityValue = jsonObject(identityResult?.value); + if (result.found !== true || identityResult?.failure != null || identityValue === null) { + reject("The original Turn's settlement identity could not be verified."); + } + const originalReceipt = jsonObject(result.original_prepared_receipt); + const identity = settlementIdentity({goal_id: String(identityValue.goal_id), + agent_id: String(identityValue.agent_id), todo_id: optionalString(identityValue.todo_id), + turn_instance_id: String(identityValue.turn_instance_id), + replan_obligation_id: optionalString(identityValue.replan_obligation_id), + request_subject: executionRequestSubject(identityValue.request_subject)}); + // The ordinary reader does not load lease decision owners. Only this + // explicitly requested ephemeral branch reads the real provider authority, + // inside the enclosing quota/source admission and without claiming a lease. + const [{openRuntimeAuthorityStore}, {readAuthoritySnapshot}, {canonicalTodoCollection}] = await Promise.all([ + import("../coordination/local_authority_provider.ts"), + import("../coordination/authority_store.ts"), + import("../coordination/local_authority_read.ts"), + ]); + const store = await openRuntimeAuthorityStore(request.runtime_root, request.goal_id, {}, {existingOnly: true}); + const snapshot = await readAuthoritySnapshot(store); + if (snapshot.identity.status !== "available" || snapshot.authority.status !== "loaded") { + reject("The current canonical execution authority is unavailable."); + } + const authority = snapshot.authority; + const collection = canonicalTodoCollection(authority.head, identity.goal_id, true); + const todo = collection.projection.todos.get(identity.todo_id!); + const lease = collection.leaseIndex?.leases.get(identity.todo_id!); + if (todo === undefined) reject("The original Todo's canonical authority is unavailable."); + // Reuse the canonical read contract's within-read incarnation check, not a + // new historical legacy incarnation/ABA gate. + const observed = await store.storeIdentity(); + if (observed.status !== "available" || observed.store_identity !== snapshot.identity.store_identity) { + reject("Canonical authority changed during resume qualification."); + } + if (originalReceipt === null) { + if (request.owner.kind === "exact_source" || authority.head.handoff_mode !== "soft_claim" + || result.prepared_lease_required === true || result.prepared_lease != null) { + reject("The original Turn's positive prepared admission is missing."); + } + return {state: "not_required", original_prepared: null, + current_task_lease_proof: null, lease_epoch: null, source: null}; + } + if (lease === undefined) reject("The original Todo's canonical lease is unavailable."); + const completion = jsonObject(result.completion_event); + const progress = jsonObject(result.progress); + let terminalProviderEffect: import("../turn_driver/settlement_provider.ts").ProviderEffect | null = null; + let terminalCompletion: JsonObject | null = null; + const providerContext = request.resume_lease_candidate!.provider_context; + if (providerContext !== undefined && jsonObject(jsonObject(result.settlement)?.payload)?.ok === true + && [result.writeback_run, result.writeback_event, result.spend_run, result.spend_event] + .every(record => record != null)) { + const {readTurnTerminalCloseoutProvider} = await import("../turn_driver/turn_journal_query.ts"); + terminalProviderEffect = await readTurnTerminalCloseoutProvider(request.runtime_root, + identity, request.owner, providerContext, todo); + if (terminalProviderEffect !== null && todo.status === "done" && lease.status === "released") { + const {projectCompletionEvidence} = await import("../todos/read_projection.ts"); + terminalCompletion = projectCompletionEvidence({operation: "terminal", todo, + expected_todo_id: identity.todo_id, completion_turn_key: identity.turn_instance_id}); + } + } + return qualifyHeartbeatResumeLease({originalReceipt, identity, + requestedProof: request.resume_lease_candidate!.requested_proof, todo, lease, + handoffMode: authority.head.handoff_mode, completionEvent: completion, + replayPhase: result.replay_phase as ReceiptBoundReplayPhase, + closeoutStarted: [result.writeback_run, result.writeback_event, result.spend_run, + result.spend_event, result.completion_event].some(record => record != null), + writebackPending: progress?.state === "writeback_required" || progress?.state === "writeback_receipt_required", + otherTerminalStarted: result.spend_run != null || result.spend_event != null + || result.replay_phase === "settled", + terminalProviderEffect, terminalCompletion, + exactSource: request.owner.kind === "exact_source", observedAt: new Date()}); } export async function readQuotaSettlement(value: unknown): Promise { @@ -1735,6 +1931,7 @@ export async function readQuotaSettlement(value: unknown): Promise { throw new EffectRuntimeRequestError("spend_request requires spend_precheck, integer slots and a source"); } if (spendPrecheck) request.include_record_bodies = true; + if (request.resume_lease_candidate !== null) request.include_record_bodies = true; const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner : withQuotaAccountingOwner; @@ -1749,6 +1946,9 @@ export async function readQuotaSettlement(value: unknown): Promise { undefined, request.tolerate_malformed_history, ), true, matchedSpend); + if (request.resume_lease_candidate !== null) { + result.resume_lease_qualification = await readResumeLeaseQualification(request, result); + } const identity = jsonObject(jsonObject(jsonObject(result.identity)?.result)?.value); const receipt = spendReceiptTerms(matchedSpend.receipt); // Only exact readback opens its matched debit artifact. Check its diff --git a/loopx/control_plane/turn_driver/turn_journal_query.ts b/loopx/control_plane/turn_driver/turn_journal_query.ts index 6ddf11cdfd..620c3a75fe 100644 --- a/loopx/control_plane/turn_driver/turn_journal_query.ts +++ b/loopx/control_plane/turn_driver/turn_journal_query.ts @@ -9,6 +9,7 @@ import { SETTLEMENT_IDENTITY_SCHEMA_VERSION, SCOPED_SETTLEMENT_IDENTITY_SCHEMA_VERSION, type BoundSettlementIdentity, + type SettlementIdentity, type JsonObject, } from "../effect_program.ts"; import { EffectRuntimeConflictError, EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; @@ -23,11 +24,17 @@ import { jsonObject, parseJsonSource, parsedSourceNumberTokens, requireJsonObjec import { interpretTurnJournal, parseTurnJournalGoalBinding, + transactionPhases, + type TurnJournalInspection, } from "./turn_journal.ts"; +import type {QuotaAccountingOwner} from "../quota/source_admission.ts"; +import type {ProviderEffect} from "./settlement_provider.ts"; interface JournalEvidence { turn_key: string; observed_capabilities: string[] | null; + journal: JsonObject; + inspection: TurnJournalInspection; } function decodeIdentity(value: unknown): BoundSettlementIdentity { @@ -136,6 +143,8 @@ function evidenceForJournal( } return { turn_key: turnKey, + journal, + inspection, observed_capabilities: inspection.replay_legal ? observedCapabilities(turnPlanWorkSource(plan ?? {}, sourceNumberTokens(journal, parsedSourceNumberTokens(journal, numbers)).filter(({path}) => path[0] === "plan" && path[1] === "work_contract") @@ -143,6 +152,57 @@ function evidenceForJournal( }; } +/** Resolve only the original journal's next terminal provider. The enclosing + * quota reader already owns source admission; this helper opens no owner and + * returns the existing settlement reducer's effect rather than a new grant. */ +export async function readTurnTerminalCloseoutProvider( + runtimeRoot: string, + identity: SettlementIdentity, + owner: QuotaAccountingOwner, + context: {turn_key: string; effect_ref: string | null}, + todo: JsonObject, +): Promise { + if (identity.binding_kind !== "todo") return null; + const evidence = await queryTurnJournal(runtimeRoot, identity, + owner.kind === "exact_source" ? owner.goalRef : null); + if (evidence === null || evidence.turn_key !== context.turn_key) { + throw new EffectRuntimeRequestError("Terminal provider journal does not match this Turn", + "refresh_task_lease_unavailable"); + } + const {journal, inspection} = evidence; + const goalBinding = parseTurnJournalGoalBinding(journal); + if (owner.kind === "alias" && goalBinding.kind !== "legacy") { + throw new EffectRuntimeRequestError("Terminal provider cannot cross Goal ownership", + "refresh_task_lease_unavailable"); + } + if (inspection.recovery_decision.action !== "continue" + || inspection.recovery_decision.reinvoke_host + || !inspection.completed_phases.includes("quota_spend") + || jsonObject(journal.host_result)?.result_kind !== "validated_completion") return null; + const [{projectCompletionEvidence}, {reduceTurnSettlementTransaction, TURN_SETTLEMENT_TRANSACTION_SCHEMA_VERSION}] = await Promise.all([ + import("../todos/read_projection.ts"), import("./settlement.ts"), + ]); + const intent = projectCompletionEvidence({operation: "intent", todo, expected_todo_id: identity.todo_id}); + if (intent.continuation !== "no_followup") return null; + const transaction = jsonObject(jsonObject(journal.plan)?.transaction); + const reduction = reduceTurnSettlementTransaction({schema_version: TURN_SETTLEMENT_TRANSACTION_SCHEMA_VERSION, + transaction_plan: transaction, transaction_phases: transactionPhases, + completed_phases: inspection.completed_phases, committed_effect_id: identity.effect_id, + writeback_payload: journal.writeback ?? null, quota_spend_payload: journal.quota_spend ?? null, + terminal_closeout_required: true, terminal_closeout_payload: journal.terminal_closeout ?? null, + effect_attempts: journal.effect_attempts ?? {}, turn_result_kind: "validated_completion"}); + if (reduction.decision !== "execute" || reduction.provider_effects.length !== 1) return null; + const effect = reduction.provider_effects[0]!; + if (effect.step_kind !== "terminal_closeout" + || (effect.action !== "prepare_and_execute" && effect.action !== "resolve_prepared" + && effect.action !== "execute_prepared")) return null; + if (context.effect_ref !== null && context.effect_ref !== effect.effect_ref) { + throw new EffectRuntimeRequestError("Terminal provider reference differs from this Turn's next effect", + "refresh_task_lease_unavailable"); + } + return effect; +} + async function queryTurnJournal( runtimeRootValue: unknown, identity: JournalAddress | BoundSettlementIdentity, diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index 943c3dba88..03a9592da1 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -1620,6 +1620,8 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): requested_proof=task_lease_proof, settlement=settlement_readback, locked_goal=current_goal if normalized_next_action else None, + goal_ref=goal_ref, + source_admission=source_admission, )) if settlement_readback is not None: persist_external_delivery_pause( diff --git a/tests/control_plane/test_execution_loss_closeout_cli.py b/tests/control_plane/test_execution_loss_closeout_cli.py index 05f6171f20..a198f75993 100644 --- a/tests/control_plane/test_execution_loss_closeout_cli.py +++ b/tests/control_plane/test_execution_loss_closeout_cli.py @@ -635,6 +635,187 @@ def test_administrative_subtype_without_complete_proof_cannot_settle( _assert_no_completion_or_debit(context) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("source_profile", [False, True]) +def test_replacement_preparation_cannot_erase_original_or_require_third_generation( + tmp_path, monkeypatch, capsys, provider, source_profile, +): + from loopx.control_plane.quota.heartbeat_receipt import ensure_turn_heartbeat_settlement_receipt + from loopx.control_plane.quota.error_codes import HeartbeatReceiptIdentityConflictError + + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=source_profile) + original = _readback(context) + original_prepared = deepcopy(original.prepared_lease) + assert original_prepared is not None + before = _unchanged_state(context) + before_bind_calls = [] + current = context["current_lease"] + with pytest.raises(HeartbeatReceiptIdentityConflictError): + ensure_turn_heartbeat_settlement_receipt( + context["runtime"], original.identity.value, + semantic_replan_guard_scoped=False, semantic_replan_obligation_id=None, + task_lease_proof={"idempotency_key": current["idempotency_key"], + "expected_version": current["version"]}, + task_lease_epoch=current["lease_epoch"], goal_ref=context["goal_ref"], + before_bind=lambda: before_bind_calls.append(True), + ) + assert before_bind_calls == [] + assert _unchanged_state(context) == before + assert _readback(context).prepared_lease == original_prepared + code, closed = context["cli"](*_closeout_args(context)) + assert code == 0 and closed["settlement_progress"]["no_spend"] is True, closed + run = json.loads(Path(closed["json_path"]).read_text()) + assert run["execution_loss_closeout"]["original_prepared"] == original_prepared + assert _lease(context) == current + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_same_generation_renewal_is_not_a_mutation_of_original_prepared_receipt( + tmp_path, monkeypatch, capsys, provider, +): + from loopx.control_plane.quota.heartbeat_receipt import ensure_turn_heartbeat_settlement_receipt + from loopx.control_plane.quota.error_codes import HeartbeatReceiptIdentityConflictError + + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False, replace=False) + original = _readback(context) + lease = context["original_lease"] + renewed = task_lease.renew_task_lease( + registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=context["goal_id"], todo_id=context["todo"], owner=OWNER, + idempotency_key=lease["idempotency_key"], expected_version=lease["version"], + ) + assert renewed["ok"] is True, renewed + current = _lease(context) + assert current["idempotency_key"] == lease["idempotency_key"] + assert current["lease_epoch"] == lease["lease_epoch"] + assert current["version"] > lease["version"] + before = _unchanged_state(context) + with pytest.raises(HeartbeatReceiptIdentityConflictError): + ensure_turn_heartbeat_settlement_receipt( + context["runtime"], original.identity.value, + semantic_replan_guard_scoped=False, semantic_replan_obligation_id=None, + task_lease_proof={"idempotency_key": current["idempotency_key"], + "expected_version": current["version"]}, + task_lease_epoch=current["lease_epoch"], + ) + assert _unchanged_state(context) == before + replayed = ensure_turn_heartbeat_settlement_receipt( + context["runtime"], original.identity.value, + semantic_replan_guard_scoped=False, semantic_replan_obligation_id=None, + task_lease_proof={"idempotency_key": lease["idempotency_key"], + "expected_version": lease["version"]}, + task_lease_epoch=lease["lease_epoch"], + ) + assert replayed["details"]["prepared_lease_version"] == lease["version"] + assert _unchanged_state(context) == before + assert _readback(context).prepared_lease == original.prepared_lease + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_direct_publication_uses_causal_original_after_readiness_revision( + tmp_path, monkeypatch, capsys, provider, +): + context = _fixture(tmp_path, monkeypatch, capsys, provider, source_profile=False, replace=False) + original = _readback(context) + lease = context["original_lease"] + renewed = task_lease.renew_task_lease( + registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=context["goal_id"], todo_id=context["todo"], owner=OWNER, + idempotency_key=lease["idempotency_key"], expected_version=lease["version"], + ) + assert renewed["ok"] is True, renewed + current = _lease(context) + code, guarded = context["cli"](*_guard_args(context)) + assert code == 0 and guarded["normal_delivery_allowed"] is True, guarded + effective = _readback(context) + assert effective.heartbeat_receipt["details"]["settlement_receipt_revision"] == "lease_readiness" + assert effective.heartbeat_receipt["details"]["prepared_lease_version"] == current["version"] + assert effective.original_prepared_receipt == original.original_prepared_receipt + assert effective.prepared_lease == original.prepared_lease + before = _unchanged_state(context) + before_bind_calls = [] + with pytest.raises(HeartbeatReceiptIdentityConflictError): + ensure_turn_heartbeat_settlement_receipt( + context["runtime"], original.identity.value, + semantic_replan_guard_scoped=False, semantic_replan_obligation_id=None, + task_lease_proof={"idempotency_key": current["idempotency_key"], + "expected_version": current["version"]}, + task_lease_epoch=current["lease_epoch"], + before_bind=lambda: before_bind_calls.append(True), + ) + assert before_bind_calls == [] + for proof in (None, {"idempotency_key": lease["idempotency_key"], + "expected_version": lease["version"]}): + replayed = ensure_turn_heartbeat_settlement_receipt( + context["runtime"], original.identity.value, + semantic_replan_guard_scoped=False, semantic_replan_obligation_id=None, + task_lease_proof=proof, task_lease_epoch=lease["lease_epoch"], + ) + assert replayed == effective.heartbeat_receipt + assert _unchanged_state(context) == before + assert _readback(context).original_prepared_receipt == original.original_prepared_receipt + _assert_no_completion_or_debit(context) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_source_resume_qualification_preserves_enclosing_admission_on_success_and_refusal( + tmp_path, monkeypatch, capsys, provider, +): + from loopx.control_plane.quota.accounting_admission import quota_accounting_admission + from loopx.control_plane.quota.execution_preparation import ( + RefreshTaskLeaseUnavailable, guard_first_writeback_execution, + read_turn_execution_lease_qualification, + ) + from loopx.file_lock import cross_runtime_lock_witness + + context = _fixture(tmp_path, monkeypatch, capsys, provider, replace=False) + original = _readback(context) + lease = context["original_lease"] + proof = {"idempotency_key": lease["idempotency_key"], "expected_version": lease["version"]} + before = _unchanged_state(context) + # This exercises the actual helper/guard under Source admission. The + # generic Source native producer remains unavailable; no mock grants it. + with quota_accounting_admission( + runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal_id"], goal_ref=context["goal_ref"], operation="resume-borrow-regression", + ) as admission: + assert admission is not None + witnesses = deepcopy(admission["locks"]) + for offered in (proof, {**proof, "expected_version": proof["expected_version"] + 1}, proof): + kwargs = dict(runtime_root=context["runtime"], registry_path=context["registry"], + identity=original.identity.value, requested_proof=offered, + goal_ref=context["goal_ref"], source_admission=admission) + if offered == proof: + qualified = read_turn_execution_lease_qualification(**kwargs) + assert qualified.resume_lease_qualification["source"] == "active" + else: + with pytest.raises(RefreshTaskLeaseUnavailable): + read_turn_execution_lease_qualification(**kwargs) + for witness in witnesses: + assert cross_runtime_lock_witness(Path(witness["target"])) == { + key: value for key, value in witness.items() if key != "role" + } + # Use the actual locked source fact rather than a generic registry read. + locked_goal = next(goal for goal in load_project_registry(context["registry"])["goals"] + if goal["id"] == context["goal_id"]) + with guard_first_writeback_execution( + runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal_id"], todo_id=context["todo"], agent_id=OWNER, + requested_proof=proof, settlement=original, locked_goal=locked_goal, + goal_ref=context["goal_ref"], source_admission=admission, + ): + qualified = read_turn_execution_lease_qualification(**{**kwargs, "requested_proof": proof}) + assert qualified.resume_lease_qualification["state"] == "ready" + for witness in witnesses: + assert cross_runtime_lock_witness(Path(witness["target"])) == { + key: value for key, value in witness.items() if key != "role" + } + assert _unchanged_state(context) == before + _assert_no_completion_or_debit(context) + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_current_original_generation_keeps_ordinary_blocked_closeout_semantics( tmp_path, monkeypatch, capsys, provider, diff --git a/tests/control_plane/test_heartbeat_receipt.py b/tests/control_plane/test_heartbeat_receipt.py index 5a7e6f04ba..377237e6a7 100644 --- a/tests/control_plane/test_heartbeat_receipt.py +++ b/tests/control_plane/test_heartbeat_receipt.py @@ -374,6 +374,7 @@ def rows(): # Both routes still owe the same result, writeback and exactly one debit. (project / "validated-result.txt").write_text("Synthetic result validated before closeout.\n") plan = reentry["interaction_contract"]["cli_channel"]["settlement_plan"] + assert plan["schema_version"] == "quota_settlement_plan_v2" command = next(s["command_template"] for s in plan["ordered_steps"] if s["kind"] == "durable_writeback") command = command.replace("", "admission_timer_validated").replace( "", "single_surface").replace("", "outcome_progress") @@ -382,14 +383,16 @@ def rows(): " --no-global-sync --suppress-external-sinks") rc, written = _run_generated_cli(command, registry_path=registry, cwd=project) assert rc == 0, written - spend = written["settlement_owed"]["command"] - for attempt in range(2): - rc, settled = _run_generated_cli(spend, registry_path=registry, cwd=project) + assert written["settlement_progress"]["state"] == "settled" + assert written["settlement_result"]["ok"] is True + assert "settlement_owed" not in written + for _ in range(2): + rc, settled = _run_generated_cli(command, registry_path=registry, cwd=project) assert rc == 0, settled + assert settled["idempotent_replay"] is True + assert settled["appended"] is False assert settled["settlement_result"]["ok"] is True - if attempt: - assert settled["idempotent_replay"] is True - assert settled["appended"] is False + assert settled["settlement_result"] == written["settlement_result"] assert _spend_run_count(runtime) == 1 rc, replay = _run_cli(registry, runtime, *guard) assert rc == 0, replay diff --git a/tests/control_plane_ts/heartbeat_resume_lease.test.ts b/tests/control_plane_ts/heartbeat_resume_lease.test.ts new file mode 100644 index 0000000000..646f91c413 --- /dev/null +++ b/tests/control_plane_ts/heartbeat_resume_lease.test.ts @@ -0,0 +1,545 @@ +import assert from "node:assert/strict"; +import {createHash} from "node:crypto"; +import {mkdir, mkdtemp, readFile, rm, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import test from "node:test"; +import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; +import {settlementIdentity, settlementIdentityPayload} from "../../loopx/control_plane/effect_program.ts"; +import {canonicalAuthoritySha256} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; +import {FileAuthorityStore} from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import {createEffectRuntimeHandlers, dispatchEffectRuntimeMethod} from "../../loopx/control_plane/effect_runtime_handlers.ts"; +import { + HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + qualifyHeartbeatPreparedPublication, + qualifyHeartbeatResumeLease, + heartbeatPreparedWorkAdmission, + heartbeatValidationEvent, +} from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; +import {EXECUTION_LOSS_QUALIFICATION_SCHEMA, qualifyReceiptBoundExecutionLossCloseout} + from "../../loopx/control_plane/quota/execution_loss_closeout.ts"; +import { + QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, + readQuotaSettlement, + readQuotaSettlementFromSnapshot, + readQuotaSettlementForAdmittedOwnerFromSnapshot, + readQuotaSettlementSnapshot, +} from "../../loopx/control_plane/quota/settlement_readback.ts"; +import {projection, todo} from "./shadow_file_fixture.ts"; + +const identity = settlementIdentity({goal_id: "goal-a", agent_id: "agent-a", + todo_id: "todo_resume", turn_instance_id: "turn-original"}); +const originalKey = "original-execution"; +const originalPrepared = {prepared_lease_key_sha256: createHash("sha256").update(originalKey).digest("hex"), + prepared_lease_version: 119, prepared_lease_epoch: 50}; +const preparedReceipt: JsonObject = {schema_version: "loopx_rollout_event_v0", + event_kind: "quota_should_run", event_id: "positive-prepared", goal_id: identity.goal_id, + agent_id: identity.agent_id, run_id: identity.turn_instance_id, status: "normal_run", + recorded_at: "2026-01-01T00:00:00Z", details: {todo_id: identity.todo_id, + selected_todo_id: identity.todo_id, settlement_effect_id: identity.effect_id, + closeout_required: true, prepared_lease_required: true, ok: true, should_run: true, + must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false, ...originalPrepared}}; +const currentLease: JsonObject = {schema_version: "task_lease_v0", todo_id: identity.todo_id, + goal_id: identity.goal_id, owner: identity.agent_id, status: "active", idempotency_key: originalKey, + version: 120, lease_epoch: 50, acquired_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:01Z", expires_at: "2099-01-01T00:00:00Z"}; +const completed: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "todo_complete", + event_id: "completed-original", goal_id: identity.goal_id, agent_id: identity.agent_id, + todo_id: identity.todo_id, run_id: identity.turn_instance_id, + details: {settlement_effect_id: identity.effect_id}}; + +function resume(overrides: Partial[0]> = {}) { + return qualifyHeartbeatResumeLease({originalReceipt: preparedReceipt, identity, + requestedProof: {idempotency_key: originalKey, expected_version: 120}, + todo: {...todo(identity.todo_id!), claimed_by: identity.agent_id}, lease: currentLease, + handoffMode: "hard_lease", completionEvent: null, replayPhase: "open", + closeoutStarted: false, writebackPending: true, otherTerminalStarted: false, + exactSource: true, observedAt: new Date("2026-01-01T00:01:00Z"), ...overrides}); +} + +test("resume distinguishes current CAS from frozen prepared generation", () => { + const before = structuredClone({preparedReceipt, currentLease}); + const renewed = resume(); + assert.deepEqual(renewed, {state: "ready", original_prepared: originalPrepared, + current_task_lease_proof: {idempotency_key: originalKey, expected_version: 120}, + lease_epoch: 50, source: "active"}); + const exact = resume({lease: {...currentLease, version: 119}, + requestedProof: {idempotency_key: originalKey, expected_version: 119}}); + assert.equal(exact.state, "ready"); + assert.deepEqual(exact.original_prepared, originalPrepared); + assert.deepEqual({preparedReceipt, currentLease}, before); +}); + +test("generation and active authority refusal matrix", () => { + const denied: [string, Partial[0]>][] = [ + ["stale supplied CAS", {requestedProof: {idempotency_key: originalKey, expected_version: 119}}], + ["replacement P2", {lease: {...currentLease, idempotency_key: "replacement", lease_epoch: 51, version: 121}, + requestedProof: {idempotency_key: "replacement", expected_version: 121}}], + ["same key new epoch", {lease: {...currentLease, lease_epoch: 51}}], + ["older version", {lease: {...currentLease, version: 118}, + requestedProof: {idempotency_key: originalKey, expected_version: 118}}], + ["missing current epoch", {lease: {...currentLease, lease_epoch: undefined}}], + ["active null candidate", {requestedProof: null}], + ["expired current lease", {lease: {...currentLease, expires_at: "2026-01-01T00:00:30Z"}}], + ["foreign current holder", {lease: {...currentLease, owner: "agent-b"}}], + ["foreign current claim", {todo: {...todo(identity.todo_id!), claimed_by: "agent-b"}}], + ["closeout already started", {closeoutStarted: true}], + ["closed original Turn", {replayPhase: "settled"}], + ]; + for (const [name, input] of denied) { + assert.throws(() => resume(input), {code: "refresh_task_lease_unavailable"}, name); + } +}); + +test("retired proof qualifies only remaining original completion writeback", () => { + const retired = {todo: {...todo(identity.todo_id!, "done"), claimed_by: null}, + lease: {...currentLease, status: "released"}, completionEvent: completed, + requestedProof: null, closeoutStarted: true}; + assert.deepEqual(resume(retired), {state: "ready", original_prepared: originalPrepared, + current_task_lease_proof: {idempotency_key: originalKey, expected_version: 120}, + lease_epoch: 50, source: "retired"}); + assert.equal(resume({...retired, + requestedProof: {idempotency_key: originalKey, expected_version: 120}}).source, "retired"); + for (const input of [ + {...retired, completionEvent: null}, + {...retired, completionEvent: {...completed, run_id: "another-turn"}}, + {...retired, completionEvent: {...completed, todo_id: "todo_foreign"}}, + {...retired, completionEvent: {...completed, goal_id: "another-goal"}}, + {...retired, completionEvent: {...completed, agent_id: "another-agent"}}, + {...retired, completionEvent: {...completed, details: {settlement_effect_id: "another-effect"}}}, + {...retired, completionEvent: {...completed, details: {...completed.details as JsonObject, todo_id: "todo_foreign"}}}, + {...retired, todo: {...todo(identity.todo_id!, "deferred"), claimed_by: null}}, + {...retired, lease: {...retired.lease, status: "active"}}, + {...retired, writebackPending: false}, + {...retired, otherTerminalStarted: true}, + {...retired, lease: {...retired.lease, lease_epoch: 51}}, + ]) assert.throws(() => resume(input), {code: "refresh_task_lease_unavailable"}); +}); + +test("qualified terminal provider cannot reopen work or execute a replacement generation", () => { + const terminalProviderEffect = {step_kind: "terminal_closeout", action: "resolve_prepared", + effect_ref: `${identity.effect_id}#terminal_closeout`, + completed_phases: ["host_execute", "typed_result", "validation", "durable_writeback", "quota_spend"]} as const; + const pending = {closeoutStarted: true, otherTerminalStarted: true, writebackPending: false, + terminalProviderEffect}; + assert.equal(resume(pending).source, "active"); + const retired = {...pending, todo: {...todo(identity.todo_id!, "done"), claimed_by: null}, + lease: {...currentLease, status: "released"}, requestedProof: null, completionEvent: null, + terminalCompletion: {kind: "committed", completion: {todo_id: identity.todo_id, continuation: "no_followup"}}}; + assert.equal(resume(retired).source, "retired"); + for (const changed of [ + {...pending, requestedProof: null}, + {...pending, terminalProviderEffect: {...terminalProviderEffect, effect_ref: `${identity.effect_id}#durable_writeback`}}, + {...pending, lease: {...currentLease, idempotency_key: "replacement", lease_epoch: 51, version: 121}, + requestedProof: {idempotency_key: "replacement", expected_version: 121}}, + {...retired, terminalCompletion: null}, + {...retired, terminalCompletion: {kind: "absent"}}, + {...retired, terminalCompletion: {kind: "committed", completion: {todo_id: "todo_other", continuation: "no_followup"}}}, + {...retired, terminalProviderEffect: {...terminalProviderEffect, action: "prepare_and_execute" as const}}, + {...retired, lease: {...retired.lease, lease_epoch: 51}}, + ]) assert.throws(() => resume(changed), {code: "refresh_task_lease_unavailable"}); +}); + +test("legacy missing epoch preserves exact history but cannot renew or rebind", () => { + const legacyPrepared = {...originalPrepared}; + delete (legacyPrepared as Partial).prepared_lease_epoch; + const legacyReceipt = {...preparedReceipt, details: {...preparedReceipt.details as JsonObject, + prepared_lease_epoch: undefined}}; + const exact = {originalReceipt: legacyReceipt, exactSource: false, + lease: {...currentLease, version: 119}, + requestedProof: {idempotency_key: originalKey, expected_version: 119}}; + assert.deepEqual(resume(exact).original_prepared, legacyPrepared); + assert.equal(resume(exact).lease_epoch, null, "An observed epoch is not a proved legacy generation"); + const legacyRetired = resume({...exact, todo: {...todo(identity.todo_id!, "done"), claimed_by: null}, + lease: {...exact.lease, status: "released"}, requestedProof: null, + completionEvent: completed, closeoutStarted: true}); + assert.equal(legacyRetired.source, "retired"); + assert.equal(legacyRetired.lease_epoch, null); + assert.throws(() => resume({...exact, exactSource: true}), {code: "refresh_task_lease_unavailable"}); + assert.throws(() => resume({originalReceipt: legacyReceipt, exactSource: false}), + {code: "refresh_task_lease_unavailable"}); + assert.throws(() => resume({...exact, todo: {...todo(identity.todo_id!, "done"), claimed_by: null}, + lease: {...currentLease, status: "released"}, requestedProof: null, + completionEvent: completed, closeoutStarted: true}), {code: "refresh_task_lease_unavailable"}); +}); + +test("ordinary legacy prepared binding is explicit alias compatibility, never administrative admission", async () => { + const legacy: JsonObject = {...preparedReceipt, status: "turn_run_once", details: { + todo_id: identity.todo_id, settlement_effect_id: identity.effect_id, + source: "loopx_turn_run_once", prepared_lease_required: true, ...originalPrepared}}; + const before = structuredClone(legacy); + assert.equal(heartbeatPreparedWorkAdmission(legacy, identity), null); + assert.deepEqual(heartbeatPreparedWorkAdmission(legacy, identity, true), originalPrepared); + assert.equal(resume({originalReceipt: legacy, exactSource: false}).source, "active"); + assert.equal(resume({originalReceipt: legacy, exactSource: false, + todo: {...todo(identity.todo_id!, "done"), claimed_by: null}, + lease: {...currentLease, status: "released"}, completionEvent: completed, + requestedProof: null, closeoutStarted: true}).source, "retired"); + assert.throws(() => resume({originalReceipt: legacy}), {code: "refresh_task_lease_unavailable"}); + assert.throws(() => qualifyReceiptBoundExecutionLossCloseout({ + schema_version: EXECUTION_LOSS_QUALIFICATION_SCHEMA, + identity: settlementIdentityPayload(identity), receipt: legacy, + }), {code: "refresh_task_lease_unavailable"}); + for (const changed of [ + {...legacy, goal_ref: null}, + {...legacy, status: "normal_run"}, + {...legacy, status: "turn_run_once_extra"}, + ...["must_attempt_work", "delivery_allowed", "quiet_noop_allowed"].map(field => + ({...legacy, details: {...legacy.details as JsonObject, [field]: true}})), + ...["ok", "should_run"].map(field => + ({...legacy, details: {...legacy.details as JsonObject, [field]: false}})), + {...legacy, details: {...legacy.details as JsonObject, closeout_required: false}}, + {...legacy, details: {...legacy.details as JsonObject, prepared_lease_required: false}}, + {...legacy, details: {...legacy.details as JsonObject, prepared_lease_required: undefined}}, + {...legacy, details: {todo_id: identity.todo_id, settlement_effect_id: identity.effect_id, + prepared_lease_required: true}}, + ]) assert.equal(heartbeatPreparedWorkAdmission(changed, identity, true), null); + assert.throws(() => heartbeatPreparedWorkAdmission({...legacy, + details: {...legacy.details as JsonObject, prepared_lease_version: undefined}}, identity, true), + {code: "malformed_settlement_state"}); + const readiness = {...legacy, event_id: "ordinary-readiness", details: {...legacy.details as JsonObject, + prepared_lease_version: 120, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: legacy.event_id, caused_by: legacy.event_id}}; + assert.deepEqual(heartbeatValidationEvent([legacy, readiness], readiness, identity, true), legacy); + const publish = {schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: readiness, original_prepared: originalPrepared, receipt_history: [legacy, readiness], + settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields: {prepared_lease_required: true, ...originalPrepared}}; + assert.equal((qualifyHeartbeatPreparedPublication(publish).prepared_fields as JsonObject).prepared_lease_version, 120); + assert.throws(() => qualifyHeartbeatPreparedPublication({...publish, + proposed_prepared_fields: {...publish.proposed_prepared_fields, prepared_lease_version: 120}}), + {code: "heartbeat_receipt_identity_conflict"}); + const goalRef = {goal_id: identity.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + assert.throws(() => qualifyHeartbeatPreparedPublication({...publish, + existing_receipt: {...readiness, goal_ref: goalRef}, + receipt_history: [{...legacy, goal_ref: goalRef}, {...readiness, goal_ref: goalRef}]}), + {code: "heartbeat_receipt_identity_conflict"}); + const f = await receiptFixture([legacy, readiness]); + try { + const result = await readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}}}); + assert.deepEqual(result.original_prepared_receipt, legacy); + assert.deepEqual(result.prepared_lease, originalPrepared); + assert.equal((result.resume_lease_qualification as JsonObject).source, "active"); + assert.equal(result.original_prepared_receipt_sha256, canonicalAuthoritySha256(legacy)); + } finally {await rm(f.root, {recursive: true, force: true});} + assert.deepEqual(legacy, before); +}); + +test("publication replay is exact even when ephemeral same-generation resume is ready", async () => { + const fields = {prepared_lease_required: true, ...originalPrepared}; + const publish = (proposal: JsonObject | null, existing = preparedReceipt, original: JsonObject | null = originalPrepared) => + qualifyHeartbeatPreparedPublication({schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: existing, original_prepared: original, proposed_prepared_fields: proposal}); + assert.deepEqual(publish(fields), {ok: true, prepared_fields: fields}); + assert.deepEqual(publish(null), {ok: true, prepared_fields: fields}); + assert.equal(resume().state, "ready"); + for (const proposal of [{...fields, prepared_lease_version: 120}, + {...fields, prepared_lease_key_sha256: "b".repeat(64)}, {...fields, prepared_lease_epoch: 51}, + {prepared_lease_required: true}, {...fields, prepared_lease_epoch: undefined}, + {...fields, prepared_lease_required: false}]) { + assert.throws(() => publish(proposal), {code: "heartbeat_receipt_identity_conflict"}); + } + const changedStorage = {...preparedReceipt, details: {...fields, prepared_lease_version: 120}}; + assert.equal((publish(null, changedStorage).prepared_fields as JsonObject).prepared_lease_version, 120); + assert.throws(() => publish(fields, changedStorage), {code: "heartbeat_receipt_identity_conflict"}); + const legacyFields: JsonObject = {...fields}; + delete legacyFields.prepared_lease_epoch; + const legacyOriginal: JsonObject = {...originalPrepared}; + delete legacyOriginal.prepared_lease_epoch; + const legacyStored = {...preparedReceipt, details: legacyFields}; + assert.deepEqual(publish(legacyFields, legacyStored, legacyOriginal), + {ok: true, prepared_fields: legacyFields}); + assert.throws(() => publish(fields, legacyStored, legacyOriginal), + {code: "heartbeat_receipt_identity_conflict"}); + const handlers = createEffectRuntimeHandlers({fingerprint: "prepared-publication", requestShutdown() {}}); + assert.deepEqual(await dispatchEffectRuntimeMethod(handlers, "quota.heartbeat_prepared_publication.qualify", { + schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, existing_receipt: null, + original_prepared: null, proposed_prepared_fields: fields}), {ok: true, prepared_fields: fields}); + for (const proposed of [{}, {prepared_lease_required: true}]) { + await assert.rejects(dispatchEffectRuntimeMethod(handlers, "quota.heartbeat_prepared_publication.qualify", { + schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, existing_receipt: null, + original_prepared: null, proposed_prepared_fields: proposed}), {code: "heartbeat_receipt_identity_conflict"}); + } +}); + +test("publication computes the original proof from history and preserves later stored readiness", () => { + const stored = {...preparedReceipt, event_id: "stored-readiness", details: { + ...preparedReceipt.details as JsonObject, prepared_lease_version: 120, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: preparedReceipt.event_id, caused_by: preparedReceipt.event_id}}; + const fields = {prepared_lease_required: true, ...originalPrepared}; + const storedFields = {...fields, prepared_lease_version: 120}; + const input = {schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: structuredClone(stored), original_prepared: null, + receipt_history: [structuredClone(preparedReceipt), structuredClone(stored)], + settlement_identity: settlementIdentityPayload(identity), proposed_prepared_fields: fields}; + const before = structuredClone(input); + assert.deepEqual(qualifyHeartbeatPreparedPublication(input), {ok: true, prepared_fields: storedFields}); + assert.deepEqual(qualifyHeartbeatPreparedPublication({...input, proposed_prepared_fields: null}), + {ok: true, prepared_fields: storedFields}); + assert.deepEqual(input, before); + for (const changed of [ + {...input, proposed_prepared_fields: storedFields}, + {...input, original_prepared: {...originalPrepared, prepared_lease_version: 120}, proposed_prepared_fields: storedFields}, + {...input, existing_receipt: {...stored, details: {...stored.details, prepared_lease_version: 121}}}, + {...input, receipt_history: [preparedReceipt, preparedReceipt, stored]}, + {...input, receipt_history: [{...preparedReceipt, run_id: "other-turn"}, stored]}, + {...input, receipt_history: [{...preparedReceipt, goal_ref: {goal_id: identity.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}}, stored]}, + ]) assert.throws(() => qualifyHeartbeatPreparedPublication(changed), {code: "heartbeat_receipt_identity_conflict"}); +}); + +function request(root: string): JsonObject { + return {schema_version: QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, runtime_root: root, + goal_id: identity.goal_id, agent_id: identity.agent_id, todo_id: identity.todo_id, + turn_instance_id: identity.turn_instance_id, infer_turn_instance_id: false, + allow_unbound_binding: false}; +} + +async function receiptFixture(events: JsonObject[], handoffMode = "hard_lease", seedLease = currentLease) { + const root = await mkdtemp(join(tmpdir(), "prepared-source-")); + const log = join(root, "goals", identity.goal_id, "rollout-event-log.jsonl"); + await mkdir(join(root, "goals", identity.goal_id), {recursive: true}); + await writeFile(log, events.map(row => JSON.stringify(row)).join("\n") + "\n"); + const store = new FileAuthorityStore(join(root, "authority", "file-v0"), identity.goal_id); + const committed = await store.commitAuthority({operation_id: "seed-prepared-source", + expected_provider_revision: null, next_projection: projection([ + {...todo(identity.todo_id!), claimed_by: identity.agent_id}], [seedLease], handoffMode), + events: [], receipts: []}); + assert.equal(committed.status, "applied", JSON.stringify(committed)); + return {root, log, store}; +} + +test("historical producer replacement conflicts before resume, publication and admitted administrative read", async () => { + const replacementKey = "replacement-execution"; + const replacementPrepared = {prepared_lease_key_sha256: createHash("sha256").update(replacementKey).digest("hex"), + prepared_lease_version: 121, prepared_lease_epoch: 51}; + const p2Lease = {...currentLease, idempotency_key: replacementKey, version: 121, lease_epoch: 51}; + const handlers = createEffectRuntimeHandlers({fingerprint: "historical-generation", requestShutdown() {}}); + for (const causal of [true, false]) { + const replacement: JsonObject = {...preparedReceipt, event_id: "old-producer-rebind", status: "turn_run_once", + details: {...preparedReceipt.details as JsonObject, ...replacementPrepared}, + ...(causal ? {causality: {source_event_id: preparedReceipt.event_id, caused_by: preparedReceipt.event_id}} : {})}; + assert.equal(Object.hasOwn(replacement.details as JsonObject, "settlement_receipt_revision"), false); + const history = [preparedReceipt, replacement]; + const f = await receiptFixture(history, "hard_lease", p2Lease); + try { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const snapshot = await readQuotaSettlementSnapshot(f.root, identity.goal_id); + assert.throws(() => readQuotaSettlementForAdmittedOwnerFromSnapshot(request(f.root), {kind: "alias"}, snapshot), + {code: "heartbeat_receipt_identity_conflict"}); + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: replacementKey, expected_version: 121}}}), + {code: "heartbeat_receipt_identity_conflict"}); + await assert.rejects(dispatchEffectRuntimeMethod(handlers, "quota.heartbeat_prepared_publication.qualify", { + schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: replacement, original_prepared: null, receipt_history: history, + settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields: {prepared_lease_required: true, ...replacementPrepared}}), + {code: "heartbeat_receipt_identity_conflict"}); + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + } + const epochless = {...preparedReceipt, details: {...preparedReceipt.details as JsonObject}}; + delete (epochless.details as JsonObject).prepared_lease_epoch; + for (const [first, changed] of [ + [preparedReceipt, {...preparedReceipt, details: {...preparedReceipt.details as JsonObject, prepared_lease_epoch: 51}}], + [epochless, {...epochless, details: {...epochless.details, prepared_lease_version: 120}}], + [epochless, preparedReceipt], + ]) { + const last = {...changed, event_id: "changed-generation"}; + assert.throws(() => heartbeatValidationEvent([first!, last], last, identity), + {code: "heartbeat_receipt_identity_conflict"}); + } +}); + +test("old producer causal renewal without revision retains original version, event and hash", async () => { + const renewed = {...preparedReceipt, event_id: "old-producer-renewed", status: "turn_run_once", + details: {...preparedReceipt.details as JsonObject, prepared_lease_version: 120}, + causality: {source_event_id: preparedReceipt.event_id, caused_by: preparedReceipt.event_id}}; + assert.deepEqual(heartbeatValidationEvent([preparedReceipt, renewed], renewed, identity), preparedReceipt); + const f = await receiptFixture([preparedReceipt, renewed]); + try { + const result = await readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}}}); + assert.deepEqual(result.original_prepared_receipt, preparedReceipt); + assert.deepEqual(result.prepared_lease, originalPrepared); + assert.equal(result.original_prepared_event_id, preparedReceipt.event_id); + assert.equal(result.original_prepared_receipt_sha256, canonicalAuthoritySha256(preparedReceipt)); + const publication = {schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: renewed, original_prepared: originalPrepared, + receipt_history: [preparedReceipt, renewed], settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields: {prepared_lease_required: true, ...originalPrepared}}; + assert.equal((qualifyHeartbeatPreparedPublication(publication).prepared_fields as JsonObject).prepared_lease_version, 120); + assert.throws(() => qualifyHeartbeatPreparedPublication({...publication, + proposed_prepared_fields: {...publication.proposed_prepared_fields, prepared_lease_version: 120}}), + {code: "heartbeat_receipt_identity_conflict"}); + for (const causality of [{source_event_id: "missing", caused_by: "missing"}, + {source_event_id: preparedReceipt.event_id, caused_by: "different-parent"}]) { + assert.equal(heartbeatValidationEvent([preparedReceipt, {...renewed, causality}], {...renewed, causality}, identity), null); + } + } finally {await rm(f.root, {recursive: true, force: true});} +}); + +test("uncaused same-generation change rejects while exact repetition retains the first event and hash", async () => { + const changed: JsonObject = {...preparedReceipt, event_id: "uncaused-renewal", status: "turn_run_once", + details: {...preparedReceipt.details as JsonObject, prepared_lease_version: 120}}; + const history = [preparedReceipt, changed]; + const f = await receiptFixture(history); + const handlers = createEffectRuntimeHandlers({fingerprint: "uncaused-original", requestShutdown() {}}); + const publication = (existing: JsonObject, receipt_history: JsonObject[], proposed_prepared_fields: JsonObject | null) => ({ + schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, existing_receipt: existing, + original_prepared: null, receipt_history, settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields}); + try { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}}}), + {code: "heartbeat_receipt_identity_conflict"}); + const snapshot = await readQuotaSettlementSnapshot(f.root, identity.goal_id); + assert.throws(() => readQuotaSettlementForAdmittedOwnerFromSnapshot(request(f.root), {kind: "alias"}, snapshot), + {code: "heartbeat_receipt_identity_conflict"}); + for (const proposal of [null, {prepared_lease_required: true, ...originalPrepared}, + {prepared_lease_required: true, ...originalPrepared, prepared_lease_version: 120}]) { + await assert.rejects(dispatchEffectRuntimeMethod(handlers, "quota.heartbeat_prepared_publication.qualify", + publication(changed, history, proposal)), {code: "heartbeat_receipt_identity_conflict"}); + } + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + const repeated = {...preparedReceipt, event_id: "uncaused-exact-repeat", status: "turn_run_once"}; + assert.deepEqual(heartbeatValidationEvent([preparedReceipt], preparedReceipt, identity), preparedReceipt); + assert.deepEqual(heartbeatValidationEvent([preparedReceipt, repeated], repeated, identity), preparedReceipt); + await writeFile(f.log, [preparedReceipt, repeated].map(row => JSON.stringify(row)).join("\n") + "\n"); + const readback = await readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}}}); + assert.deepEqual(readback.original_prepared_receipt, preparedReceipt); + assert.equal(readback.original_prepared_event_id, preparedReceipt.event_id); + assert.equal(readback.original_prepared_receipt_sha256, canonicalAuthoritySha256(preparedReceipt)); + assert.deepEqual(readback.prepared_lease, originalPrepared); + assert.equal((readback.resume_lease_qualification as JsonObject).state, "ready"); + assert.deepEqual(await dispatchEffectRuntimeMethod(handlers, "quota.heartbeat_prepared_publication.qualify", + publication(repeated, [preparedReceipt, repeated], {prepared_lease_required: true, ...originalPrepared})), + {ok: true, prepared_fields: {prepared_lease_required: true, ...originalPrepared}}); + const legacy = {...preparedReceipt, status: "turn_run_once", details: {todo_id: identity.todo_id, + settlement_effect_id: identity.effect_id, prepared_lease_required: true, ...originalPrepared}}; + assert.deepEqual(heartbeatValidationEvent([legacy], legacy, identity, true), legacy); + const legacyRepeat = {...legacy, event_id: "legacy-exact-repeat"}; + assert.deepEqual(heartbeatValidationEvent([legacy, legacyRepeat], legacyRepeat, identity, true), legacy); + } finally {await rm(f.root, {recursive: true, force: true});} +}); + +test("prepared source follows causal positive admission instead of binding clock or effective proof", async () => { + const binding = {...preparedReceipt, event_id: "earlier-binding", recorded_at: "2025-01-01T00:00:00Z", + status: "blocked", details: {...preparedReceipt.details as JsonObject, closeout_required: false, + must_attempt_work: false, delivery_allowed: false, prepared_lease_version: 80}}; + const admitted = {...preparedReceipt, details: {...preparedReceipt.details as JsonObject, + settlement_receipt_revision: "work_admission"}, + causality: {source_event_id: binding.event_id, caused_by: binding.event_id}}; + const readiness = {...preparedReceipt, event_id: "later-readiness", + details: {...preparedReceipt.details as JsonObject, prepared_lease_version: 120, + settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: admitted.event_id, caused_by: admitted.event_id}}; + const effective = {...readiness, event_id: "later-identity-upgrade", + details: {...readiness.details, settlement_receipt_revision: "identity_upgrade"}, + causality: {source_event_id: readiness.event_id, caused_by: readiness.event_id}}; + const unrelated = {...preparedReceipt, event_id: "not-a-causal-ancestor", + details: {...preparedReceipt.details as JsonObject, prepared_lease_version: 60}}; + const f = await receiptFixture([unrelated, binding, admitted, readiness, effective]); + try { + const before = await readFile(f.log); + const authorityBefore = await f.store.loadAuthority(); + const result = await readQuotaSettlement({...request(f.root), include_record_bodies: false, + resume_lease_candidate: {requested_proof: {idempotency_key: originalKey, expected_version: 120}}}); + assert.deepEqual(result.prepared_lease, originalPrepared); + assert.equal(result.original_prepared_event_id, admitted.event_id); + assert.deepEqual(result.original_prepared_receipt, admitted); + assert.equal(result.original_prepared_receipt_sha256, canonicalAuthoritySha256(admitted)); + assert.equal(result.admission_event_id, unrelated.event_id); + assert.equal((result.resume_lease_qualification as JsonObject).source, "active"); + assert.deepEqual((result.resume_lease_qualification as JsonObject).original_prepared, originalPrepared); + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}, active: true}}), /only requested_proof/); + const snapshot = await readQuotaSettlementSnapshot(f.root, identity.goal_id); + assert.throws(() => readQuotaSettlementFromSnapshot({...request(f.root), + resume_lease_candidate: {requested_proof: null}}, snapshot), /live canonical authority/); + } finally {await rm(f.root, {recursive: true, force: true});} +}); + +test("pending selection can precede binding but cannot become prepared admission", async () => { + const reservation = {...preparedReceipt, event_id: "reserved", details: { + prepared_lease_required: true, ok: true, should_run: false, must_attempt_work: false, + delivery_allowed: false, quiet_noop_allowed: true, closeout_required: false}}; + const pending = {...reservation, event_id: "pending-selection", details: {...reservation.details, + pending_action_selection_todo_id: identity.todo_id, pending_action_selection_state: "deferred", + pending_action_selection_reason: "claim_required", todo_id: null, settlement_effect_id: null}, + causality: {source_event_id: reservation.event_id, caused_by: reservation.event_id}}; + const bound = {...pending, event_id: "bound-negative", details: {...pending.details, + todo_id: identity.todo_id, settlement_effect_id: identity.effect_id, settlement_receipt_revision: "identity_upgrade"}, + causality: {source_event_id: pending.event_id, caused_by: pending.event_id}}; + const admitted = {...preparedReceipt, details: {...preparedReceipt.details as JsonObject, + settlement_receipt_revision: "work_admission"}, + causality: {source_event_id: bound.event_id, caused_by: bound.event_id}}; + const readiness = {...admitted, event_id: "renewed-readiness", details: {...admitted.details, + prepared_lease_version: 120, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: admitted.event_id, caused_by: admitted.event_id}}; + const f = await receiptFixture([reservation, pending, bound]); + try { + const negative = await readQuotaSettlement({...request(f.root), heartbeat_reentry_guard: preparedReceipt.details}); + assert.ok((negative.identity as any).result.value); + assert.equal((negative.heartbeat_reentry_qualification as JsonObject).append, true); + assert.equal(negative.original_prepared_receipt, null); + const history = [reservation, pending, bound, admitted, readiness]; + await writeFile(f.log, history.map(row => JSON.stringify(row)).join("\n") + "\n"); + const ready = await readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}}}); + assert.deepEqual(ready.prepared_lease, originalPrepared); + assert.equal(ready.original_prepared_event_id, admitted.event_id); + assert.equal((ready.resume_lease_qualification as JsonObject).source, "active"); + for (const badParent of [ + {...pending, details: {...pending.details, pending_action_selection_todo_id: "todo_other"}}, + {...pending, run_id: "another-turn"}, + {...pending, goal_id: "another-goal"}, + {...pending, goal_ref: {goal_id: identity.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}}, + ]) assert.equal(heartbeatValidationEvent([reservation, badParent, bound, admitted, readiness], readiness, identity), null); + assert.equal(heartbeatValidationEvent([reservation, pending, bound, admitted, {...readiness, + causality: {source_event_id: admitted.event_id, caused_by: "different-parent"}}], {...readiness, + causality: {source_event_id: admitted.event_id, caused_by: "different-parent"}}, identity), null); + } finally {await rm(f.root, {recursive: true, force: true});} +}); + +test("absence of receipt preparedness cannot downgrade real hard lease authority", async () => { + const noPrepared = {...preparedReceipt, details: {todo_id: identity.todo_id, + settlement_effect_id: identity.effect_id, closeout_required: true, ok: true, + should_run: true, must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false}}; + const hard = await receiptFixture([noPrepared]); + const soft = await receiptFixture([noPrepared], "soft_claim"); + try { + const candidate = {requested_proof: {idempotency_key: originalKey, expected_version: 120}}; + await assert.rejects(readQuotaSettlement({...request(hard.root), resume_lease_candidate: candidate}), + {code: "refresh_task_lease_unavailable"}); + const result = await readQuotaSettlement({...request(soft.root), resume_lease_candidate: candidate}); + assert.deepEqual(result.resume_lease_qualification, {state: "not_required", original_prepared: null, + current_task_lease_proof: null, lease_epoch: null, source: null}); + } finally {await rm(hard.root, {recursive: true, force: true}); await rm(soft.root, {recursive: true, force: true});} +}); + +test("legacy identity upgrade retains prepared source that predated the effect id", async () => { + const original = {...preparedReceipt, details: {...preparedReceipt.details as JsonObject}}; + delete original.details.settlement_effect_id; + const upgraded = {...preparedReceipt, event_id: "legacy-upgraded", + details: {...preparedReceipt.details as JsonObject, settlement_receipt_revision: "identity_upgrade"}, + causality: {source_event_id: original.event_id, caused_by: original.event_id}}; + const f = await receiptFixture([original, upgraded]); + try { + const result = await readQuotaSettlement(request(f.root)); + assert.deepEqual(result.original_prepared_receipt, original); + assert.deepEqual(result.prepared_lease, originalPrepared); + assert.equal(result.original_prepared_receipt_sha256, canonicalAuthoritySha256(original)); + await writeFile(f.log, JSON.stringify(original) + "\n" + JSON.stringify({...upgraded, + causality: {source_event_id: "missing", caused_by: "missing"}}) + "\n"); + const broken = await readQuotaSettlement(request(f.root)); + assert.equal((broken.identity as any).result.failure.kind, "receipt_missing"); + } finally {await rm(f.root, {recursive: true, force: true});} +}); diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index c2592471e9..14f12bc442 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -12,7 +12,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { settlementIdentity, settlementIdentityPayload } from "../../loopx/control_plane/effect_program.ts"; +import { settlementIdentity, settlementIdentityPayload, type JsonObject } from "../../loopx/control_plane/effect_program.ts"; import { createEffectRuntimeHandlers, dispatchEffectRuntimeMethod, @@ -25,6 +25,9 @@ import { BLOCKED_WAIT_REQUEST_SCHEMA, prepareBlockedWait } from "../../loopx/con import { evaluateTodoResumeConditions, TODO_RESUME_EVALUATION_REQUEST_SCHEMA_VERSION } from "../../loopx/control_plane/todos/resume_condition.ts"; import { projectSemanticReplanGuard, + projectNativeTaskLeaseCleanup, + NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA, + NATIVE_TASK_LEASE_CLEANUP_SCHEMA, QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, readQuotaSettlement, readQuotaSettlementFromSnapshot, @@ -53,6 +56,118 @@ const identity = settlementIdentity({ todo_id: todoId, turn_instance_id: turnId, }); +const cleanupRequest = {schema_version: NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA, + recovery: false, admission_retry: false, resume_turn_key_present: false, original_journal_present: false}; + +test("native cleanup distinguishes proven release, owed closeout and unconfirmed receipt authority", () => { + const bound = (details: JsonObject, phase = "open"): JsonObject => ({found: true, + identity: {result: {value: settlementIdentityPayload(identity), failure: null}}, + heartbeat_receipt: {details}, replay_phase: phase}); + const failed = (kind: string): JsonObject => ({found: true, + identity: {result: {value: null, failure: {kind}}}, replay_phase: null}); + const result = (disposition: string, reason: string) => ({schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, + disposition, reason}); + assert.deepEqual(projectNativeTaskLeaseCleanup(failed("receipt_missing"), cleanupRequest), + result("release", "fresh_pre_admission_missing")); + for (const flag of ["recovery", "admission_retry", "resume_turn_key_present", "original_journal_present"]) { + assert.deepEqual(projectNativeTaskLeaseCleanup(failed("receipt_missing"), {...cleanupRequest, [flag]: true}), + result("unconfirmed", "original_identity_unconfirmed")); + } + assert.deepEqual(projectNativeTaskLeaseCleanup(failed("receipt_unbound"), {...cleanupRequest, recovery: true}), + result("release", "receipt_unbound")); + for (const details of [{settlement_receipt_revision: "turn_reservation"}, + {settlement_receipt_revision: "turn_reservation", closeout_required: false}]) { + assert.deepEqual(projectNativeTaskLeaseCleanup(bound(details), cleanupRequest), + result("release", "original_reservation_without_closeout")); + } + for (const revision of ["work_admission", "lease_readiness", "identity_upgrade"]) { + assert.deepEqual(projectNativeTaskLeaseCleanup(bound({settlement_receipt_revision: revision, + closeout_required: true}), cleanupRequest), result("retain", "original_turn_owes_closeout")); + for (const required of [undefined, false, null, "true", 1]) { + assert.equal(projectNativeTaskLeaseCleanup(bound({settlement_receipt_revision: revision, + closeout_required: required}), cleanupRequest).disposition, "unconfirmed"); + } + } + for (const details of [{closeout_required: true}, {closeout_required: false}, + {settlement_receipt_revision: "unknown", closeout_required: true}, + ...[true, null, "false", 0].map(required => ({settlement_receipt_revision: "turn_reservation", closeout_required: required}))]) { + assert.equal(projectNativeTaskLeaseCleanup(bound(details), cleanupRequest).disposition, "unconfirmed"); + } + assert.deepEqual(projectNativeTaskLeaseCleanup(bound({}, "settled"), cleanupRequest), + result("release", "original_turn_settled")); + assert.equal(projectNativeTaskLeaseCleanup({...failed("identity_mismatch"), replay_phase: "settled"}, cleanupRequest) + .disposition, "unconfirmed"); + assert.equal(projectNativeTaskLeaseCleanup({found: false}, cleanupRequest).disposition, "unconfirmed"); + assert.throws(() => projectNativeTaskLeaseCleanup(bound({}), {...cleanupRequest, original_journal_present: null})); + assert.throws(() => projectNativeTaskLeaseCleanup(bound({}), {...cleanupRequest, closeout_required: false})); +}); + +test("native cleanup attachment preserves B3 original proof and borrowed Source ownership", async () => { + for (const exact of [false, true]) { + const root = await fixture(exact ? {goalRef: {goal_id: goalId, goal_instance_id: instanceA}} : {}); + try { + const path = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const reservation = JSON.parse((await readFile(path, "utf8")).trim()); + reservation.details = {...reservation.details, closeout_required: false}; + const original = {...reservation, event_id: "cleanup-admitted", status: "normal_run", + details: {...reservation.details, settlement_receipt_revision: "work_admission", closeout_required: true, + prepared_lease_required: true, prepared_lease_key_sha256: "a".repeat(64), + prepared_lease_version: 119, prepared_lease_epoch: 50, + ok: true, should_run: true, must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false}, + causality: {source_event_id: reservation.event_id, caused_by: reservation.event_id}}; + await writeFile(path, [reservation, original].map(row => JSON.stringify(row)).join("\n") + "\n"); + const before = await readFile(path); + const inspect = async (binding: Record = {}) => { + const ordinaryRequest = request(root, {...binding, ...(exact ? {borrow_source_admission: true} : {})}); + const ordinary = await readQuotaSettlement(ordinaryRequest); + assert.equal(Object.hasOwn(ordinary, "native_task_lease_cleanup"), false); + const requested = await readQuotaSettlement({...ordinaryRequest, native_task_lease_cleanup: cleanupRequest}); + assert.deepEqual(requested.native_task_lease_cleanup, {schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, + disposition: "retain", reason: "original_turn_owes_closeout"}); + const rest = {...requested}; delete rest.native_task_lease_cleanup; + assert.deepEqual(rest, ordinary); + assert.deepEqual(requested.original_prepared_receipt, original); + assert.equal(requested.original_prepared_event_id, original.event_id); + assert.deepEqual(await readQuotaSettlement(ordinaryRequest), ordinary); + }; + if (exact) await withSourceAdmission(root, instanceA, instanceA, inspect); + else { + await inspect(); + const snapshot = await readQuotaSettlementSnapshot(root, goalId); + const projected = readQuotaSettlementFromSnapshot({...request(root), native_task_lease_cleanup: cleanupRequest}, snapshot); + assert.equal((projected.native_task_lease_cleanup as JsonObject).disposition, "retain"); + } + assert.deepEqual(await readFile(path), before); + } finally {await rm(root, {recursive: true, force: true});} + } +}); + +test("native cleanup is an explicit read and cannot mix with resume or settlement admission", async () => { + const root = await fixture({guard: false}); + try { + const snapshot = await readQuotaSettlementSnapshot(root, goalId); + const requested = readQuotaSettlementFromSnapshot({...request(root), native_task_lease_cleanup: cleanupRequest}, snapshot); + assert.deepEqual(requested.native_task_lease_cleanup, {schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, + disposition: "release", reason: "fresh_pre_admission_missing"}); + for (const incompatible of [{infer_turn_instance_id: true}, {allow_unbound_binding: true}, + {resolve_original_binding: true}, {pending_closeout: true}, {refresh_retry: {}}, + {agent_id: null}, {todo_id: null}, {turn_instance_id: null}, + {replan_obligation_id: "replan-0000000000000001"}, {heartbeat_reentry_guard: {}}, + {spend_precheck: true}, {tolerate_malformed_history: true}, + {resume_lease_candidate: {requested_proof: null}}]) { + assert.throws(() => readQuotaSettlementFromSnapshot({...request(root), ...incompatible, + native_task_lease_cleanup: cleanupRequest}, snapshot)); + } + } finally {await rm(root, {recursive: true, force: true});} + const settledRoot = await fixture({writeback: true, spend: true, completion: true}); + try { + const compact = await readQuotaSettlement({...request(settledRoot), include_record_bodies: false, + native_task_lease_cleanup: cleanupRequest}); + assert.equal(compact.heartbeat_receipt, null); + assert.deepEqual(compact.native_task_lease_cleanup, {schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, + disposition: "release", reason: "original_turn_settled"}); + } finally {await rm(settledRoot, {recursive: true, force: true});} +}); test("semantic replan guard distinguishes legacy, none, and exact selection", () => { assert.deepEqual(projectSemanticReplanGuard({}), { @@ -783,6 +898,23 @@ async function withSourceAdmission( } } +test("exact Source resume cannot become not_required from missing prepared admission", async () => { + const root = await fixture({goalRef: {goal_id: goalId, goal_instance_id: instanceA}}); + try { + const {FileAuthorityStore} = await import("../../loopx/control_plane/coordination/file_authority_store.ts"); + const {projection, todo} = await import("./shadow_file_fixture.ts"); + const head = {...projection([{...todo(todoId), claimed_by: agentId}], [], "soft_claim"), goal_id: goalId}; + const store = new FileAuthorityStore(join(root, "authority", "file-v0"), goalId); + const seeded = await store.commitAuthority({operation_id: "seed-source-no-prepared", + expected_provider_revision: null, next_projection: head, events: [], receipts: []}); + assert.equal(seeded.status, "applied", JSON.stringify(seeded)); + await withSourceAdmission(root, instanceA, instanceA, async binding => { + await assert.rejects(readQuotaSettlement(request(root, {...binding, borrow_source_admission: true, + resume_lease_candidate: {requested_proof: null}})), {code: "refresh_task_lease_unavailable"}); + }); + } finally {await rm(root, {recursive: true, force: true});} +}); + test("scoped supersede settles only its exact Turn and never terminal acceptance", async t => { const cases = [ {name: "original tuple", expected: "settled", patch: {}}, @@ -881,6 +1013,16 @@ test("settlement progress requires both effects and exact receipts", async t => const events = (await readFile(path, "utf8")).trim().split("\n").map(line => JSON.parse(line)); await writeFile(path, events.filter(event => event.event_kind !== entry.remove).map(event => JSON.stringify(event)).join("\n") + "\n"); } + if (entry.state === "settled") { + const path = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const events = (await readFile(path, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + const guard = events.find(event => event.event_kind === "quota_should_run")!; + guard.status = "normal_run"; + Object.assign(guard.details, {closeout_required: true, prepared_lease_required: true, + prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 119, prepared_lease_epoch: 50, + ok: true, should_run: true, must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false}); + await writeFile(path, events.map(event => JSON.stringify(event)).join("\n") + "\n"); + } const result = await readQuotaSettlement(request(root)); assert.deepEqual(result.progress, { schema_version: "quota_settlement_progress_v0", state: entry.state, @@ -891,7 +1033,10 @@ test("settlement progress requires both effects and exact receipts", async t => const expected = {...result}; if (entry.state === "settled") { assert.ok(result.writeback_run && result.spend_run && result.heartbeat_receipt); - for (const field of ["writeback_run", "spend_run", "heartbeat_receipt", + assert.ok(result.original_prepared_receipt); + assert.equal(controls.original_prepared_event_id, "event-guard"); + assert.equal(controls.original_prepared_receipt_sha256, result.original_prepared_receipt_sha256); + for (const field of ["writeback_run", "spend_run", "heartbeat_receipt", "original_prepared_receipt", "writeback_event", "spend_event", "completion_event"]) expected[field] = null; for (const field of ["writeback", "spend", "delivery", "settlement"]) { const original = result[field] as {result: {value: unknown}}; diff --git a/tests/control_plane_ts/turn_journal_query.test.ts b/tests/control_plane_ts/turn_journal_query.test.ts index ac2dd6bc28..a4b396c655 100644 --- a/tests/control_plane_ts/turn_journal_query.test.ts +++ b/tests/control_plane_ts/turn_journal_query.test.ts @@ -6,7 +6,8 @@ import test, { type TestContext } from "node:test"; import { EffectRuntimeConflictError, EffectRuntimeRequestError } from "../../loopx/control_plane/effect_runtime_errors.ts"; import { commitTurnJournal } from "../../loopx/control_plane/turn_driver/turn_journal_effects.ts"; -import { findTurnJournalBySettlement, readTurnJournalCapabilities } from "../../loopx/control_plane/turn_driver/turn_journal_query.ts"; +import { findTurnJournalBySettlement, readTurnJournalCapabilities, readTurnTerminalCloseoutProvider } from "../../loopx/control_plane/turn_driver/turn_journal_query.ts"; +import {settlementIdentity, type JsonObject} from "../../loopx/control_plane/effect_program.ts"; const phases = ["host_execute", "typed_result", "validation", "durable_writeback", "quota_spend", "scheduler_apply", "scheduler_ack"]; const selector = { goal_id: "goal", agent_id: "actor", todo_id: "todo_task", turn_instance_id: "turn-7" }; @@ -51,6 +52,49 @@ const capabilities = ( ...(goal_ref === undefined ? {} : { goal_ref }), }); +test("terminal provider reuses the actual journal and settlement owners without altering their files", async t => { + const {root, path, directory} = await fixture(t); + const expected = settlementIdentity(selector); + const terminalRef = `${identity.effect_id}#terminal_closeout`; + const canonicalTodo = {todo_id: selector.todo_id, status: "open", no_followup: true, + completion_continuation: null, successor_todo_ids: []}; + const pending = (): JsonObject => ({...journal(), status: "in_progress", completed_phases: phases.slice(0, 5), + host_result: {turn_key: key, result_kind: "validated_completion"}, + writeback: {ok: true, appended: true, effect_ref: `${identity.effect_id}#durable_writeback`}, + quota_spend: {ok: true, appended: true, effect_ref: `${identity.effect_id}#quota_spend`}, + effect_attempts: {terminal_closeout: {status: "prepared", effect_ref: terminalRef}}}); + const read = (effect_ref: string | null = null, turn_key = key) => readTurnTerminalCloseoutProvider( + root, expected, {kind: "alias"}, {turn_key, effect_ref}, canonicalTodo); + await writeFile(path, JSON.stringify(pending())); + const before = await readFile(path), files = await readdir(directory); + const terminal = await read(); + assert.equal(terminal?.step_kind, "terminal_closeout"); + assert.equal(terminal?.action, "resolve_prepared"); + assert.equal(terminal?.effect_ref, terminalRef); + assert.deepEqual(await read(terminalRef), terminal); + await assert.rejects(read(`${identity.effect_id}#durable_writeback`), {code: "refresh_task_lease_unavailable"}); + await assert.rejects(read(null, `sha256:${"b".repeat(64)}`), {code: "refresh_task_lease_unavailable"}); + assert.deepEqual(await readFile(path), before); + assert.deepEqual(await readdir(directory), files); + const completedTerminal = {...pending(), terminal_closeout: {ok: true, appended: true, + completion: {todo_id: selector.todo_id, continuation: "no_followup"}}}; + delete (completedTerminal as JsonObject).effect_attempts; + for (const changed of [ + {...pending(), host_result: {turn_key: key, result_kind: "outcome_progress"}}, + {...pending(), quota_spend: {ok: false, appended: false}}, + completedTerminal, + journal(), + ]) { + await writeFile(path, JSON.stringify(changed)); + assert.equal(await read(), null); + } + const sourcePending = pending(); + (sourcePending.plan as JsonObject).goal_ref = goalRefA; + ((sourcePending.plan as JsonObject).transaction as JsonObject).goal_ref = goalRefA; + await writeFile(path, JSON.stringify(sourcePending)); + await assert.rejects(read(), {code: "refresh_task_lease_unavailable"}); +}); + test("missing history is a read-only observation", async t => { const { root, directory } = await fixture(t); assert.deepEqual(await lookup(root), { turn_key: null }); diff --git a/tests/test_codex_turn_completion.py b/tests/test_codex_turn_completion.py index 614cebc3de..3cc73cd989 100644 --- a/tests/test_codex_turn_completion.py +++ b/tests/test_codex_turn_completion.py @@ -6,6 +6,7 @@ import io import json import sys +from hashlib import sha256 from pathlib import Path from types import SimpleNamespace @@ -16,6 +17,10 @@ from loopx.cli_commands import turn_run_once from loopx.cli_commands.turn import handle_turn_command from loopx.control_plane.todos.mutation_api import add_goal_todo +from loopx.control_plane.work_items.task_lease import ( + release_task_lease, + renew_task_lease, +) from loopx.control_plane.turn_driver.codex_cli import codex_cli_result_schema from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.test_loopx_turn_codex_cli import _fake_codex @@ -487,7 +492,14 @@ def lose_terminal_attempt(**kwargs): "todo_id": todo_id, "continuation": "no_followup", } - assert len(lease_reads) == (2 if interruption == "after_debit" else 1) + assert lease_reads and set(lease_reads) == {lease["version"]} + if interruption is None: + assert len(lease_reads) == 1 + else: + # Resume rechecks the original lease at each remaining effect boundary. + # Correctness is the retained proof and once-only effects below, rather + # than the old resolver implementation's exact number of reads. + assert len(lease_reads) >= 2 assert _cli(registry, runtime, *lease_argv)[1]["active"] is False assert _effect_kinds(runtime, journal) == [ "refresh_state", @@ -505,6 +517,332 @@ def lose_terminal_attempt(**kwargs): ] +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "scenario", + [ + "init_replacement", + "completion_replacement", + "writeback_replacement", + "renewed_retired_writeback", + "accepted_writeback_spend", + "resolver_value_error", + "resolver_runtime_error", + "resolver_timeout_error", + ], +) +def test_native_resume_qualifies_fresh_authority_without_replacing_prepared_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + provider: str, + scenario: str, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture( + tmp_path, provider=provider, handoff_mode="hard_lease", + ) + goal_id, agent_id, todo_id = "loopx-turn-fixture", "codex-fixture", "todo_fixture0001" + claim_args = ("todo", "claim", "--goal-id", goal_id, "--todo-id", todo_id, + "--agent-id", agent_id, "--claimed-by", agent_id) + code, claimed = _cli(registry, runtime, *claim_args) + assert code == 0, claimed + + def inspect_lease(): + code, inspected = _cli(registry, runtime, "task-lease", "inspect", + "--goal-id", goal_id, "--todo-id", todo_id) + assert code == 0, inspected + return inspected["lease"] + + original_lease = inspect_lease() + resolver_lease = original_lease + adapter = SimpleNamespace(controller=SimpleNamespace( + registry_path=registry, coordination_runtime_root=runtime, + )) + host_calls, resolver_errors, completion_calls, refresh_requests = [], [], [], [] + resolver_calls = [] + resolver_fault = None + progress = scenario == "writeback_replacement" + accounting_only = scenario == "accepted_writeback_spend" + + def resolve_lease(): + resolver_calls.append(resolver_lease["version"]) + if resolver_fault is not None: + raise resolver_fault + try: + return StewardTurnHost._task_lease_proof( + adapter, {"actor": {"goal_id": goal_id}, "todo_id": todo_id}, resolver_lease, + ) + except ValueError as exc: + resolver_errors.append(str(exc)) + raise + + def host(request, launch_lifetime): + with launch_lifetime(): + host_calls.append(request["turn_key"]) + (project / "completion-artifact.txt").write_text("completed", encoding="utf-8") + return { + "schema_version": "loopx_turn_result_v0", "turn_key": request["turn_key"], + "result_kind": "validated_progress" if progress else "validated_completion", + "completed_phases": ["host_execute", "typed_result"], + "classification": "fixture_completion", "recommended_action": "Refresh the active Goal.", + "next_action": "Read the fresh decision.", "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_progress", "vision_unchanged_reason": "The remaining Goal is unchanged.", + "work_evidence": ["artifact:completion-artifact"], "validation_evidence": ["check:artifact-contents"], + } + + _, validation = _completion_host_and_validation_scripts() + base = ["turn", "run-once", "--host", "generic-cli", "--goal-id", goal_id, + "--agent-id", agent_id, "--project", str(project), "--no-global-sync", + "--validation-command-json", json.dumps([sys.executable, "-c", validation]), "--execute"] + + def invoke(*extra): + args = build_parser().parse_args([*base, *extra]) + args.host, args.session_scope = "codex-app-server", "agent" + result = {} + code = handle_turn_command( + args, registry_path=registry, runtime_root_arg=str(runtime), + output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), + native_host_runner=host, native_completion_lease_resolver=resolve_lease, + native_session_binding_resolver=lambda _: { + "schema_version": "loopx_turn_session_binding_v0", "goal_id": goal_id, "agent_id": agent_id, + }, + ) + return code, result + + def pause_before_commit(*_args, **_kwargs): + raise OSError("fixture pause before native settlement") + + with monkeypatch.context() as paused: + pause_target = ("spend_quota_slot" if accounting_only else + "refresh_state_run" if progress else "write_turn_validated_completion") + paused.setattr(turn_run_once, pause_target, + pause_before_commit) + code, failed = invoke("--todo-id", todo_id) + assert code == 1 and failed["error"] == "fixture pause before native settlement", failed + journal = _turn_journal(runtime) + assert journal["host_attempt_count"] == 1 and journal["host_result"]["result_kind"] == ( + "validated_progress" if progress else "validated_completion" + ) + frozen_result = journal["host_result"] + transaction = journal["plan"]["transaction"] + run_id = transaction.get("turn_instance_id", journal["turn_key"]) + event_path = runtime / "goals" / goal_id / "rollout-event-log.jsonl" + index_path = runtime / "goals" / goal_id / "runs" / "index.jsonl" + + def original_receipt_lines(): + return [line for line in event_path.read_bytes().splitlines(keepends=True) + if (row := json.loads(line)).get("event_kind") == "quota_should_run" + and row.get("run_id") == run_id and row.get("agent_id") == agent_id] + + receipt_bytes = original_receipt_lines() + assert receipt_bytes and _effect_kinds(runtime, journal) == ( + ["todo_complete", "refresh_state"] if accounting_only else [] + ) + original_readback = turn_run_once.read_heartbeat_settlement( + runtime, goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, + turn_instance_id=run_id, registry_path=registry, + ) + assert original_readback is not None and original_readback.prepared_lease is not None + original_prepared = { + "prepared_lease_key_sha256": sha256(original_lease["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": original_lease["version"], + "prepared_lease_epoch": original_lease["lease_epoch"], + } + assert original_readback.prepared_lease == original_prepared + resolver_calls_before_resume = list(resolver_calls) + if accounting_only: + # This is accepted business writeback followed by independent accounting. + # The real active-proof resolver cannot offer the already retired lease. + retired = inspect_lease() + assert retired["status"] == "released" + code, current_todo = _cli(registry, runtime, "todo", "list", "--goal-id", goal_id, "--todo-id", todo_id) + assert code == 0 and current_todo["todo"]["todo_id"] == todo_id and current_todo["todo"]["status"] == "done", current_todo + assert original_readback.completion_event is not None + assert original_readback.writeback.failure is None and original_readback.writeback_run is not None + assert original_readback.spend_event is None and original_readback.spend_run is None + assert original_readback.progress is not None and original_readback.progress["state"] == "spend_required" + assert journal["completed_phases"][-1] == "durable_writeback" + inspection = turn_run_once.inspect_loopx_turn_journal( + runtime, goal_id=goal_id, agent_id=agent_id, turn_key=journal["turn_key"], retry_failed=True, + ) + recovery = inspection["recovery_decision"] + assert recovery["action"] == "continue" and recovery["can_continue"] is True + assert recovery["resume_from"] == "quota_spend" and recovery["reinvoke_host"] is False + with pytest.raises(ValueError, match="original task execution lease changed"): + StewardTurnHost._task_lease_proof( + adapter, {"actor": {"goal_id": goal_id}, "todo_id": todo_id}, resolver_lease, + ) + actual_qualification = turn_run_once.read_turn_execution_lease_qualification + qualification_requests, qualifications = [], [] + + def observe_qualification(**kwargs): + qualification_requests.append(kwargs["requested_proof"]) + observed = actual_qualification(**kwargs) + qualifications.append(observed.resume_lease_qualification) + return observed + + monkeypatch.setattr(turn_run_once, "read_turn_execution_lease_qualification", observe_qualification) + replacement = None + + def replace_execution(*, offer_current=True): + nonlocal replacement, resolver_lease + current = inspect_lease() + released = release_task_lease( + registry_path=registry, runtime_root=runtime, goal_id=goal_id, todo_id=todo_id, + owner=agent_id, idempotency_key=current["idempotency_key"], expected_version=current["version"], + ) + assert released["ok"] is True, released + code, reclaimed = _cli(registry, runtime, *claim_args, + "--claim-operation-id", "native-resume-replacement") + assert code == 0, reclaimed + replacement = inspect_lease() + assert replacement["status"] == "active" and replacement["owner"] == agent_id + assert replacement["lease_epoch"] > original_lease["lease_epoch"] + assert replacement["version"] > original_lease["version"] + assert replacement["idempotency_key"] != original_lease["idempotency_key"] + if offer_current: + resolver_lease = replacement + + if scenario == "init_replacement": + replace_execution() + elif scenario == "resolver_value_error": + # The real resolver still holds P1 and sees canonical active P2. Its + # ValueError is not proof that the original completion retired P1. + replace_execution(offer_current=False) + elif scenario == "resolver_runtime_error": + resolver_fault = RuntimeError("fixture resolver unavailable") + elif scenario == "resolver_timeout_error": + resolver_fault = TimeoutError("fixture resolver timeout") + elif scenario == "renewed_retired_writeback": + renewed = renew_task_lease( + registry_path=registry, runtime_root=runtime, goal_id=goal_id, todo_id=todo_id, + owner=agent_id, idempotency_key=original_lease["idempotency_key"], + expected_version=original_lease["version"], + ) + assert renewed["ok"] is True, renewed + resolver_lease = inspect_lease() + assert resolver_lease["idempotency_key"] == original_lease["idempotency_key"] + assert resolver_lease["lease_epoch"] == original_lease["lease_epoch"] + assert resolver_lease["version"] > original_lease["version"] + + actual_resolvers = turn_run_once.build_turn_settlement_resolvers + + def guarded_resolvers(**kwargs): + resolvers = actual_resolvers(**kwargs) + if scenario == "writeback_replacement": + # Init already read P1. Ordinary writeback must fetch authority + # again instead of using that successful qualification. + assert qualifications and qualifications[-1]["source"] == "active" + replace_execution() + if scenario != "completion_replacement": + return resolvers + completion_readback, *remaining = resolvers + + def replace_after_completion_readback(): + observed = completion_readback() + assert observed is None + assert qualifications and qualifications[-1]["source"] == "active" + replace_execution() + return observed + + return (replace_after_completion_readback, *remaining) + + monkeypatch.setattr(turn_run_once, "build_turn_settlement_resolvers", guarded_resolvers) + actual_completion, actual_refresh = turn_run_once.write_turn_validated_completion, turn_run_once.refresh_state_run + lose_refresh = scenario == "renewed_retired_writeback" + + def record_completion(**kwargs): + completion_calls.append(kwargs) + return actual_completion(**kwargs) + + def record_refresh(**kwargs): + nonlocal lose_refresh + refresh_requests.append(kwargs) + if lose_refresh: + lose_refresh = False + raise OSError("fixture refresh interruption after canonical completion") + return actual_refresh(**kwargs) + + monkeypatch.setattr(turn_run_once, "write_turn_validated_completion", record_completion) + monkeypatch.setattr(turn_run_once, "refresh_state_run", record_refresh) + spend_requests = [] + actual_spend = turn_run_once.spend_quota_slot + + def record_spend(*args, **kwargs): + spend_requests.append(kwargs) + return actual_spend(*args, **kwargs) + + monkeypatch.setattr(turn_run_once, "spend_quota_slot", record_spend) + resume = ("--resume-turn-key", journal["turn_key"], "--retry-failed-turn") + code, resumed = invoke(*resume) + if accounting_only: + assert code == 0 and resumed["status"] == "committed", resumed + assert resumed["effects"]["host_invoked"] is False and resumed["effects"]["state_written"] is False + assert resolver_calls == resolver_calls_before_resume and resolver_errors == [] + assert qualification_requests == [] and completion_calls == [] and refresh_requests == [] + assert len(spend_requests) == 1 and original_readback.identity.value is not None + assert spend_requests[0]["effect_ref"] == f"{original_readback.identity.value.effect_id}#quota_spend" + assert _effect_kinds(runtime, journal) == ["todo_complete", "refresh_state", "quota_spend"] + assert sum(json.loads(line).get("event_kind") == "quota_spend" + for line in event_path.read_bytes().splitlines()) == 1 + assert inspect_lease() == retired + elif scenario == "renewed_retired_writeback": + assert code == 1 and resumed["error"] == "fixture refresh interruption after canonical completion", resumed + assert _effect_kinds(runtime, journal) == ["todo_complete"] + assert not index_path.exists() or not index_path.read_text().strip() + retired = inspect_lease() + assert retired["status"] == "released" and retired["lease_epoch"] == original_lease["lease_epoch"] + assert retired["idempotency_key"] == original_lease["idempotency_key"] + assert len(completion_calls) == 1 + assert completion_calls[0]["task_lease_expected_version"] == resolver_lease["version"] + assert completion_calls[0]["task_lease_idempotency_key"] == original_lease["idempotency_key"] + assert any(row["source"] == "retired" for row in qualifications) + code, resumed = invoke(*resume) + assert code == 0 and resumed["status"] == "committed", resumed + assert resumed["effects"]["host_invoked"] is False and resolver_errors + assert len(completion_calls) == 1 and len(refresh_requests) == 2 + assert refresh_requests[-1]["task_lease_proof"] == { + "idempotency_key": retired["idempotency_key"], "expected_version": retired["version"], + } + assert _effect_kinds(runtime, journal) == ["todo_complete", "refresh_state", "quota_spend"] + assert sum(json.loads(line).get("event_kind") == "quota_spend" + for line in event_path.read_bytes().splitlines()) == 1 + else: + assert code == 1, resumed + if scenario in {"resolver_runtime_error", "resolver_timeout_error"}: + assert resumed["error"] == str(resolver_fault) + assert resumed.get("error_code") != "refresh_task_lease_unavailable" + assert qualification_requests == [] + else: + assert resumed["error_code"] == "refresh_task_lease_unavailable", resumed + assert qualification_requests[-1] == (None if scenario == "resolver_value_error" else { + "idempotency_key": replacement["idempotency_key"], "expected_version": replacement["version"], + }) + assert inspect_lease() == replacement + assert bool(resolver_errors) is (scenario == "resolver_value_error") + assert completion_calls == [] and refresh_requests == [] and spend_requests == [] + assert _effect_kinds(runtime, journal) == [] + assert not any(json.loads(line).get("event_kind") == "quota_spend" + for line in event_path.read_bytes().splitlines()) + assert not index_path.exists() or not index_path.read_text().strip() + code, current_todo = _cli(registry, runtime, "todo", "list", "--goal-id", goal_id, "--todo-id", todo_id) + assert code == 0 and current_todo["todo"]["todo_id"] == todo_id and current_todo["todo"]["status"] == "open", current_todo + assert host_calls == [journal["turn_key"]] + assert _turn_journal(runtime)["host_attempt_count"] == 1 + assert _turn_journal(runtime)["host_result"] == frozen_result + assert original_receipt_lines() == receipt_bytes + assert all(row["state"] == "ready" and row["original_prepared"] == original_prepared + for row in qualifications) + final_readback = turn_run_once.read_heartbeat_settlement( + runtime, goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, + turn_instance_id=run_id, registry_path=registry, + ) + assert final_readback is not None and final_readback.prepared_lease == original_prepared + if accounting_only: + assert final_readback.writeback_run == original_readback.writeback_run + assert final_readback.spend_event is not None and final_readback.spend_run is not None + + def test_builtin_codex_advertises_explicit_completion_without_promoting_progress() -> ( None ): diff --git a/tests/test_loopx_turn_driver.py b/tests/test_loopx_turn_driver.py index 88acff531d..3b4a4f7e81 100644 --- a/tests/test_loopx_turn_driver.py +++ b/tests/test_loopx_turn_driver.py @@ -2913,6 +2913,219 @@ def run(extra=()): assert not index_path.exists() +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("fault", ["initial_after", "attempt_before"]) +def test_native_resume_rechecks_original_lease_when_binding_admitted_start( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, fault: str, +) -> None: + from hashlib import sha256 + from types import SimpleNamespace + + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + from loopx.cli import build_parser + from loopx.cli_commands.turn import handle_turn_command + from loopx.control_plane.effect_runtime import effect_runtime_result + from loopx.control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from loopx.control_plane.turn_driver import executor + from loopx.control_plane.work_items.task_lease import release_task_lease + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture( + tmp_path, provider=provider, handoff_mode="hard_lease", + ) + goal_id, agent_id, todo_id = "loopx-turn-fixture", "codex-fixture", "todo_fixture0001" + scope = {"runtime_root": str(runtime), "goal_id": goal_id, + "agent_id": agent_id, "automation_id": None} + configured = effect_runtime_result("quota.automation_cadence.manage", { + **scope, "operation": "configure", "min_interval_minutes": 3, + "expected_revision": 0, "owner_reference": "fixture-owner", "execute": True, + }) + assert configured["eligible_now"] is True + + def cli(*arguments): + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli_main(["--registry", str(registry), "--runtime-root", str(runtime), + "--format", "json", *arguments]) + return code, json.loads(output.getvalue()) + + claim_args = ("todo", "claim", "--goal-id", goal_id, "--todo-id", todo_id, + "--agent-id", agent_id, "--claimed-by", agent_id) + code, claimed = cli(*claim_args) + assert code == 0, claimed + + def inspect_lease(): + code, inspected = cli("task-lease", "inspect", "--goal-id", goal_id, "--todo-id", todo_id) + assert code == 0 and inspected["active"] is True, inspected + return inspected["lease"] + + original_lease = inspect_lease() + resolver_lease = original_lease + adapter = SimpleNamespace(controller=SimpleNamespace( + registry_path=registry, coordination_runtime_root=runtime, + )) + host_calls = [] + artifact = project / "completion-artifact.txt" + + def resolve_lease(): + return StewardTurnHost._task_lease_proof( + adapter, {"actor": {"goal_id": goal_id}, "todo_id": todo_id}, resolver_lease, + ) + + def host(request, launch_lifetime): + host_calls.append(request["turn_key"]) + with launch_lifetime(): + artifact.write_text("completed", encoding="utf-8") + return { + "schema_version": "loopx_turn_result_v0", "turn_key": request["turn_key"], + "result_kind": "validated_completion", "completed_phases": ["host_execute", "typed_result"], + "classification": "fixture_completion", "recommended_action": "Refresh the active Goal.", + "next_action": "Read the fresh decision.", "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_progress", "vision_unchanged_reason": "The remaining Goal is unchanged.", + "work_evidence": ["artifact:completion-artifact"], "validation_evidence": ["check:artifact-contents"], + } + + _, validation = _completion_host_and_validation_scripts() + base = ["turn", "run-once", "--host", "generic-cli", "--goal-id", goal_id, + "--agent-id", agent_id, "--project", str(project), "--no-global-sync", + "--validation-command-json", json.dumps([sys.executable, "-c", validation]), "--execute"] + + def invoke(*extra): + args = build_parser().parse_args([*base, *extra]) + args.host, args.session_scope = "codex-app-server", "agent" + result = {} + code = handle_turn_command( + args, registry_path=registry, runtime_root_arg=str(runtime), + output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), + native_host_runner=host, native_completion_lease_resolver=resolve_lease, + native_session_binding_resolver=lambda _: { + "schema_version": "loopx_turn_session_binding_v0", "goal_id": goal_id, "agent_id": agent_id, + }, + ) + return code, result + + persist = executor._write_journal + interrupted = False + + def checkpoint(path, journal, **kwargs): + nonlocal interrupted + if not interrupted and fault == "attempt_before" and journal["host_attempt_count"] == 1: + interrupted = True + raise RuntimeError("fixture native start interruption") + persist(path, journal, **kwargs) + if not interrupted and fault == "initial_after" and journal["host_attempt_count"] == 0: + interrupted = True + raise RuntimeError("fixture native start interruption") + + with monkeypatch.context() as paused: + paused.setattr(executor, "_write_journal", checkpoint) + code, failed = invoke("--todo-id", todo_id, "--turn-instance-id", "native-bind-freshcheck") + assert code == 1 and interrupted, failed + assert failed["error"] == "fixture native start interruption" + journal = _turn_journal(runtime) + assert journal["host_attempt_count"] == 0 and "host_result" not in journal + assert host_calls == [] and not artifact.exists() + run_id = journal["plan"]["transaction"].get("turn_instance_id", journal["turn_key"]) + event_path = runtime / "goals" / goal_id / "rollout-event-log.jsonl" + index_path = runtime / "goals" / goal_id / "runs" / "index.jsonl" + capture_path = runtime / "goals" / goal_id / "turns" / "decisions" / ( + journal["turn_key"].removeprefix("sha256:") + ".json" + ) + original_capture = capture_path.read_bytes() + + def original_receipt_lines(): + return [line for line in event_path.read_bytes().splitlines(keepends=True) + if (row := json.loads(line)).get("event_kind") == "quota_should_run" + and row.get("run_id") == run_id and row.get("agent_id") == agent_id] + + receipt_bytes = original_receipt_lines() + assert receipt_bytes + original_prepared = { + "prepared_lease_key_sha256": sha256(original_lease["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": original_lease["version"], "prepared_lease_epoch": original_lease["lease_epoch"], + } + original_readback = turn_run_once_command.read_heartbeat_settlement( + runtime, goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, + turn_instance_id=run_id, registry_path=registry, + ) + assert original_readback is not None and original_readback.prepared_lease == original_prepared + original_cadence = effect_runtime_result("quota.automation_cadence.manage", {**scope, "operation": "read"}) + assert original_cadence["next_eligible_at_ms"] is not None + actual_qualification = turn_run_once_command.read_turn_execution_lease_qualification + qualification_requests, qualifications = [], [] + + def observe_qualification(**kwargs): + qualification_requests.append(kwargs["requested_proof"]) + observed = actual_qualification(**kwargs) + qualifications.append(observed.resume_lease_qualification) + return observed + + monkeypatch.setattr(turn_run_once_command, "read_turn_execution_lease_qualification", observe_qualification) + make_cadence = turn_run_once_command.managed_cadence_start + replacement = None + bind_refused = False + + def replace_before_bind(**kwargs): + original_callback = kwargs["on_admitted"] + + def on_admitted(): + nonlocal replacement, resolver_lease, bind_refused + # The real native resume read has accepted P1 before cadence admit. + # Its success must not authorize binding after a new claim appears. + assert any(row["state"] == "ready" and row["source"] == "active" + and row["original_prepared"] == original_prepared for row in qualifications) + assert inspect_lease() == original_lease + released = release_task_lease( + registry_path=registry, runtime_root=runtime, goal_id=goal_id, todo_id=todo_id, + owner=agent_id, idempotency_key=original_lease["idempotency_key"], + expected_version=original_lease["version"], + ) + assert released["ok"] is True, released + code, reclaimed = cli(*claim_args, "--claim-operation-id", "native-bind-replacement") + assert code == 0, reclaimed + replacement = inspect_lease() + assert replacement["owner"] == agent_id + assert replacement["idempotency_key"] != original_lease["idempotency_key"] + assert replacement["lease_epoch"] > original_lease["lease_epoch"] + assert replacement["version"] > original_lease["version"] + resolver_lease = replacement + try: + original_callback() + except RefreshTaskLeaseUnavailable: + bind_refused = True + raise + + # Keep the real TS cadence admit/confirm; only interpose at its callback. + return make_cadence(**{**kwargs, "on_admitted": on_admitted}) + + monkeypatch.setattr(turn_run_once_command, "managed_cadence_start", replace_before_bind) + code, resumed = invoke("--resume-turn-key", journal["turn_key"], "--retry-failed-turn") + assert code == 1 and bind_refused, resumed + assert resumed["error_code"] == "refresh_task_lease_unavailable", resumed + assert replacement is not None and inspect_lease() == replacement + assert qualification_requests[-1] == { + "idempotency_key": replacement["idempotency_key"], "expected_version": replacement["version"], + } + current = _turn_journal(runtime) + assert current["host_attempt_count"] == 0 and "host_result" not in current + assert current["completed_phases"] == [] + assert host_calls == [] and not artifact.exists() + assert original_receipt_lines() == receipt_bytes + assert capture_path.read_bytes() == original_capture + final_readback = turn_run_once_command.read_heartbeat_settlement( + runtime, goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, + turn_instance_id=run_id, registry_path=registry, + ) + assert final_readback is not None and final_readback.prepared_lease == original_prepared + assert not index_path.exists() or not index_path.read_text().strip() + assert not any(json.loads(line).get("event_kind") in {"refresh_state", "quota_spend", "todo_complete"} + for line in event_path.read_bytes().splitlines()) + current_cadence = effect_runtime_result("quota.automation_cadence.manage", {**scope, "operation": "read"}) + assert current_cadence["next_eligible_at_ms"] == original_cadence["next_eligible_at_ms"] + code, todo = cli("todo", "list", "--goal-id", goal_id, "--todo-id", todo_id) + assert code == 0 and todo["todo"]["todo_id"] == todo_id and todo["todo"]["status"] == "open", todo + + def test_turn_run_once_cli_dsh_fresh_sessions_and_iteration_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_steward_report_recovery_and_validator.py b/tests/test_steward_report_recovery_and_validator.py index ae879b9e0a..b97efff389 100644 --- a/tests/test_steward_report_recovery_and_validator.py +++ b/tests/test_steward_report_recovery_and_validator.py @@ -1,5 +1,6 @@ """Focused regressions for report validator path portability and cancel recovery.""" from datetime import datetime, timedelta, timezone +from hashlib import sha256 import json import logging @@ -181,7 +182,11 @@ def test_selection_recovery_admission_resume_tolerates_validator_python_upgrade( steward, tmp_path, monkeypatch, caplog, ): from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.capabilities.native_chat import steward_commands from loopx.configure_goal import configure_goal + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.rollout_event_log import rollout_event_log_path from test_native_steward_reports import _REPORT_MODEL_SOURCE store, runtime, _, transport, binding, _, _ = steward @@ -208,6 +213,35 @@ def now(cls, tz=None): runtime._steward_monitor_next_check.clear() rejected = finish(runtime, pump_steward_monitors(runtime)[0]) assert rejected["response"]["native_loopx"]["error_code"] == "turn_todo_selection_deferred" + request = rejected["loopx_request"] + report_acquires = [] + actual_command = steward_commands.native_command + + def observe_acquire(controller, argv): + result = actual_command(controller, argv) + if argv[:2] == ["task-lease", "acquire"]: + assert argv[argv.index("--todo-id") + 1] == request["todo_id"] + report_acquires.append(dict(result["lease"])) + return result + + def report_readback(): + return read_heartbeat_settlement(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"], registry_path=runtime.registry_path) + + def model_call_count(): + return sum(json.loads(line).get("method") == "turn/start" + for line in steward[5].read_text().splitlines()) + + rollout_path = rollout_event_log_path(reports.root, actor["goal_id"]) + + def replan_receipt_lines(): + return [line for line in rollout_path.read_bytes().splitlines(keepends=True) + if (row := json.loads(line)).get("event_kind") == "quota_should_run" + and row.get("run_id") == request["turn_instance_id"] + ".replan"] + + monkeypatch.setattr(steward_commands, "native_command", observe_acquire) + model_calls_before_probe = model_call_count() bad_vision = { "schema_version": "goal_vision_replan_contract_v0", @@ -247,6 +281,22 @@ def now(cls, tz=None): ) runtime._steward_monitor_next_check.clear() probe = finish(runtime, pump_steward_monitors(runtime)[0]) + assert report_acquires == [] + assert model_call_count() == model_calls_before_probe + 1 + governance = next(row["payload"] for row in store.events_after(sid, probe["turn_id"], None) + if row["kind"] == "native_loopx.replan_result") + assert governance["status"] == "failed" and governance["validation_stage"] == "host_result_contract" + assert governance["effects"]["state_written"] is False and governance["effects"]["quota_spent"] is False + business = report_readback() + assert business is not None and business.identity.failure is not None + assert business.identity.failure.kind == "receipt_missing" + assert business.prepared_lease is None and business.original_prepared_receipt is None + assert business.writeback_run is None and business.spend_run is None + probe_journal_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], + turn_key=governance["resume_turn_key"]) + probe_journal_bytes = probe_journal_path.read_bytes() + probe_receipt_bytes = replan_receipt_lines() + assert probe_receipt_bytes # Simulate host Python interpreter path upgrade on both the original refused turn and probe turn. for row_id in (rejected["turn_id"], probe["turn_id"]): @@ -273,8 +323,21 @@ def now(cls, tz=None): assert resumed[0]["client_turn_id"] == expected_client_id done = finish(runtime, resumed[0]) assert done["response"]["native_loopx"]["status"] == "committed", done + assert len(report_acquires) == 1 + assert model_call_count() == model_calls_before_probe + 2 + business = report_readback() + assert business is not None and business.prepared_lease == { + "prepared_lease_key_sha256": sha256(report_acquires[0]["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": report_acquires[0]["version"], + "prepared_lease_epoch": report_acquires[0]["lease_epoch"], + } + assert business.original_prepared_receipt is not None + assert probe_journal_path.read_bytes() == probe_journal_bytes + assert replan_receipt_lines() == probe_receipt_bytes + model_bytes = steward[5].read_bytes() runtime._steward_monitor_next_check.clear() assert not pump_steward_monitors(runtime) + assert steward[5].read_bytes() == model_bytes assert transport.steward_reports.reconcile() == 1 assert reports.health(owner)["publication_verified"] task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == done["loopx_request"]["todo_id"]) @@ -300,12 +363,16 @@ def test_steward_report_normalizes_profile_minimum_scale_shorthand(steward, tmp_ assert reports.health(binding["binding_id"])["publication_verified"] +@pytest.mark.parametrize("replace_original_lease", [False, True], ids=["same-generation", "replacement-refused"]) def test_frozen_report_recovers_host_result_contract_failure_without_model_call( - steward, tmp_path, monkeypatch, caplog, + steward, tmp_path, monkeypatch, caplog, replace_original_lease, ): from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.capabilities.native_chat import steward_commands import loopx.capabilities.native_chat.steward_turn as steward_turn_mod from loopx.configure_goal import configure_goal + from loopx.control_plane.quota import execution_preparation + from loopx.control_plane.quota.settlement import read_heartbeat_settlement _, runtime, _, transport, binding, _, _ = steward point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) @@ -358,19 +425,203 @@ def now(cls, tz=None): assert native["status"] == "failed" assert native["result_kind"] == "validation_failed" assert native["validation_stage"] == "host_result_contract" + assert native["effects"]["state_written"] is False + assert native["effects"]["quota_spent"] is False window = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"])["window"] - assert (attempt_directory(reports.root, window) / "private-generation.json").is_file() + generation_path = attempt_directory(reports.root, window) / "private-generation.json" + generation_bytes = generation_path.read_bytes() + request = resumed["loopx_request"] + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + + def readback(): + return read_heartbeat_settlement( + reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], + todo_id=request["todo_id"], turn_instance_id=request["turn_instance_id"], + registry_path=runtime.registry_path, + ) + original_lease = steward_commands.native_command(runtime, ["task-lease", "inspect", *route])["lease"] + original_readback = readback() + assert original_readback is not None and original_readback.original_prepared_receipt is not None + original_prepared = original_readback.prepared_lease + assert original_prepared == { + "prepared_lease_key_sha256": sha256(original_lease["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": original_lease["version"], + "prepared_lease_epoch": original_lease["lease_epoch"], + } + assert original_lease["status"] == "active" + assert original_readback.writeback_run is None and original_readback.writeback_event is None + assert original_readback.spend_run is None and original_readback.spend_event is None + + commands, qualification_requests, qualifications = [], [], [] + actual_command = steward_commands.native_command + actual_qualification = execution_preparation.read_turn_execution_lease_qualification + + def observe_command(controller, argv): + result = actual_command(controller, argv) + if argv[:2] in (["task-lease", "renew"], ["task-lease", "acquire"]): + commands.append((list(argv), dict(result["lease"]))) + return result + + def observe_qualification(**kwargs): + qualification_requests.append(kwargs["requested_proof"]) + result = actual_qualification(**kwargs) + qualifications.append(result.resume_lease_qualification) + return result + + monkeypatch.setattr(steward_commands, "native_command", observe_command) + monkeypatch.setattr(execution_preparation, "read_turn_execution_lease_qualification", observe_qualification) + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) model_calls = steward[5].read_bytes() + replacement = None + if replace_original_lease: + actual_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], + "--idempotency-key", original_lease["idempotency_key"], + "--expected-version", str(original_lease["version"])]) + released = actual_command(runtime, ["task-lease", "inspect", *route])["lease"] + replacement = actual_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", request["turn_instance_id"] + ".replacement", "--ttl-seconds", "3600", + "--expected-version", str(released["version"])])["lease"] + assert replacement["idempotency_key"] != original_lease["idempotency_key"] + assert replacement["lease_epoch"] > original_lease["lease_epoch"] + assert replacement["version"] > original_lease["version"] + with pytest.raises(execution_preparation.RefreshTaskLeaseUnavailable, match="replacement lease"): + StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + runtime._steward_monitor_next_check.clear() with caplog.at_level(logging.WARNING): assert not pump_steward_monitors(runtime) - assert "An original steward report is unavailable" not in caplog.text assert steward[5].read_bytes() == model_calls - assert transport.steward_reports.reconcile() == 1 - assert reports.health(owner)["publication_verified"] + assert generation_path.read_bytes() == generation_bytes + final_readback = readback() + assert final_readback is not None + assert final_readback.original_prepared_receipt == original_readback.original_prepared_receipt + assert final_readback.original_prepared_event_id == original_readback.original_prepared_event_id + assert final_readback.original_prepared_receipt_sha256 == original_readback.original_prepared_receipt_sha256 + current_lease = actual_command(runtime, ["task-lease", "inspect", *route])["lease"] task = next( row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == resumed["loopx_request"]["todo_id"] ) - assert task["status"] == "done" + if replace_original_lease: + assert "An original steward report is unavailable" in caplog.text + assert commands == [] and qualifications == [] + assert qualification_requests and all(proof == { + "idempotency_key": replacement["idempotency_key"], "expected_version": replacement["version"], + } for proof in qualification_requests) + assert current_lease == replacement + assert task["status"] == "open" + assert final_readback.writeback_run is None and final_readback.writeback_event is None + assert final_readback.spend_run is None and final_readback.spend_event is None + assert final_readback.completion_event is None + assert transport.steward_reports.reconcile() == 0 + else: + assert "An original steward report is unavailable" not in caplog.text + assert commands and all(argv[:2] == ["task-lease", "renew"] for argv, _ in commands) + assert qualifications and all(row["state"] == "ready" and row["original_prepared"] == original_prepared + for row in qualifications) + for lease in [current_lease, *(lease for _, lease in commands)]: + assert lease["idempotency_key"] == original_lease["idempotency_key"] + assert lease["lease_epoch"] == original_lease["lease_epoch"] + assert lease["version"] >= original_lease["version"] + assert current_lease["version"] > original_lease["version"] + assert current_lease["status"] == "released" + assert actual_command(runtime, ["task-lease", "inspect", *route])["active"] is False + assert final_readback.writeback_run is not None and final_readback.spend_run is not None + assert task["status"] == "done" + assert transport.steward_reports.reconcile() == 1 + assert reports.health(owner)["publication_verified"] + + +@pytest.mark.parametrize("cleanup_boundary", ["normal", "rollout_occupied"]) +def test_legacy_native_task_lease_cleanup_uses_fenced_release( + steward, tmp_path, monkeypatch, cleanup_boundary, +): + from concurrent.futures import ThreadPoolExecutor + import threading + + from loopx.capabilities.native_chat.steward_commands import native_command + from loopx.control_plane.quota import settlement + from loopx.file_lock import exclusive_cross_runtime_file_lock + from loopx.rollout_event_log import rollout_event_log_path + + store, runtime, *_ = steward + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + assert actor.get("goal_instance_id") is None + added = native_command(runtime, ["todo", "add", "--goal-id", actor["goal_id"], + "--role", "agent", "--text", "[P1] Inspect the authorized task before admission.", + "--claimed-by", actor["agent_id"], "--operation-id", "lease-cleanup-fixture", "--execute"]) + turn, _ = store.create_queued_turn(sid, client_turn_id="legacy-lease-cleanup", + message="Inspect the original task before admission.") + request = {"actor": actor, "todo_id": added["todo_id"], "turn_instance_id": "legacy-lease-cleanup"} + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + actual_readback = settlement.read_heartbeat_settlement + dispositions = [] + rollout_path = rollout_event_log_path(reports.root, actor["goal_id"]) + original_rollout = rollout_path.read_bytes() if rollout_path.exists() else None + rollout_held, release_rollout, rollout_released = threading.Event(), threading.Event(), threading.Event() + + def occupy_rollout(): + with exclusive_cross_runtime_file_lock( + rollout_event_log_path(reports.root, actor["goal_id"]), operation="fixture_competing_rollout_owner", + ): + rollout_held.set() + assert release_rollout.wait(15), "cleanup waited for the occupied rollout lock" + rollout_released.set() + + with ThreadPoolExecutor(max_workers=1) as workers: + competing_owner = None + + def observe_cleanup(*args, **kwargs): + nonlocal competing_owner + result = actual_readback(*args, **kwargs) + if kwargs.get("native_task_lease_cleanup") is None: + return result + assert kwargs["goal_ref"] is None and kwargs["borrow_source_admission"] is False + assert kwargs["native_task_lease_cleanup"] == { + "schema_version": "native_task_lease_cleanup_request_v0", "recovery": False, + "admission_retry": False, "resume_turn_key_present": False, "original_journal_present": False, + } + assert result.identity.failure is not None and result.identity.failure.kind == "receipt_missing" + assert result.heartbeat_receipt is None + assert result.native_task_lease_cleanup == { + "schema_version": "native_task_lease_cleanup_v0", "disposition": "release", + "reason": "fresh_pre_admission_missing", + } + assert result.writeback_run is None and result.writeback_event is None + assert result.spend_run is None and result.spend_event is None + assert result.completion_event is None + dispositions.append(result.native_task_lease_cleanup["disposition"]) + if len(dispositions) == 1 and cleanup_boundary == "rollout_occupied": + competing_owner = workers.submit(occupy_rollout) + assert rollout_held.wait(10), "competing owner did not acquire rollout" + return result + + monkeypatch.setattr(settlement, "read_heartbeat_settlement", observe_cleanup) + try: + with StewardTurnHost(runtime)._task_execution_lease(request, turn["turn_id"]) as lease: + original_lease = dict(lease) + if cleanup_boundary == "rollout_occupied": + assert competing_owner is not None and not competing_owner.done() + assert rollout_held.is_set() and not rollout_released.is_set() + finally: + release_rollout.set() + if competing_owner is not None: + competing_owner.result(timeout=10) + + inspected = native_command(runtime, ["task-lease", "inspect", *route]) + diagnostics = [row for row in store.events_after(sid, turn["turn_id"], None) + if row["kind"] == "native_loopx.task_lease_release_unconfirmed"] + assert (rollout_path.read_bytes() if rollout_path.exists() else None) == original_rollout + if cleanup_boundary == "normal": + assert dispositions == ["release", "release"] + assert inspected["active"] is False and inspected["lease"]["status"] == "released" + assert inspected["lease"]["idempotency_key"] == original_lease["idempotency_key"] + assert inspected["lease"]["lease_epoch"] == original_lease["lease_epoch"] + assert diagnostics == [] + else: + assert dispositions == ["release"] + assert inspected["active"] is True and inspected["lease"] == original_lease + assert len(diagnostics) == 1 and diagnostics[0]["payload"] == { + "todo_id": request["todo_id"], "lease_epoch": original_lease["lease_epoch"], + } From 99eebb9db0c6943bf69ce0f49141e90f3bf31ea3 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 10:49:32 +0800 Subject: [PATCH 07/12] fix(quota): qualify semantic writeback as an independent operation Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 10 + .../monorepo-distribution-split-v0.zh-CN.md | 8 + docs/reference/protocols/turn-envelope-v0.md | 28 +- .../quota/execution_preparation.py | 66 +- .../quota/heartbeat_receipt_identity.ts | 89 +- loopx/control_plane/quota/settlement.py | 11 + .../quota/settlement_readback.ts | 328 +++++++- loopx/state_refresh.py | 35 +- .../test_quota_execution_preparation.py | 783 ++++++++++++------ .../heartbeat_resume_lease.test.ts | 170 +++- .../quota_settlement_readback.test.ts | 193 +++++ 11 files changed, 1403 insertions(+), 318 deletions(-) diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 2e3912e85f..d64a4f8a00 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4820,6 +4820,16 @@ view shows its independently checked result in the same conversation. existing TS admission must permit the report before its preparation acquires one. Historical generation conflicts reject, and validated causal links preserve the original prepared revision. +- **Semantic writeback scope:** an explicit semantic closeout independently + qualifies the actual Run, original receipt/Goal/Todo/Turn, current generation, + progress and Vision/checkpoint in TS. Exact supersession allows only the + original semantic writeback; business resume still rejects it. A Todo-bound + replan without a business lease proves its selected obligation and semantic + evidence. Required preparation without an original positive proof rejects. + The scope grants no execution lease. The supported writer rereads inside the + shared maintenance marker and retains Goal/source ownership through JSON, + Markdown and index durability; direct provider writes bypassing the marker + remain outside that guarantee. - **Source evidence:** CLI tests cover both local providers, stale/foreign proofs, ordinary parity and zero completion/debit. Recovery tests inject interruption after JSON, Markdown, index and receipt durability. SourceSession diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index 59acec0b78..9e1a0e76f6 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3722,6 +3722,14 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 报告业务 lease;既有 TS admission 允许报告后,才在 preparation 前获取。 历史 generation 冲突拒绝,已核验因果链接保留 原 prepared revision。 +- **语义写回范围:**显式 semantic closeout 在 TS 内独立核验实际 Run、原 + receipt/Goal/Todo/Turn、当前 generation、progress 与 Vision/checkpoint。 + Exact supersede 只允许原 semantic writeback,business resume 仍拒绝。 + 无业务 lease 的 Todo-bound replan 须证明 selected obligation 与语义证据; + 要求 preparation 却缺少原 positive proof 时拒绝。该 scope 不授予 execution + lease。受支持 writer 在共享 maintenance marker 内重新读取,并持有 Goal/source + ownership 直到 JSON、Markdown、index 持久化;绕过 marker 的直接 provider + 写仍不属于此保证。 - **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index c47fa41e2c..d23bc945c5 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -96,7 +96,8 @@ and provides no ABA guarantee. A different generation cannot complete the old Turn, including a frozen report; it requires execution-loss closeout or a fresh Turn through the existing C1 owner. Every effect boundary rereads its owning authority; retired authority is usable only for the original Turn's proven post-completion -writeback or its verified pending terminal-receipt repair. A conditional terminal +writeback, independently qualified semantic writeback after exact supersession, +or its verified pending terminal-receipt repair. A conditional terminal effect after writeback and spend uses the existing journal/settlement owner and exact effect reference; an active proof may execute that remaining original effect, while a canonical completed Todo permits only receipt repair. Committed @@ -116,6 +117,18 @@ validated causal links retain the original proof even on older revisions without a revision marker. Direct receipt publication permits exact replay and rejects any change to already prepared fields before appending a receipt. +An explicit `semantic_closeout` uses a separate TS writeback qualification. It +checks the actual Run, original Todo/Turn/effect and Goal owner, current canonical +generation, typed progress, Vision and checkpoint. Exact supersession permits +only that original Turn's semantic writeback; business resume still rejects it. +A Todo-bound replan with no business lease must prove its original selected +obligation and actual semantic evidence. Required preparation without a positive +original proof is rejected. Qualification returns only the durable-writeback +effect scope, never an execution lease. The supported writer rereads inside the +shared maintenance marker and holds Goal/source ownership and the marker through +Run JSON, Markdown and index durability. Direct provider writes bypassing that +marker are outside this canonical writer guarantee. + `refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / semantic_closeout 证据及当前 control writer 的确切租约 key/version。TS owner 在真实 @@ -141,7 +154,8 @@ key 与 epoch 保持相同,version 不降低。Legacy exact-proof 兼容保留 不提供 ABA 保证。不同 generation 不能完成旧 Turn,即便已有 frozen report; 恢复使用 execution-loss closeout 或既有 C1 owner 下的 fresh Turn。每个 effect 边界均重新读取对应 owner 的 authority;retired proof 只支持原 Turn 已证明 completion 后尚余的 -writeback 或已核验的 pending terminal receipt 修复。writeback 与 spend 后的条件 +writeback、exact supersede 后独立核验的 semantic writeback,或已核验的 pending +terminal receipt 修复。writeback 与 spend 后的条件 terminal effect 复用既有 journal/settlement owner 及 exact effect ref:active 原 generation proof 只能执行该剩余 effect;canonical Todo 已完成时只能修复收据。 已提交重放直接返回原结果,不请求租约或执行 effect。durable writeback 已被接受后, @@ -156,6 +170,16 @@ prepared generation 均拒绝;旧收据没有 revision 标记时,已核验 原 proof。Direct receipt publication 只允许 exact replay,任何已准备字段变化都 必须在追加收据前拒绝。 +显式 `semantic_closeout` 使用独立 TS writeback qualification,核验实际 Run、原 +Todo/Turn/effect、Goal owner、当前 canonical generation、typed progress、Vision +与 checkpoint。Exact supersede 只允许原 Turn 的 semantic writeback,business +resume 仍拒绝。没有业务 lease 的 Todo-bound replan 须证明原 selected obligation +与实际语义证据;要求 preparation 却缺少 positive original proof 时拒绝。返回值 +仅为 durable-writeback effect scope,不授予 execution lease。受支持 writer 在共享 +maintenance marker 内重新读取,并持有 Goal/source ownership 与 marker 直到 Run +JSON、Markdown、index 持久化完成。绕过 marker 的直接 provider 写不属于此 +canonical writer 保证。 + Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. User task time requirements remain task context. Caller-private deadlines diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index aae5e6cb53..6ccc42b8d9 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -287,6 +287,46 @@ def read_turn_execution_lease_qualification( return readback +def read_turn_writeback_qualification( + *, runtime_root: Path, registry_path: Path, identity: SettlementIdentity, + candidate: Mapping[str, Any], refresh_retry: Mapping[str, Any] | None, + goal_ref: Mapping[str, Any] | None = None, + source_admission: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Transport the actual Run to the independent TS writeback owner. + + Callers retain the shared maintenance/source lifetime through commit. + This result names one writeback operation and supplies no execution lease. + """ + from .settlement import read_heartbeat_settlement + + readback = read_heartbeat_settlement( + runtime_root, goal_id=identity.goal_id, agent_id=identity.agent_id, + todo_id=identity.todo_id, turn_instance_id=identity.turn_instance_id, + replan_obligation_id=identity.replan_obligation_id, + registry_path=registry_path, goal_ref=goal_ref, + source_admission=source_admission, + borrow_source_admission=source_admission is not None, + refresh_retry=dict(refresh_retry) if refresh_retry is not None else None, + writeback_candidate=dict(candidate), + ) + qualification = readback.writeback_qualification if readback is not None else None + if qualification is None: + raise RefreshTaskLeaseUnavailable("The original Turn's writeback scope could not be qualified.") + if qualification.failure is not None: + raise RefreshTaskLeaseUnavailable(qualification.failure.reason) + scope = qualification.value + if ( + not isinstance(scope, dict) + or set(scope) != {"step_kind", "effect_ref", "delivery_boundary", "original_prepared", "selected_obligation_id"} + or scope.get("step_kind") != "durable_writeback" + or scope.get("effect_ref") != f"{identity.effect_id}#durable_writeback" + or scope.get("delivery_boundary") != "semantic_closeout" + ): + raise RefreshTaskLeaseUnavailable("The qualified scope does not describe this Turn's semantic writeback.") + return scope + + def prepared_lease_receipt_fields( proof: Mapping[str, Any], *, lease_epoch: int | None = None, ) -> dict[str, Any]: @@ -328,10 +368,12 @@ def guard_first_writeback_execution( source_admission: Mapping[str, Any] | None = None, allow_same_generation_lease_renewal: bool = False, validation_recovery_turn_instance_id: str | None = None, + writeback_candidate: Mapping[str, Any] | None = None, + refresh_retry: Mapping[str, Any] | None = None, ) -> Iterator[None]: prepared_lease = settlement.prepared_lease prepared_required = settlement.prepared_lease_required or prepared_lease is not None - if (requested_proof is None and not prepared_required + if (writeback_candidate is None and requested_proof is None and not prepared_required and not local_authority_is_promoted(runtime_root=runtime_root, goal_id=goal_id)): yield return @@ -345,8 +387,20 @@ def guard_first_writeback_execution( operation="refresh-state-execution", )) require_shadow_primary_write_allowed(runtime_root, goal_id) - hard_lease_required = prepared_required - if prepared_required: + goal_lifetime_required = prepared_required + if writeback_candidate is not None: + identity = settlement.identity.value + if identity is None: + raise RefreshTaskLeaseUnavailable("The original Turn's settlement identity is unavailable.") + # Always read inside M. Never consume a qualification carried by + # the caller's earlier settlement snapshot. + read_turn_writeback_qualification( + runtime_root=runtime_root, registry_path=registry_path, + identity=identity, candidate=writeback_candidate, refresh_retry=refresh_retry, + goal_ref=goal_ref, source_admission=source_admission, + ) + goal_lifetime_required = True + elif prepared_required: identity = settlement.identity.value if identity is None: raise RefreshTaskLeaseUnavailable("The original Turn's settlement identity is unavailable.") @@ -363,8 +417,8 @@ def guard_first_writeback_execution( snapshot = _read_execution_snapshot(runtime_root, goal_id, todo_id) if snapshot is None: raise RefreshTaskLeaseUnavailable("The selected Todo's execution authority could not be read.") - hard_lease_required = snapshot.get("handoff_mode") == "hard_lease" - if hard_lease_required: + goal_lifetime_required = snapshot.get("handoff_mode") == "hard_lease" + if goal_lifetime_required: todo_open = snapshot["todo"].get("status") == "open" facts = _execution_facts(snapshot, goal_id, todo_id, agent_id) current = _project_execution_facts({**facts, "include_retired_lease_for_writeback": not todo_open}) @@ -381,7 +435,7 @@ def guard_first_writeback_execution( "The original writeback needs a current task lease recorded for this Turn. " "Read the same Todo and Turn through its current guard before retrying." ) - if hard_lease_required: + if goal_lifetime_required: goal = locked_goal if goal is None: source, _ = load_recommendation_source_goal(registry_path, goal_id) diff --git a/loopx/control_plane/quota/heartbeat_receipt_identity.ts b/loopx/control_plane/quota/heartbeat_receipt_identity.ts index 026c50b1ff..8491632a1b 100644 --- a/loopx/control_plane/quota/heartbeat_receipt_identity.ts +++ b/loopx/control_plane/quota/heartbeat_receipt_identity.ts @@ -382,6 +382,39 @@ export function heartbeatValidationEvent( receipt: JsonObject, identity: SettlementIdentity, allowLegacyPrepared = false, +): JsonObject | null { + return selectHeartbeatValidationEvent(events, receipt, identity, allowLegacyPrepared, false); +} + +/** A semantic admission remains bound to its original Todo and obligation; + * its report permission supplies no business execution lease. */ +function heartbeatSemanticWorkAdmission(event: JsonObject, identity: SettlementIdentity): string | null { + const fact = heartbeatReceiptFactFromEvent(event), receiptDetails = heartbeatReceiptDetails(event); + const binding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); + const selected = normalizeHeartbeatReplanObligationId(receiptDetails.semantic_replan_obligation_id); + if (selected === null || event.schema_version !== "loopx_rollout_event_v0" + || event.event_kind !== "quota_should_run" || event.goal_id !== identity.goal_id + || event.agent_id !== identity.agent_id || fact.run_id !== identity.turn_instance_id + || fact.todo_id !== identity.todo_id || binding?.binding_kind !== "todo" + || binding.binding_id !== identity.todo_id || fact.closeout_required !== true + || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id)) + || receiptDetails.interaction_mode !== "autonomous_replan" || receiptDetails.delivery_allowed !== false + || receiptDetails.ok !== true || receiptDetails.should_run !== true + || receiptDetails.must_attempt_work !== true || receiptDetails.quiet_noop_allowed !== false + || nativeChildReportAdmission(receiptDetails, event.status, identity.effect_id, "open", false, selected) + .report_permission !== "new_operation") return null; + return selected; +} + +export function heartbeatSemanticAdmissionEvent(events: readonly JsonObject[], receipt: JsonObject, + identity: SettlementIdentity, allowLegacyPrepared = false): JsonObject | null { + const selected = selectHeartbeatValidationEvent(events, receipt, identity, allowLegacyPrepared, true); + return selected !== null && heartbeatSemanticWorkAdmission(selected, identity) !== null ? selected : null; +} + +function selectHeartbeatValidationEvent( + events: readonly JsonObject[], receipt: JsonObject, identity: SettlementIdentity, + allowLegacyPrepared: boolean, semanticAdmission: boolean, ): JsonObject | null { const scoped = events.filter(event => event.event_kind === "quota_should_run" && event.goal_id === identity.goal_id && event.agent_id === identity.agent_id @@ -402,7 +435,10 @@ export function heartbeatValidationEvent( && !effectIdsMatch(initialFact.settlement_effect_id, identity.effect_id)))) return null; const generations = new Set(); const positivePrepared = new Map(); + const semanticObligations = new Set(); for (const [index, event] of scoped.entries()) { + const semantic = semanticAdmission ? heartbeatSemanticWorkAdmission(event, identity) : null; + if (semantic !== null) semanticObligations.add(semantic); const prepared = heartbeatPreparedWorkAdmission(event, identity, allowLegacyPrepared); if (prepared === null) continue; positivePrepared.set(index, prepared); @@ -412,12 +448,15 @@ export function heartbeatValidationEvent( } if (generations.size > 1) throw receiptIdentityConflict( "one Turn cannot contain positive prepared admissions from different lease generations"); + if (semanticObligations.size > 1) throw receiptIdentityConflict( + "one Turn cannot replace its admitted semantic replan obligation"); let position = start[0]!; let validation = scoped[position]!; let preparedAdmission: JsonObject | null = null; while (position >= 0) { const prepared = positivePrepared.get(position); - const positive = prepared !== undefined; + const positive = semanticAdmission ? heartbeatSemanticWorkAdmission(validation, identity) !== null + : prepared !== undefined; if (positive) preparedAdmission = validation; const validationDetails = heartbeatReceiptDetails(validation); const revision = validationDetails.settlement_receipt_revision; @@ -425,7 +464,7 @@ export function heartbeatValidationEvent( && positive && Object.hasOwn(validation, "causality"); if (revision !== "lease_readiness" && revision !== "work_admission" && revision !== "identity_upgrade" && revision !== "review_guard_upgrade" && !legacyLineage) { - if (positive && !Object.hasOwn(validationDetails, "settlement_receipt_revision") + if (!semanticAdmission && positive && !Object.hasOwn(validationDetails, "settlement_receipt_revision") && !Object.hasOwn(validation, "causality")) { let first: JsonObject | null = null; for (const [index, priorPrepared] of positivePrepared) { @@ -451,7 +490,8 @@ export function heartbeatValidationEvent( if ((parentBinding !== null && (parentBinding.binding_kind !== identity.binding_kind || parentBinding.binding_id !== identity.binding_id)) || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id))) return null; - if (heartbeatPreparedWorkAdmission(parent, identity, allowLegacyPrepared) === null) return preparedAdmission; + if (semanticAdmission ? heartbeatSemanticWorkAdmission(parent, identity) === null + : heartbeatPreparedWorkAdmission(parent, identity, allowLegacyPrepared) === null) return preparedAdmission; } const pendingParent = revision === "identity_upgrade" && identity.binding_kind === "todo" && fact.todo_id === null && fact.replan_obligation_id === null && fact.request_subject == null @@ -469,23 +509,20 @@ export function heartbeatValidationEvent( /** Ephemeral qualification over freshly loaded canonical facts. This never * publishes a new prepared proof or grants a replacement generation work. */ -export function qualifyHeartbeatResumeLease(input: { +interface HeartbeatPreparedGenerationInput { originalReceipt: JsonObject; identity: SettlementIdentity; requestedProof: JsonObject | null; todo: JsonObject; lease: JsonObject; handoffMode: unknown; - completionEvent: JsonObject | null; - replayPhase: ReceiptBoundReplayPhase; - closeoutStarted: boolean; - writebackPending: boolean; - otherTerminalStarted: boolean; - terminalProviderEffect?: ProviderEffect | null; - terminalCompletion?: JsonObject | null; exactSource: boolean; - observedAt: Date; -}): JsonObject { +} + +/** Shared generation proof; phase and operation permission stay with callers. */ +export function qualifyHeartbeatPreparedGeneration(input: HeartbeatPreparedGenerationInput): { + original: HeartbeatPreparedLease; currentProof: JsonObject; +} { function reject(message: string): never { throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); } @@ -506,21 +543,35 @@ export function qualifyHeartbeatResumeLease(input: { if (input.requestedProof !== null && !isDeepStrictEqual(input.requestedProof, currentProof)) { reject("The supplied resume proof does not match the current canonical lease."); } - const keyDigest = createHash("sha256").update(lease.idempotency_key).digest("hex"); - if (keyDigest !== original.prepared_lease_key_sha256) { + if (createHash("sha256").update(lease.idempotency_key).digest("hex") !== original.prepared_lease_key_sha256) { reject("A replacement lease cannot execute this Turn's original work."); } if (original.prepared_lease_epoch === undefined) { - // Preserve the existing legacy exact-version contract, without claiming - // incarnation/ABA protection or allowing an unrecorded renewal/rebind. if (input.exactSource || lease.version !== original.prepared_lease_version) { reject("This prepared receipt cannot prove a renewed lease generation."); } } else if (typeof lease.lease_epoch !== "number" || !Number.isSafeInteger(lease.lease_epoch) - || lease.lease_epoch !== original.prepared_lease_epoch - || lease.version < original.prepared_lease_version) { + || lease.lease_epoch !== original.prepared_lease_epoch || lease.version < original.prepared_lease_version) { reject("The current lease does not preserve the original prepared generation."); } + return {original, currentProof}; +} + +export function qualifyHeartbeatResumeLease(input: HeartbeatPreparedGenerationInput & { + completionEvent: JsonObject | null; + replayPhase: ReceiptBoundReplayPhase; + closeoutStarted: boolean; + writebackPending: boolean; + otherTerminalStarted: boolean; + terminalProviderEffect?: ProviderEffect | null; + terminalCompletion?: JsonObject | null; + observedAt: Date; +}): JsonObject { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const {original, currentProof} = qualifyHeartbeatPreparedGeneration(input); + const {todo, lease, identity} = input; const terminalPending = input.terminalProviderEffect?.step_kind === "terminal_closeout" && input.terminalProviderEffect.effect_ref === `${identity.effect_id}#terminal_closeout` && ["prepare_and_execute", "resolve_prepared", "execute_prepared"].includes(input.terminalProviderEffect.action); diff --git a/loopx/control_plane/quota/settlement.py b/loopx/control_plane/quota/settlement.py index 02a57f8ded..396624e98e 100644 --- a/loopx/control_plane/quota/settlement.py +++ b/loopx/control_plane/quota/settlement.py @@ -192,6 +192,7 @@ class QuotaSettlementReadback: original_prepared_event_id: str | None = None original_prepared_receipt_sha256: str | None = None native_task_lease_cleanup: dict[str, Any] | None = None + writeback_qualification: SettlementResult[dict[str, Any]] | None = None def render_owed_host_command(argv: list[str]) -> str: @@ -508,6 +509,7 @@ def read_heartbeat_settlement( heartbeat_reentry_guard: dict[str, Any] | None = None, resume_lease_candidate: Mapping[str, Any] | None = None, native_task_lease_cleanup: Mapping[str, Any] | None = None, + writeback_candidate: Mapping[str, Any] | None = None, include_record_bodies: bool = True, tolerate_malformed_history: bool = False, ) -> QuotaSettlementReadback | None: @@ -552,6 +554,11 @@ def read_heartbeat_settlement( if native_task_lease_cleanup is not None else {} ), + **( + {"writeback_candidate": dict(writeback_candidate)} + if writeback_candidate is not None + else {} + ), }, registry_path=registry_path, goal_ref=goal_ref, @@ -617,4 +624,8 @@ def read_heartbeat_settlement( original_prepared_event_id=_optional_readback_text(payload.get("original_prepared_event_id")), original_prepared_receipt_sha256=_optional_readback_text(payload.get("original_prepared_receipt_sha256")), native_task_lease_cleanup=_optional_readback_record(payload.get("native_task_lease_cleanup")), + writeback_qualification=( + _readback_result(payload["writeback_qualification"]) + if "writeback_qualification" in payload else None + ), ) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index fc6a0f718a..621c851620 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -15,6 +15,7 @@ import { settlementFailed, executionRequestSubject, settlementIdentity, settlementIdentityPayload, + settlementIdentityFromPlan, settlementPure, settlementReceipt, settlementResultPayload, @@ -72,6 +73,8 @@ import { heartbeatPreparedLease, heartbeatPreparedWorkAdmission, heartbeatValidationEvent, + heartbeatSemanticAdmissionEvent, + qualifyHeartbeatPreparedGeneration, qualifyHeartbeatResumeLease, type HeartbeatReceiptFact, } from "./heartbeat_receipt_identity.ts"; @@ -168,11 +171,24 @@ interface OwedPriorTurn extends PriorHostTurnCloseoutCandidate { missing_receipts: string[]; } -interface ResumeLeaseCandidate { +interface ExecutionQualificationHints { + allow_same_generation_lease_renewal?: boolean; + validation_recovery_turn_instance_id?: string | null; +} + +interface ResumeLeaseCandidate extends ExecutionQualificationHints { requested_proof: JsonObject | null; provider_context?: {turn_key: string; effect_ref: string | null}; } +interface WritebackCandidate extends ExecutionQualificationHints { + requested_proof: JsonObject | null; + effect_ref: string; + completion_todo_id: string | null; + completion_turn_key: string | null; + run: JsonObject; +} + type OwedPriorTurnLookup = (agentId: string, turnInstanceId: string) => OwedPriorTurn | null; const TURN_INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; @@ -195,6 +211,7 @@ interface ReadbackRequest { refresh_retry: RefreshRetryRequest | null; heartbeat_reentry_guard: JsonObject | null; resume_lease_candidate: ResumeLeaseCandidate | null; + writeback_candidate: WritebackCandidate | null; native_task_lease_cleanup: NativeTaskLeaseCleanupRequest | null; owner: QuotaAccountingOwner; } @@ -229,10 +246,23 @@ interface SpendTerms { function decodeResumeLeaseCandidate(value: unknown): ResumeLeaseCandidate | null { if (value === undefined) return null; const candidate = requireJsonObject(value, "resume_lease_candidate"); - if (Object.keys(candidate).some(key => key !== "requested_proof" && key !== "provider_context") + if (Object.keys(candidate).some(key => !["requested_proof", "provider_context", + "allow_same_generation_lease_renewal", "validation_recovery_turn_instance_id"].includes(key)) || !Object.hasOwn(candidate, "requested_proof")) { throw new EffectRuntimeRequestError("resume_lease_candidate requires only requested_proof and optional provider_context"); } + const hints: ExecutionQualificationHints = {}; + if (Object.hasOwn(candidate, "allow_same_generation_lease_renewal")) { + hints.allow_same_generation_lease_renewal = requireBoolean(candidate.allow_same_generation_lease_renewal, + "allow_same_generation_lease_renewal"); + } + if (Object.hasOwn(candidate, "validation_recovery_turn_instance_id")) { + const recovery = candidate.validation_recovery_turn_instance_id; + if (recovery !== null && (typeof recovery !== "string" || !recovery || recovery !== recovery.trim())) { + throw new EffectRuntimeRequestError("validation_recovery_turn_instance_id requires an exact Turn or null"); + } + hints.validation_recovery_turn_instance_id = recovery; + } let providerContext: ResumeLeaseCandidate["provider_context"]; if (Object.hasOwn(candidate, "provider_context")) { const context = requireJsonObject(candidate.provider_context, "resume provider_context"); @@ -244,7 +274,7 @@ function decodeResumeLeaseCandidate(value: unknown): ResumeLeaseCandidate | null } providerContext = {turn_key: context.turn_key, effect_ref: context.effect_ref}; } - const contextFields = providerContext === undefined ? {} : {provider_context: providerContext}; + const contextFields = {...hints, ...(providerContext === undefined ? {} : {provider_context: providerContext})}; if (candidate.requested_proof === null) return {requested_proof: null, ...contextFields}; const proof = requireJsonObject(candidate.requested_proof, "requested resume proof"); if (Object.keys(proof).length !== 2 || typeof proof.idempotency_key !== "string" @@ -256,6 +286,36 @@ function decodeResumeLeaseCandidate(value: unknown): ResumeLeaseCandidate | null return {requested_proof: proof, ...contextFields}; } +function decodeWritebackCandidate(value: unknown): WritebackCandidate | null { + if (value === undefined) return null; + const candidate = requireJsonObject(value, "writeback_candidate"); + if (Object.keys(candidate).some(key => !["requested_proof", "effect_ref", "completion_todo_id", + "completion_turn_key", "run", "allow_same_generation_lease_renewal", + "validation_recovery_turn_instance_id"].includes(key))) { + throw new EffectRuntimeRequestError("writeback_candidate contains an unsupported operation field"); + } + const proof = decodeResumeLeaseCandidate({requested_proof: candidate.requested_proof, + ...(Object.hasOwn(candidate, "allow_same_generation_lease_renewal") + ? {allow_same_generation_lease_renewal: candidate.allow_same_generation_lease_renewal} : {}), + ...(Object.hasOwn(candidate, "validation_recovery_turn_instance_id") + ? {validation_recovery_turn_instance_id: candidate.validation_recovery_turn_instance_id} : {})})!; + const nullableExactString = (key: string): string | null => { + if (candidate[key] === null) return null; + const value = requireNonEmptyString(candidate[key], `writeback_candidate.${key}`); + if (value !== value.trim()) throw new EffectRuntimeRequestError(`writeback_candidate.${key} must be exact`); + return value; + }; + const run = requireJsonObject(candidate.run, "writeback_candidate.run"); + if (Object.keys(run).some(key => !["goal_id", "agent_id", "todo_id", "turn_instance_id", + "settlement_identity", "goal_ref", "classification", "delivery_outcome", "delivery_batch_scale", "delivery_boundary", + "progress_observation", "agent_vision", "vision_checkpoint", "autonomous_replan_ack", "refresh_recovery"].includes(key))) { + throw new EffectRuntimeRequestError("writeback_candidate.run contains an unsupported payload field"); + } + return {...proof, effect_ref: requireNonEmptyString(candidate.effect_ref, "writeback_candidate.effect_ref"), + completion_todo_id: nullableExactString("completion_todo_id"), + completion_turn_key: nullableExactString("completion_turn_key"), run}; +} + const ORIGINAL_TURN_SPEND_SLOTS = 1; type SettlementProgressState = "identity_required" | "writeback_required" | @@ -416,12 +476,31 @@ function decodeRequest( || optionalRequestString(request.turn_instance_id, "turn_instance_id") === null)) { throw new EffectRuntimeRequestError("resume lease qualification requires an exact Agent/Todo/Turn readback"); } + const writebackCandidate = decodeWritebackCandidate(request.writeback_candidate); + if (writebackCandidate !== null && (resumeCandidate !== null || request.infer_turn_instance_id + || request.allow_unbound_binding || request.resolve_original_binding || request.pending_closeout + || request.refresh_retry == null || request.heartbeat_reentry_guard != null || request.spend_precheck === true + || request.tolerate_malformed_history === true + || optionalRequestString(request.agent_id, "agent_id") === null + || optionalRequestString(request.todo_id, "todo_id") === null + || optionalRequestString(request.turn_instance_id, "turn_instance_id") === null + || optionalRequestString(request.replan_obligation_id, "replan_obligation_id") !== null)) { + throw new EffectRuntimeRequestError("writeback qualification requires only an explicit original Todo operation and payload"); + } + if (writebackCandidate !== null) { + const retry = requireJsonObject(request.refresh_retry, "writeback refresh_retry"); + if (Object.keys(retry).some(key => !["checkpoint_read_context_id", "external_delivery", "vision", + "unchanged_reason", "merge_patch", "workspace_requested", "mutation", "delivery_outcome", + "delivery_batch_scale", "delivery_boundary", "progress_observation"].includes(key))) { + throw new EffectRuntimeRequestError("writeback refresh_retry contains an unsupported operation field"); + } + } const nativeCleanup = request.native_task_lease_cleanup === undefined ? null : decodeNativeTaskLeaseCleanup(request.native_task_lease_cleanup); if (nativeCleanup !== null && (request.infer_turn_instance_id || request.allow_unbound_binding || request.resolve_original_binding || request.pending_closeout || request.refresh_retry != null || request.heartbeat_reentry_guard != null || request.spend_precheck === true - || request.tolerate_malformed_history === true || resumeCandidate !== null + || request.tolerate_malformed_history === true || resumeCandidate !== null || writebackCandidate !== null || !optionalRequestString(request.agent_id, "agent_id") || !optionalRequestString(request.todo_id, "todo_id") || !optionalRequestString(request.turn_instance_id, "turn_instance_id") @@ -452,6 +531,7 @@ function decodeRequest( heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null : requireJsonObject(request.heartbeat_reentry_guard, "heartbeat_reentry_guard"), resume_lease_candidate: resumeCandidate, + writeback_candidate: writebackCandidate, native_task_lease_cleanup: nativeCleanup, owner, }; @@ -1796,7 +1876,7 @@ export function readQuotaSettlementFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value); - if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); if (request.owner.kind !== "alias") { throw new EffectRuntimeRequestError( @@ -1813,7 +1893,7 @@ export function readAdmittedQuotaSettlementFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value); - if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); } @@ -1825,12 +1905,14 @@ export function readQuotaSettlementForAdmittedOwnerFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value, owner); - if (request.resume_lease_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); } -async function readResumeLeaseQualification(request: ReadbackRequest, result: JsonObject): Promise { +async function readSettlementExecutionAuthority(request: ReadbackRequest, result: JsonObject): Promise<{ + identity: SettlementIdentity; todo: JsonObject; lease: JsonObject | undefined; handoffMode: unknown; +}> { function reject(message: string): never { throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); } @@ -1839,12 +1921,16 @@ async function readResumeLeaseQualification(request: ReadbackRequest, result: Js if (result.found !== true || identityResult?.failure != null || identityValue === null) { reject("The original Turn's settlement identity could not be verified."); } - const originalReceipt = jsonObject(result.original_prepared_receipt); const identity = settlementIdentity({goal_id: String(identityValue.goal_id), agent_id: String(identityValue.agent_id), todo_id: optionalString(identityValue.todo_id), turn_instance_id: String(identityValue.turn_instance_id), replan_obligation_id: optionalString(identityValue.replan_obligation_id), request_subject: executionRequestSubject(identityValue.request_subject)}); + const hint = request.resume_lease_candidate ?? request.writeback_candidate; + if (hint?.validation_recovery_turn_instance_id != null + && hint.validation_recovery_turn_instance_id !== identity.turn_instance_id) { + reject("Recovery Turn hint differs from the original settlement identity."); + } // The ordinary reader does not load lease decision owners. Only this // explicitly requested ephemeral branch reads the real provider authority, // inside the enclosing quota/source admission and without claiming a lease. @@ -1869,8 +1955,17 @@ async function readResumeLeaseQualification(request: ReadbackRequest, result: Js if (observed.status !== "available" || observed.store_identity !== snapshot.identity.store_identity) { reject("Canonical authority changed during resume qualification."); } + return {identity, todo, lease, handoffMode: authority.head.handoff_mode}; +} + +async function readResumeLeaseQualification(request: ReadbackRequest, result: JsonObject): Promise { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const {identity, todo, lease, handoffMode} = await readSettlementExecutionAuthority(request, result); + const originalReceipt = jsonObject(result.original_prepared_receipt); if (originalReceipt === null) { - if (request.owner.kind === "exact_source" || authority.head.handoff_mode !== "soft_claim" + if (request.owner.kind === "exact_source" || handoffMode !== "soft_claim" || result.prepared_lease_required === true || result.prepared_lease != null) { reject("The original Turn's positive prepared admission is missing."); } @@ -1897,7 +1992,7 @@ async function readResumeLeaseQualification(request: ReadbackRequest, result: Js } return qualifyHeartbeatResumeLease({originalReceipt, identity, requestedProof: request.resume_lease_candidate!.requested_proof, todo, lease, - handoffMode: authority.head.handoff_mode, completionEvent: completion, + handoffMode, completionEvent: completion, replayPhase: result.replay_phase as ReceiptBoundReplayPhase, closeoutStarted: [result.writeback_run, result.writeback_event, result.spend_run, result.spend_event, result.completion_event].some(record => record != null), @@ -1908,6 +2003,206 @@ async function readResumeLeaseQualification(request: ReadbackRequest, result: Js exactSource: request.owner.kind === "exact_source", observedAt: new Date()}); } +/** A first semantic writeback has its own operation scope. Its result is not + * a task lease and is never consumed by host execution or Todo completion. */ +async function readWritebackQualification(request: ReadbackRequest, result: JsonObject, + snapshot: QuotaSettlementReadbackSnapshot): Promise { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const candidate = request.writeback_candidate!, run = candidate.run; + const {identity, todo, lease, handoffMode} = await readSettlementExecutionAuthority(request, result); + if (identity.binding_kind !== "todo" || candidate.effect_ref !== `${identity.effect_id}#durable_writeback` + || candidate.completion_todo_id !== null || candidate.completion_turn_key !== null + || run.goal_id !== identity.goal_id || run.agent_id !== identity.agent_id + || run.todo_id !== identity.todo_id || run.turn_instance_id !== identity.turn_instance_id + || run.delivery_boundary !== "semantic_closeout" + || typeof run.classification !== "string" || !run.classification.trim()) { + reject("Writeback candidate does not describe this original Todo's semantic operation."); + } + const parsed = settlementIdentityFromPlan({settlement_plan: {identity: run.settlement_identity}}); + if (parsed.failure !== null || !isDeepStrictEqual(settlementIdentityPayload(parsed.value), settlementIdentityPayload(identity)) + || !quotaOwnerOwnsProjection(request.owner, run.goal_ref) + || (request.owner.kind === "alias" && Object.hasOwn(run, "goal_ref"))) { + reject("Writeback candidate crosses its original settlement or Goal ownership."); + } + const retry = request.refresh_retry!; + const progress = jsonObject(run.progress_observation); + if (run.progress_observation != null && (progress === null || progress.work_item_id !== identity.todo_id)) { + reject("Writeback progress must describe the original Todo, without substituting another work item."); + } + const recovery = jsonObject(result.refresh_recovery); + if (retry.external_delivery != null || retry.workspace_requested || retry.checkpoint_read_context_id != null + || retry.delivery_boundary !== "semantic_closeout" || retry.delivery_outcome !== run.delivery_outcome + || retry.delivery_batch_scale !== (run.delivery_batch_scale ?? null) + || !isDeepStrictEqual(retry.progress_observation, run.progress_observation ?? null) + || recovery?.decision !== "append" || recovery.reason !== "first_writeback" + || !isDeepStrictEqual(run.refresh_recovery, recovery) + || Object.keys(retry.mutation).some(key => !["next_action", "next_action_basis", "autonomous_replan_recorded", + "repair_delta_kinds", "usage_measurement", "usage_codex_session", "explore_result"].includes(key))) { + reject("Writeback candidate differs from its actual semantic payload or carries another operation."); + } + if ([result.writeback_run, result.writeback_event, result.spend_run, result.spend_event].some(record => record != null)) { + reject("Committed writeback and accounting use their existing replay owner, not a new writeback qualification."); + } + const originalReceipt = jsonObject(result.original_prepared_receipt); + if (originalReceipt === null && (result.prepared_lease_required === true || result.prepared_lease != null)) { + reject("The original Turn's required positive preparation is missing."); + } + const effective = jsonObject(result.heartbeat_receipt); + if (effective === null) reject("The original semantic receipt is unavailable."); + const events = indexedEvents(snapshot, request.goal_id, identity.agent_id, identity.turn_instance_id) + .filter(event => quotaOwnerOwnsProjection(request.owner, event.goal_ref)); + const scope = (original: JsonObject | null, selected: string | null): JsonObject => bundle(settlementPure({ + step_kind: "durable_writeback", effect_ref: candidate.effect_ref, delivery_boundary: "semantic_closeout", + original_prepared: original, selected_obligation_id: selected, + }, (jsonObject(result.identity)?.result as SettlementResult).receipts)); + const closeoutStarted = [result.completion_event, result.supersede_event].some(record => record != null); + let activePrepared: JsonObject | null = null; + if (originalReceipt !== null && lease !== undefined && !closeoutStarted) { + const qualified = qualifyHeartbeatResumeLease({originalReceipt, identity, requestedProof: candidate.requested_proof, + todo, lease, handoffMode, completionEvent: null, replayPhase: result.replay_phase as ReceiptBoundReplayPhase, + closeoutStarted: false, writebackPending: true, otherTerminalStarted: false, + exactSource: request.owner.kind === "exact_source", observedAt: new Date()}); + activePrepared = jsonObject(qualified.original_prepared); + } + const [{buildVisionCheckpoint, VISION_REFRESH_REQUEST_SCHEMA, qualifiesOpenTodoSemanticCheckpoint}, + {projectExecutionPreparation, EXECUTION_PREPARATION_FACTS_SCHEMA_VERSION}, {leaseIsActive}, {projectProgressDelta}] = await Promise.all([ + import("../goals/vision_checkpoint.ts"), import("../work_items/execution_preparation.ts"), + import("../work_items/task_lease_record.ts"), import("../work_items/progress_delta.ts"), + ]); + const ownerRuns = snapshot.runs.filter(row => row.goal_id === identity.goal_id && row.agent_id === identity.agent_id + && quotaOwnerOwnsProjection(request.owner, row.goal_ref)); + const progressSource = progress === null ? null : {current: progress, baseline: null, + window: ownerRuns.map(row => jsonObject(row.progress_observation)).filter(value => value !== null)}; + // Reuse the typed observation codec for every semantic branch. Its outcome + // qualification remains with the existing replan owner below. + if (progressSource !== null) projectProgressDelta(progressSource); + const existingVision = [...ownerRuns].reverse().map(row => jsonObject(row.agent_vision)).find(value => value !== null) ?? null; + let vision: JsonObject | null = null; + if (run.agent_vision != null) { + vision = jsonObject(buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "prepare", + goal_id: identity.goal_id, agent_id: identity.agent_id, agent_vision_packet: run.agent_vision, + existing_agent_vision: null, merge_patch: false, require_path_delta_for_durable_change: true, + allow_terminal_stride_path: false}).agent_vision); + } + if (retry.vision === null && vision !== null) { + reject("Writeback cannot introduce a Vision absent from its actual request."); + } + if (retry.vision !== null) { + const expected = jsonObject(buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "prepare", + goal_id: identity.goal_id, agent_id: identity.agent_id, agent_vision_packet: retry.vision, + existing_agent_vision: existingVision, merge_patch: retry.merge_patch, + require_path_delta_for_durable_change: true, allow_terminal_stride_path: false}).agent_vision); + if (expected === null || vision === null || Object.keys(expected).some(key => key !== "generated_at" + && !isDeepStrictEqual(expected[key], vision![key]))) reject("Writeback Vision differs from its actual request."); + } + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: identity.agent_id, agent_vision: vision, existing_agent_vision: existingVision, + vision_unchanged_reason: retry.unchanged_reason, delivery_outcome: run.delivery_outcome, + active_state_next_action_would_update: retry.mutation.next_action != null, + delivery_boundary: "semantic_closeout", todo_id: identity.todo_id, completion_todo_id: null, + autonomous_replan_recorded: retry.mutation.autonomous_replan_recorded === true, blocked_retry: null}); + const offeredCheckpoint = jsonObject(run.vision_checkpoint); + if (checkpoint.satisfied !== true || offeredCheckpoint === null + || ["schema_version", "agent_id", "required", "satisfied", "decision", "triggers", + "delivery_boundary", "agent_vision_state"].some(key => !isDeepStrictEqual(offeredCheckpoint[key], checkpoint[key]))) { + reject("Writeback checkpoint was not qualified from its actual semantic inputs."); + } + const qualifiedRun = {...run, settlement_identity: settlementIdentityPayload(identity), agent_vision: vision, + vision_checkpoint: checkpoint}; + // Prepared execution authority alone cannot validate semantic writeback. + // The same actual Vision, progress identity and checkpoint checks above + // apply before returning this operation's scope. + if (activePrepared !== null) return scope(activePrepared, null); + const supersede = jsonObject(result.supersede_event); + if (originalReceipt !== null && supersede !== null && result.completion_event == null + && todo.status === "done" && lease?.status === "released") { + const proof = qualifyHeartbeatPreparedGeneration({originalReceipt, identity, todo, lease, handoffMode, + requestedProof: candidate.requested_proof, exactSource: request.owner.kind === "exact_source"}); + if (supersede.event_kind !== "todo_supersede" || supersede.status !== "done" + || supersede.goal_id !== identity.goal_id || supersede.agent_id !== identity.agent_id + || supersede.run_id !== identity.turn_instance_id || supersede.todo_id !== identity.todo_id + || !effectIdsMatch(optionalString(details(supersede).settlement_effect_id), identity.effect_id) + || !qualifiesOpenTodoSemanticCheckpoint(qualifiedRun, identity)) { + reject("Retired writeback requires exact supersession and a fresh nonterminal semantic checkpoint."); + } + return scope({...proof.original}, null); + } + const completion = jsonObject(result.completion_event), receiptDetails = details(effective); + if (request.owner.kind === "alias" && originalReceipt === null && result.prepared_lease_required !== true + && !["prepared_lease_key_sha256", "prepared_lease_version", "prepared_lease_epoch", "prepared_lease_required", + "must_attempt_work", "delivery_allowed", "quiet_noop_allowed", "settlement_receipt_revision"] + .some(key => Object.hasOwn(receiptDetails, key)) + && todo.status === "done" && handoffMode === "hard_lease" && lease?.status === "released" + && lease.owner === identity.agent_id && (todo.claimed_by == null || todo.claimed_by === identity.agent_id) + && jsonObject(result.native_child_admission)?.report_permission === "existing_operation_only" + && completion?.event_kind === "todo_complete" && completion.goal_id === identity.goal_id + && completion.agent_id === identity.agent_id && completion.run_id === identity.turn_instance_id + && (completion.todo_id == null || completion.todo_id === identity.todo_id) + && effectIdsMatch(optionalString(details(completion).settlement_effect_id), identity.effect_id) + && qualifiesOpenTodoSemanticCheckpoint(qualifiedRun, identity)) { + const legacy = projectExecutionPreparation({schema_version: EXECUTION_PREPARATION_FACTS_SCHEMA_VERSION, + goal_id: identity.goal_id, agent_id: identity.agent_id, handoff_mode: handoffMode, authority_available: true, + selected_todo: todo, include_retired_lease_for_writeback: true, + leases: [{status: lease.status, active: false, owner: lease.owner, + lease_proof: {idempotency_key: lease.idempotency_key, expected_version: lease.version}, lease_epoch: lease.lease_epoch}]}); + const currentProof = jsonObject(legacy.retired_lease_proof); + if (currentProof === null || (candidate.requested_proof !== null + && !isDeepStrictEqual(candidate.requested_proof, currentProof))) { + reject("Legacy semantic writeback requires its exact current retired holder."); + } + // Historical absence proves no incarnation. Return only this writeback's + // scope, never the observed epoch or a business execution lease. + return scope(null, null); + } + const semanticReceipt = heartbeatSemanticAdmissionEvent(events, effective, identity, request.owner.kind === "alias"); + const guard = semanticReceipt === null ? null : projectSemanticReplanGuard(details(semanticReceipt)); + const selected = optionalString(guard?.selected_obligation_id); + if (semanticReceipt === null || guard?.scope !== "turn_guard" || selected === null + || selected !== optionalString(jsonObject(result.semantic_replan_guard)?.selected_obligation_id) + || guard.selected_capability_guard != null || todo.status !== "open") { + reject("Semantic writeback needs the original typed replan admission and selected obligation."); + } + if (originalReceipt !== null) reject("Semantic writeback cannot replace this Turn's prepared business generation."); + const currentProof = lease === undefined ? null : {idempotency_key: lease.idempotency_key, expected_version: lease.version}; + if (candidate.requested_proof !== null && !isDeepStrictEqual(candidate.requested_proof, currentProof)) { + reject("The supplied writeback proof does not match current canonical authority."); + } + const preparation = projectExecutionPreparation({schema_version: EXECUTION_PREPARATION_FACTS_SCHEMA_VERSION, + goal_id: identity.goal_id, agent_id: identity.agent_id, handoff_mode: handoffMode, authority_available: true, + selected_todo: todo, leases: lease === undefined ? [] : [{status: lease.status, + active: leaseIsActive(lease, new Date()), owner: lease.owner, lease_proof: currentProof, lease_epoch: lease.lease_epoch}]}); + if (preparation.state !== "ready" && preparation.state !== "claim_required" && preparation.state !== "not_required") { + reject("The original semantic Todo's current authority conflicts with this actor."); + } + if (candidate.requested_proof !== null && preparation.state !== "ready") { + reject("An offered semantic writeback proof requires current active authority."); + } + const [{projectReplanSettlementContract, REPLAN_SETTLEMENT_REQUEST_SCHEMA}, {projectReplanWriteback}] = await Promise.all([ + import("../work_items/replan_settlement.ts"), import("../work_items/replan_semantics.ts"), + ]); + const contract = projectReplanSettlementContract({schema_version: REPLAN_SETTLEMENT_REQUEST_SCHEMA, + selected_todo_id: identity.todo_id, semantic_replan_obligation_id: selected}); + if (contract.settlement_binding.id !== identity.todo_id || contract.semantic_obligation.discharge !== "todo_bound_writeback") { + reject("Semantic obligation does not belong to the original Todo settlement."); + } + const obligation = projectReplanWriteback({operation: "receipt_bound_obligation", guard_scoped: true, + selected_obligation_id: selected, agent_id: identity.agent_id, goal_id: identity.goal_id, + guard_recorded_at: semanticReceipt.recorded_at, guard_turn_instance_id: identity.turn_instance_id, + replay_phase: "open", current_obligation: null, receipt_review: guard.selected_review_source, + transition_candidates: []}).obligation; + const semantic = projectReplanWriteback({operation: "qualify", obligation, agent_vision: vision ?? {}, + progress_source: progressSource}); + const ack = jsonObject(run.autonomous_replan_ack), offered = jsonObject(ack?.semantic_delta); + if (semantic.accepted !== true || ack?.schema_version !== "autonomous_replan_ack_v0" || ack.recorded !== true + || offered === null || ["schema_version", "accepted", "outcomes", "satisfying_outcomes", "required_any_of", + "observation_fingerprint"].some(key => !isDeepStrictEqual(offered[key], semantic[key]))) { + reject("Semantic writeback payload does not satisfy its original replan source."); + } + return scope(null, selected); +} + export async function readQuotaSettlement(value: unknown): Promise { if (jsonObject(value)?.schema_version === "loopx_quota_history_qualification_request_v0") { const input = jsonObject(value)!, runs = input.runs, observedAt = input.observed_at; @@ -1935,7 +2230,7 @@ export async function readQuotaSettlement(value: unknown): Promise { throw new EffectRuntimeRequestError("spend_request requires spend_precheck, integer slots and a source"); } if (spendPrecheck) request.include_record_bodies = true; - if (request.resume_lease_candidate !== null) request.include_record_bodies = true; + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) request.include_record_bodies = true; const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner : withQuotaAccountingOwner; @@ -1943,16 +2238,19 @@ export async function readQuotaSettlement(value: unknown): Promise { request.owner, async () => { const matchedSpend: MatchedSpendRecords = {run: null, receipt: null}; - const result = readQuotaSettlementFromRequest(request, - await readQuotaSettlementSnapshot( + const snapshot = await readQuotaSettlementSnapshot( request.runtime_root, request.goal_id, undefined, request.tolerate_malformed_history, - ), true, matchedSpend); + ); + const result = readQuotaSettlementFromRequest(request, snapshot, true, matchedSpend); if (request.resume_lease_candidate !== null) { result.resume_lease_qualification = await readResumeLeaseQualification(request, result); } + if (request.writeback_candidate !== null) { + result.writeback_qualification = await readWritebackQualification(request, result, snapshot); + } const identity = jsonObject(jsonObject(jsonObject(result.identity)?.result)?.value); const receipt = spendReceiptTerms(matchedSpend.receipt); // Only exact readback opens its matched debit artifact. Check its diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index d485771983..13ff647431 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -880,6 +880,7 @@ def refresh_state_run( settlement_workspace_requirement = None settlement_readback = None refresh_recovery = None + refresh_retry_request = None prior_writeback_run = None checkpoint_supplement = False if todo_id or normalized_replan_obligation_id or turn_instance_id: @@ -1611,11 +1612,41 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): # An administrative closeout can report inability to start. # A prepared Turn's blocker is still a first writeback. and ( - blocked_retry is None + normalized_delivery_boundary == "semantic_closeout" + or blocked_retry is None or task_lease_proof is not None or (settlement_readback is not None and settlement_readback.prepared_lease is not None) ) ): + writeback_candidate = None + if normalized_delivery_boundary == "semantic_closeout": + # Project the final Run without replacing its observed + # work identity, semantic delta or recovery metadata. + writeback_run = { + field: record[field] + for field in ( + "goal_id", "agent_id", "todo_id", "turn_instance_id", + "settlement_identity", "classification", "delivery_outcome", + "delivery_batch_scale", "delivery_boundary", "progress_observation", + "agent_vision", "vision_checkpoint", "autonomous_replan_ack", + "refresh_recovery", + ) + if field in record + } + writeback_run["delivery_boundary"] = normalized_delivery_boundary + if goal_ref is not None: + writeback_run["goal_ref"] = dict(goal_ref) + writeback_candidate = { + "requested_proof": dict(task_lease_proof) if task_lease_proof is not None else None, + "effect_ref": f"{settlement_identity.effect_id}#durable_writeback", + "completion_todo_id": completion_todo_id, + "completion_turn_key": completion_turn_key, + "run": writeback_run, + **({"allow_same_generation_lease_renewal": allow_same_generation_lease_renewal} + if allow_same_generation_lease_renewal is not False else {}), + **({"validation_recovery_turn_instance_id": validation_recovery_turn_instance_id} + if validation_recovery_turn_instance_id is not None else {}), + } usage_booking_guard.enter_context(guard_first_writeback_execution( runtime_root=runtime_root, registry_path=registry_path, goal_id=safe_goal_id, todo_id=settlement_identity.todo_id, agent_id=settlement_identity.agent_id, @@ -1626,6 +1657,8 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): source_admission=source_admission, allow_same_generation_lease_renewal=allow_same_generation_lease_renewal, validation_recovery_turn_instance_id=validation_recovery_turn_instance_id, + writeback_candidate=writeback_candidate, + refresh_retry=refresh_retry_request, )) if settlement_readback is not None: persist_external_delivery_pause( diff --git a/tests/control_plane/test_quota_execution_preparation.py b/tests/control_plane/test_quota_execution_preparation.py index 8072d2b5da..2dcad022a4 100644 --- a/tests/control_plane/test_quota_execution_preparation.py +++ b/tests/control_plane/test_quota_execution_preparation.py @@ -19,8 +19,6 @@ from loopx.control_plane.quota.execution_preparation import ( RefreshTaskLeaseUnavailable, apply_selected_todo_execution_preparation, - guard_first_writeback_execution, - prepared_lease_receipt_fields, ) @@ -1319,10 +1317,12 @@ def _writeback_guard_fixture( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, - todo_status: str = "open", - leases: list[dict[str, object]] | None = None, + provider: str, + turn_instance_id: str, + lease: dict[str, object], + legacy_no_epoch_receipt: bool = False, ): - from types import SimpleNamespace + from loopx.control_plane.quota.settlement import read_heartbeat_settlement isolate_sqlite_runtime(tmp_path, monkeypatch) project = tmp_path / "project" @@ -1339,314 +1339,553 @@ def _writeback_guard_fixture( domain="fixture", adapter_kind="read_only_project_map_v0", state_file="ACTIVE_GOAL_STATE.md", - registered_agents=(AGENT,), + registered_agents=(AGENT, "agent-b"), + extra_goal_fields={"coordination": { + "registered_agents": [AGENT, "agent-b"], + "handoff_mode": "hard_lease", + }}, ) - todo = {**_todo(), "status": todo_status, "done": todo_status == "done"} initialize_canonical_authority( runtime, GOAL, build_todo_runtime_shadow_projection( goal_id=GOAL, handoff_mode="hard_lease", - todos=[todo], - leases=list(leases or ()), + todos=[_todo()], + leases=[lease], ), state_path=state_path, - provider="sqlite", + provider=provider, ) - - def make_settlement(**overrides): - defaults = { - "prepared_lease": None, - "prepared_lease_required": True, - "completion_event": None, - "native_child_admission": None, - "semantic_replan_guard": None, - "heartbeat_receipt": None, - "supersede_event": None, - "identity": SimpleNamespace( - value=SimpleNamespace(turn_instance_id="turn-1"), + proof = { + "idempotency_key": lease["idempotency_key"], + "expected_version": lease["version"], + } + if legacy_no_epoch_receipt: + from loopx.control_plane.quota.effect_program import SettlementIdentity + from loopx.control_plane.quota.heartbeat_receipt import ensure_turn_heartbeat_settlement_receipt + + ensure_turn_heartbeat_settlement_receipt( + runtime, + SettlementIdentity( + goal_id=GOAL, agent_id=AGENT, todo_id=TODO, + turn_instance_id=turn_instance_id, ), - } - defaults.update(overrides) - return SimpleNamespace(**defaults) + semantic_replan_guard_scoped=True, + semantic_replan_obligation_id=None, + task_lease_proof=proof, + task_lease_epoch=None, + prepared_lease_required=True, + ) + else: + code, packet = run_json_cli_result( + "quota", "should-run", "--codex-app", "--context-delivery", "off", + "--goal-id", GOAL, "--agent-id", AGENT, "--todo-id", TODO, + "--turn-instance-id", turn_instance_id, + "--task-lease-idempotency-key", str(proof["idempotency_key"]), + "--task-lease-expected-version", str(proof["expected_version"]), + registry_path=registry, runtime_root=runtime, cwd=project, + ) + assert code == 0 and packet["execution_preparation"]["state"] == "ready", packet + settlement = read_heartbeat_settlement( + runtime, goal_id=GOAL, agent_id=AGENT, todo_id=TODO, + turn_instance_id=turn_instance_id, registry_path=registry, + ) + assert settlement is not None + assert settlement.identity.value is not None + assert settlement.identity.value.turn_instance_id == turn_instance_id + return { + "project": project, "runtime": runtime, "registry": registry, + "state": state_path, "turn": turn_instance_id, "proof": proof, + "settlement": settlement, + } - def run_guard(**kwargs): - with guard_first_writeback_execution( - runtime_root=runtime, - registry_path=registry, - goal_id=GOAL, - todo_id=TODO, - agent_id=AGENT, - locked_goal=None, - **kwargs, - ): - return True - return make_settlement, run_guard +def _run_real_semantic_writeback( + context: dict[str, object], *, proof: dict[str, object] | None, + allow_same_generation_lease_renewal: bool = False, + validation_recovery_turn_instance_id: str | None = None, +): + from loopx.state_refresh import refresh_state_run + + return refresh_state_run( + registry_path=context["registry"], + runtime_root_override=str(context["runtime"]), + goal_id=str(context.get("goal_id") or GOAL), + project=context["project"], state_file=None, + classification="receipt_bound_semantic_writeback", + recommended_action="Keep the synthetic Todo open while recording this Turn.", + delivery_batch_scale="single_surface", delivery_outcome="outcome_gap", + delivery_boundary="semantic_closeout", + todo_id=str(context.get("todo") or TODO), + turn_instance_id=str(context["turn"]), + agent_id=str(context.get("agent_id") or AGENT), + progress_observation={ + "schema_version": "typed_progress_observation_v0", + "result_class": "blocked", + "work_item_id": str(context.get("todo") or TODO), + "blocker_id": "synthetic-writeback-fence", + "evidence_ids": ["evidence:synthetic-writeback-fence"], + }, + vision_unchanged_reason="The synthetic Todo remains open after this scoped writeback.", + task_lease_proof=proof, + goal_ref=context.get("goal_ref"), + allow_same_generation_lease_renewal=allow_same_generation_lease_renewal, + validation_recovery_turn_instance_id=validation_recovery_turn_instance_id, + dry_run=False, sync_global=False, + ) -def test_same_generation_lease_renewal_admits_open_todo_writeback_and_rejects_mismatches( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -) -> None: - """Same-epoch lease renewals pass when explicitly enabled, never across epochs or keys.""" - current = {**_lease(), "idempotency_key": f"{AGENT}-execution", "version": 3, "lease_epoch": 1} - make_settlement, run_guard = _writeback_guard_fixture( - tmp_path, monkeypatch, todo_status="open", leases=[current], - ) - prepared = prepared_lease_receipt_fields( - {"idempotency_key": f"{AGENT}-execution", "expected_version": 1}, - lease_epoch=1, - ) - settlement = make_settlement(prepared_lease=prepared, prepared_lease_required=True) - stale_proof = {"idempotency_key": f"{AGENT}-execution", "expected_version": 1} - renewed_proof = {"idempotency_key": f"{AGENT}-execution", "expected_version": 3} - - assert run_guard( - requested_proof=stale_proof, - settlement=settlement, - allow_same_generation_lease_renewal=True, - ) - assert run_guard( - requested_proof=renewed_proof, - settlement=settlement, - allow_same_generation_lease_renewal=False, - ) - with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard( - requested_proof=stale_proof, - settlement=settlement, - allow_same_generation_lease_renewal=False, - ) +def _writeback_side_effect_snapshot(context: dict[str, object]) -> dict[str, object]: + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.rollout_event_log import rollout_event_log_path - for bad_settlement, bad_proof in ( - # Cross-epoch renewal must fail closed even for the same key. - ( - make_settlement( - prepared_lease={**prepared, "prepared_lease_epoch": 2}, - prepared_lease_required=True, - ), - renewed_proof, - ), - # Cross-key proof must fail closed. - ( - make_settlement( - prepared_lease={**prepared, "prepared_lease_idempotency_key": "other-key"}, - prepared_lease_required=True, - ), - renewed_proof, - ), - # Legacy prepared receipt without lease_epoch must not advance versions. - ( - make_settlement( - prepared_lease={"prepared_lease_idempotency_key": f"{AGENT}-execution", "prepared_lease_version": 1}, - prepared_lease_required=True, - ), - renewed_proof, - ), - # Version regression in requested_proof must fail closed. - ( - settlement, - {"idempotency_key": f"{AGENT}-execution", "expected_version": 4}, + goal_id = str(context.get("goal_id") or GOAL) + agent_id = str(context.get("agent_id") or AGENT) + todo_id = str(context.get("todo") or TODO) + settlement = read_heartbeat_settlement( + context["runtime"], goal_id=goal_id, agent_id=agent_id, + todo_id=todo_id, turn_instance_id=str(context["turn"]), + registry_path=context["registry"], goal_ref=context.get("goal_ref"), + ) + assert settlement is not None + runs = context["runtime"] / "goals" / goal_id / "runs" + receipt_log = rollout_event_log_path(context["runtime"], goal_id) + return { + "authority": read_canonical_todos_if_promoted( + runtime_root=context["runtime"], goal_id=goal_id, include_leases=True, ), - ): - with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard( - requested_proof=bad_proof, - settlement=bad_settlement, - allow_same_generation_lease_renewal=True, - ) + "original_prepared": settlement.original_prepared_receipt, + "original_event_id": settlement.original_prepared_event_id, + "original_sha256": settlement.original_prepared_receipt_sha256, + "goal_file": context["state"].read_bytes(), + "receipt_log": receipt_log.read_bytes(), + "run_files": { + str(path.relative_to(runs)): path.read_bytes() + for path in runs.rglob("*") if path.is_file() + } if runs.exists() else {}, + } -def test_validation_recovery_turn_accepts_only_matching_retired_recovery_lease( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_writeback_requires_current_same_generation_cas( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, ) -> None: - """Resumed recovery matches exact `.validation-recovery.` or `.admission-retry` retired leases.""" - for index, suffix in enumerate((".validation-recovery.1", ".admission-retry", ".admission-retry.turn-2")): - retired_recovery = { - **_lease(status="released"), - "idempotency_key": f"turn-1{suffix}", - "version": 1, - "lease_epoch": 2, - } - make_settlement, run_guard = _writeback_guard_fixture( - tmp_path / f"recovery-{index}", monkeypatch, todo_status="done", leases=[retired_recovery], - ) - prepared = prepared_lease_receipt_fields( - {"idempotency_key": "turn-1", "expected_version": 1}, - lease_epoch=1, - ) - completion = { - "event_kind": "todo_complete", - "todo_id": TODO, - "agent_id": AGENT, - "run_id": "turn-1", - } - valid_proof = {"idempotency_key": "turn-1", "expected_version": 1} - assert run_guard( - requested_proof=valid_proof, - settlement=make_settlement( - prepared_lease=prepared, - prepared_lease_required=True, - completion_event=completion, - ), - validation_recovery_turn_instance_id="turn-1", + from loopx.control_plane.work_items.task_lease import renew_task_lease + + for case, expected_version in (("current", 3), ("stale", 1)): + original = {**_lease(), "idempotency_key": "turn-p1", "version": 1, "lease_epoch": 1} + context = _writeback_guard_fixture( + tmp_path / case, monkeypatch, provider=provider, + turn_instance_id=f"turn-p1-{case}", lease=original, ) - for bad_completion, bad_recovery_turn in ( - (None, "turn-1"), - ({**completion, "run_id": "other-turn"}, "turn-1"), - ({**completion, "agent_id": "other-agent"}, "turn-1"), - (completion, None), - (completion, "other-turn"), - ): + renewed = None + for version in (1, 2): + renewed = renew_task_lease( + registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=GOAL, todo_id=TODO, owner=AGENT, + idempotency_key="turn-p1", expected_version=version, + ) + assert renewed["lease"]["version"] == version + 1 + assert renewed["lease"]["lease_epoch"] == 1 + proof = {"idempotency_key": "turn-p1", "expected_version": expected_version} + if case == "current": + result = _run_real_semantic_writeback( + context, proof=proof, allow_same_generation_lease_renewal=True, + ) + assert result["ok"] is True + run = json.loads(Path(result["json_path"]).read_text(encoding="utf-8")) + assert run["settlement_identity"]["turn_instance_id"] == context["turn"] + assert run["refresh_recovery"]["decision"] == "append" + else: + before = _writeback_side_effect_snapshot(context) with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard( - requested_proof=valid_proof, - settlement=make_settlement( - prepared_lease=prepared, - prepared_lease_required=True, - completion_event=bad_completion, - ), - validation_recovery_turn_instance_id=bad_recovery_turn, + _run_real_semantic_writeback( + context, proof=proof, allow_same_generation_lease_renewal=True, ) + assert _writeback_side_effect_snapshot(context) == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_validation_recovery_cannot_execute_replaced_epoch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, provider: str, +) -> None: + from tests.control_plane.test_execution_loss_closeout_cli import ( + OWNER, TURN, _fixture as execution_loss_fixture, + ) + + context = execution_loss_fixture( + tmp_path, monkeypatch, capsys, provider, + source_profile=True, replace=True, replacement_owner=OWNER, + ) + original = context["original_lease"] + current = context["current_lease"] + assert current["lease_epoch"] > original["lease_epoch"] + assert current["idempotency_key"] != original["idempotency_key"] + writeback = { + **context, "turn": TURN, "agent_id": OWNER, + "proof": {"idempotency_key": original["idempotency_key"], + "expected_version": original["version"]}, + } + before = _writeback_side_effect_snapshot(writeback) + with pytest.raises(RefreshTaskLeaseUnavailable): + _run_real_semantic_writeback( + writeback, proof=writeback["proof"], + validation_recovery_turn_instance_id=TURN, + ) + assert _writeback_side_effect_snapshot(writeback) == before -def test_validation_recovery_rejects_unrelated_retired_key( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_legacy_prepared_receipt_is_exact_version_only( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, ) -> None: - """An unrelated retired lease without recovery/retry suffix never satisfies recovery.""" - for index, bad_key in enumerate(("unrelated-key", "turn-1.admission-retry2")): - foreign_retired = { - **_lease(status="released"), - "idempotency_key": bad_key, - "version": 1, - "lease_epoch": 2, - } - make_settlement, run_guard = _writeback_guard_fixture( - tmp_path / f"reject-{index}", monkeypatch, todo_status="done", leases=[foreign_retired], + from loopx.control_plane.work_items.task_lease import renew_task_lease + + legacy = {**_lease(), "idempotency_key": "legacy-turn", "version": 1} + legacy.pop("lease_epoch") + exact = _writeback_guard_fixture( + tmp_path / "exact", monkeypatch, provider=provider, + turn_instance_id="turn-legacy-exact", lease=legacy, + legacy_no_epoch_receipt=True, + ) + expected_legacy_receipt = { + "prepared_lease_key_sha256": hashlib.sha256(b"legacy-turn").hexdigest(), + "prepared_lease_version": 1, + } + assert exact["settlement"].prepared_lease == expected_legacy_receipt + assert "prepared_lease_epoch" not in exact["settlement"].prepared_lease + result = _run_real_semantic_writeback(exact, proof=exact["proof"]) + assert result["ok"] is True + + advanced = _writeback_guard_fixture( + tmp_path / "advanced", monkeypatch, provider=provider, + turn_instance_id="turn-legacy-advanced", lease=legacy, + legacy_no_epoch_receipt=True, + ) + renewed = renew_task_lease( + registry_path=advanced["registry"], runtime_root=advanced["runtime"], + goal_id=GOAL, todo_id=TODO, owner=AGENT, + idempotency_key="legacy-turn", expected_version=1, + )["lease"] + assert renewed["version"] == 2 + before = _writeback_side_effect_snapshot(advanced) + with pytest.raises(RefreshTaskLeaseUnavailable): + _run_real_semantic_writeback( + advanced, + proof={"idempotency_key": renewed["idempotency_key"], + "expected_version": renewed["version"]}, + allow_same_generation_lease_renewal=True, ) - prepared = prepared_lease_receipt_fields( - {"idempotency_key": "turn-1", "expected_version": 1}, - lease_epoch=1, + assert _writeback_side_effect_snapshot(advanced) == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_exact_source_supersede_admits_only_its_original_turn_writeback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, provider: str, +) -> None: + from tests.control_plane.test_execution_loss_closeout_cli import ( + OWNER, TURN, _binding, _fixture as execution_loss_fixture, _proof, _readback, + ) + + context = execution_loss_fixture( + tmp_path, monkeypatch, capsys, provider, + source_profile=True, replace=False, + ) + expected_todo = "todo_" + hashlib.sha256(b"execution-loss-task").hexdigest()[:24] + expected_effect = ( + f"execution-loss-closeout:original-worker:{expected_todo}:original-prepared-turn" + ) + admission_identity = _readback(context).identity.value.as_dict() + assert admission_identity["goal_id"] == "execution-loss-closeout" + assert admission_identity["agent_id"] == "original-worker" + assert admission_identity["todo_id"] == expected_todo + assert admission_identity["turn_instance_id"] == "original-prepared-turn" + assert admission_identity["effect_id"] == expected_effect + lease = context["original_lease"] + code, retired = context["cli"]( + "todo", "supersede", "--reason", "Retire only the original synthetic Turn.", + *_binding(context), *_proof(lease), + ) + assert code == 0 and retired["superseded"] is True, retired + assert retired["settlement_identity"]["turn_instance_id"] == TURN + writeback = { + **context, "turn": TURN, "agent_id": OWNER, + "proof": {"idempotency_key": lease["idempotency_key"], + "expected_version": lease["version"]}, + } + assert retired["settlement_identity"] == admission_identity + before_wrong_turn = _writeback_side_effect_snapshot(writeback) + with pytest.raises(RuntimeError, match="exact settlement readback unexpectedly returned not-found"): + _run_real_semantic_writeback( + {**writeback, "turn": "foreign-prepared-turn"}, proof=writeback["proof"], ) - with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard( - requested_proof={"idempotency_key": "turn-1", "expected_version": 1}, - settlement=make_settlement( - prepared_lease=prepared, - prepared_lease_required=True, - completion_event={ - "event_kind": "todo_complete", - "todo_id": TODO, - "agent_id": AGENT, - "run_id": "turn-1", - }, - ), - validation_recovery_turn_instance_id="turn-1", - ) + assert _writeback_side_effect_snapshot(writeback) == before_wrong_turn + result = _run_real_semantic_writeback(writeback, proof=writeback["proof"]) + assert result["ok"] is True + run = json.loads(Path(result["json_path"]).read_text(encoding="utf-8")) + assert run["settlement_identity"] == admission_identity + assert run["todo_id"] == expected_todo + index = context["runtime"] / "goals" / "execution-loss-closeout" / "runs" / "index.jsonl" + [index_row] = [json.loads(line) for line in index.read_text(encoding="utf-8").splitlines()] + assert index_row["settlement_identity"] == admission_identity -def test_supersede_event_admits_done_todo_only_for_exact_turn( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_writeback_holds_shared_marker_until_run_is_durable_before_lease_renewal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, ) -> None: - """Superseded done Todos satisfy completion receipt checks only for their exact Turn.""" - retired = {**_lease(status="released"), "idempotency_key": "turn-1", "version": 1, "lease_epoch": 1} - make_settlement, run_guard = _writeback_guard_fixture( - tmp_path, monkeypatch, todo_status="done", leases=[retired], + """Composition proof: the actual writer retains M through its Run append. + + The zero-wait process probe observes the shared marker, not a native waiter. + Canonical-writer coverage separately proves public lease acquire uses M. + """ + from threading import Event, Thread + from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target + from loopx.control_plane.quota import execution_preparation + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.control_plane.work_items.task_lease import renew_task_lease + from loopx.rollout_event_log import rollout_event_log_path + from loopx import state_refresh + + original = {**_lease(), "idempotency_key": "same-generation-writeback", "version": 1, + "lease_epoch": 1} + context = _writeback_guard_fixture( + tmp_path, monkeypatch, provider=provider, + turn_instance_id="turn-same-generation-writeback", lease=original, ) - prepared = prepared_lease_receipt_fields( - {"idempotency_key": "turn-1", "expected_version": 1}, - lease_epoch=1, + original_settlement = context["settlement"] + original_receipt = original_settlement.original_prepared_receipt + original_event_id = original_settlement.original_prepared_event_id + original_receipt_sha256 = original_settlement.original_prepared_receipt_sha256 + receipt_log = rollout_event_log_path(context["runtime"], GOAL) + original_receipt_bytes = receipt_log.read_bytes() + current = None + for version in (1, 2): + current = renew_task_lease( + registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=GOAL, todo_id=TODO, owner=AGENT, + idempotency_key="same-generation-writeback", expected_version=version, + )["lease"] + assert current is not None + assert current["version"] == 3 and current["lease_epoch"] == 1 + + boundary_names = ("qualification", "json", "markdown", "index") + reached = {name: Event() for name in boundary_names} + release = {name: Event() for name in boundary_names} + writer_result: list[dict[str, object]] = [] + writer_error: list[BaseException] = [] + qualification_scopes: list[dict[str, object]] = [] + original_write_text = Path.write_text + original_append_index = state_refresh.append_run_index_record + original_qualification = execution_preparation.read_turn_writeback_qualification + + def wait_at_boundary(name: str) -> None: + reached[name].set() + if not release[name].wait(timeout=20): + raise AssertionError(f"writeback {name} barrier was not released") + + def pause_at_run_write(path: Path, *args, **kwargs): + if path.parent.name == "runs" and path.suffix in {".json", ".md"}: + wait_at_boundary("json" if path.suffix == ".json" else "markdown") + return original_write_text(path, *args, **kwargs) + + def pause_at_index_append(*args, **kwargs): + wait_at_boundary("index") + return original_append_index(*args, **kwargs) + + def observe_fresh_qualification(*args, **kwargs): + scope = original_qualification(*args, **kwargs) + qualification_scopes.append(scope) + wait_at_boundary("qualification") + return scope + + monkeypatch.setattr(Path, "write_text", pause_at_run_write) + monkeypatch.setattr(state_refresh, "append_run_index_record", pause_at_index_append) + monkeypatch.setattr( + execution_preparation, "read_turn_writeback_qualification", observe_fresh_qualification, ) - supersede = { - "event_kind": "todo_supersede", - "status": "done", - "todo_id": TODO, - "agent_id": AGENT, - "run_id": "turn-1", + writeback = { + **context, + "proof": {"idempotency_key": current["idempotency_key"], + "expected_version": current["version"]}, } - assert run_guard( - requested_proof={"idempotency_key": "turn-1", "expected_version": 1}, - settlement=make_settlement( - prepared_lease=prepared, - prepared_lease_required=True, - supersede_event=supersede, + + def write_run() -> None: + try: + writer_result.append(_run_real_semantic_writeback(writeback, proof=writeback["proof"])) + except BaseException as error: # preserve the writer failure for the parent test thread + writer_error.append(error) + + writer = Thread(target=write_run, name="synthetic-semantic-writeback") + + repo = Path(__file__).resolve().parents[2] + maintenance_target = shadow_maintenance_lock_target(context["runtime"], GOAL).resolve() + probe_source = """ +from pathlib import Path +import sys +from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target +from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock +target = shadow_maintenance_lock_target(Path(sys.argv[1]), sys.argv[2]) +assert target.resolve() == Path(sys.argv[3]).resolve() +try: + with exclusive_mutation_file_lock(target, timeout_seconds=0, operation="writeback-test-probe"): + pass +except LockAcquireTimeoutError: + raise SystemExit(23) +""" + + def probe_marker() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", probe_source, str(context["runtime"]), GOAL, + str(maintenance_target)], + cwd=repo, env={**os.environ, "PYTHONPATH": str(repo)}, + capture_output=True, text=True, timeout=15, check=False, + ) + + writer_started = False + try: + writer.start() + writer_started = True + for name in boundary_names: + assert reached[name].wait(timeout=20), f"writer never reached the {name} durability boundary" + held = probe_marker() + assert held.returncode == 23, f"{name}: {held.stdout}{held.stderr}" + release[name].set() + finally: + for event in release.values(): + event.set() + if writer_started: + writer.join(timeout=20) + assert not writer.is_alive(), "writeback writer did not leave the shared marker" + assert writer_error == [] + assert len(writer_result) == 1 and writer_result[0]["ok"] is True + assert qualification_scopes == [{ + "step_kind": "durable_writeback", + "effect_ref": ( + "goal-execution-preparation:agent-a:todo_execution_preparation:" + "turn-same-generation-writeback#durable_writeback" ), + "delivery_boundary": "semantic_closeout", + "original_prepared": original_receipt, + "selected_obligation_id": None, + }] + released = probe_marker() + assert released.returncode == 0, released.stdout + released.stderr + + # The supported public writer runs only after the semantic writer returned. + code, renewed = run_json_cli_result( + "task-lease", "renew", "--goal-id", GOAL, "--todo-id", TODO, + "--owner", AGENT, "--idempotency-key", current["idempotency_key"], + "--expected-version", str(current["version"]), "--ttl-seconds", "600", + registry_path=context["registry"], runtime_root=context["runtime"], + cwd=context["project"], ) - for bad_supersede in ( - None, - {**supersede, "run_id": "foreign-turn"}, - {**supersede, "todo_id": "other-todo"}, - {**supersede, "status": "open"}, - ): - with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard( - requested_proof={"idempotency_key": "turn-1", "expected_version": 1}, - settlement=make_settlement( - prepared_lease=prepared, - prepared_lease_required=True, - supersede_event=bad_supersede, - ), - ) + assert code == 0, renewed + assert renewed["lease"]["version"] == 4 + assert renewed["lease"]["lease_epoch"] == current["lease_epoch"] + assert renewed["lease"]["idempotency_key"] == current["idempotency_key"] + assert receipt_log.read_bytes() == original_receipt_bytes + final = read_heartbeat_settlement( + context["runtime"], goal_id=GOAL, agent_id=AGENT, todo_id=TODO, + turn_instance_id=context["turn"], registry_path=context["registry"], + ) + assert final is not None + assert final.original_prepared_receipt == original_receipt + assert final.original_prepared_event_id == original_event_id + assert final.original_prepared_receipt_sha256 == original_receipt_sha256 + runs = list((context["runtime"] / "goals" / GOAL / "runs").glob("*.json")) + assert len(runs) == 1 + run = json.loads(runs[0].read_text(encoding="utf-8")) + assert final.writeback_run == run + assert run["settlement_identity"]["turn_instance_id"] == context["turn"] + markdown = runs[0].with_suffix(".md") + assert markdown.exists() and context["turn"] in markdown.read_text(encoding="utf-8") + index = context["runtime"] / "goals" / GOAL / "runs" / "index.jsonl" + index_rows = [json.loads(line) for line in index.read_text(encoding="utf-8").splitlines()] + assert len(index_rows) == 1 + assert index_rows[0]["settlement_identity"] == run["settlement_identity"] + assert Path(index_rows[0]["json_path"]) == runs[0] + assert Path(index_rows[0]["markdown_path"]) == markdown -def test_todo_bound_autonomous_replan_admits_without_prepared_lease_only_on_replan_guard( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_todo_semantic_writeback_is_operation_scoped_without_business_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, ) -> None: - """Todo-bound autonomous replan Turns pass without prepared_lease but stay lease-fenced.""" - active = {**_lease(), "idempotency_key": f"{AGENT}-execution", "version": 1, "lease_epoch": 1} - make_settlement, run_guard = _writeback_guard_fixture( - tmp_path, monkeypatch, todo_status="open", leases=[active], + from tests.control_plane.test_quota_settlement_cli import ( + AGENT_ID, GOAL_ID, SELECTED_REPLAN_TODO_ID, + _bind_selected_replan_guard, _configure_selected_todo_replan_fixture, + _initialize_fixture_authority, _projected_cli_args, _run_cli, _write_fixture, ) - valid_proof = {"idempotency_key": f"{AGENT}-execution", "expected_version": 1} - replan_settlement = make_settlement( - prepared_lease=None, - prepared_lease_required=False, - semantic_replan_guard={"scope": "turn_guard", "selected_obligation_id": "replan-1"}, - heartbeat_receipt={"status": "autonomous_replan_required", "details": {"delivery_allowed": False}}, + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path) + _configure_selected_todo_replan_fixture(project, registry, stride_strategy="off") + _initialize_fixture_authority( + project, runtime, provider=provider, handoff_mode="hard_lease", leases=[], ) - assert run_guard(requested_proof=valid_proof, settlement=replan_settlement) - assert run_guard(requested_proof=None, settlement=replan_settlement) - - for bad_settlement, proof in ( - (replan_settlement, {"idempotency_key": f"{AGENT}-execution", "expected_version": 99}), - ( - make_settlement( - prepared_lease=None, - prepared_lease_required=False, - semantic_replan_guard={"scope": "turn_guard", "selected_obligation_id": "replan-1"}, - heartbeat_receipt={"status": "normal_run", "details": {"delivery_allowed": True}}, - ), - valid_proof, - ), - ( - make_settlement( - prepared_lease=None, - prepared_lease_required=False, - semantic_replan_guard={"scope": "turn_guard", "selected_obligation_id": ""}, - heartbeat_receipt={"status": "autonomous_replan_required", "details": {"delivery_allowed": False}}, - ), - valid_proof, - ), - ): - with pytest.raises(RefreshTaskLeaseUnavailable): - run_guard(requested_proof=proof, settlement=bad_settlement) - - # Foreign active lease must reject even on a todo-bound replan guard. - foreign_dir = tmp_path / "foreign" - make_foreign, run_foreign = _writeback_guard_fixture( - foreign_dir, - monkeypatch, - todo_status="open", - leases=[{**_lease(owner="other-agent"), "idempotency_key": "other-exec", "version": 1}], + turn = "turn-operation-scoped-semantic-writeback" + admitted = _bind_selected_replan_guard(registry, runtime, project, turn) + assert admitted["normal_delivery_allowed"] is False + assert admitted["heartbeat_receipt"]["settlement_identity"]["todo_id"] == SELECTED_REPLAN_TODO_ID + assert "prepared_lease" not in admitted["heartbeat_receipt"] + + refresh = next( + command for command in admitted["interaction_contract"]["cli_channel"]["next_cli_actions"] + if "refresh-state" in command ) + vision = tmp_path / "semantic-writeback-vision.json" + vision.write_text(json.dumps({ + "schema_version": "goal_vision_replan_contract_v0", + "state": "vision_patch_proposed", + "vision_patch": { + "vision_summary": "Validate the selected bounded slice.", + "acceptance_summary": "The slice retains independent evidence.", + "advancement_policy": "as_needed", + }, + "path_delta": { + "schema_version": "goal_path_delta_v0", "outcome": "replan", + "prior_assumption": "The long chain needed one review.", + "observed_reality": "The selected slice now has a verified route.", + "retained": ["Existing acceptance boundaries"], + "changed": ["Proceed through the selected validation slice"], + "evidence_refs": ["evidence:semantic-writeback"], + }, + }) + "\n", encoding="utf-8") + refresh = refresh.replace( + "", str(vision), + ) + for key, value in { + "": "advanced", + "": "semantic-writeback-surface", + "": "semantic-writeback-hypothesis", + "": "semantic-writeback-probe", + "": "evidence:semantic-writeback", + }.items(): + refresh = refresh.replace(key, value) + code, result = _run_cli( + registry, runtime, *_projected_cli_args(refresh, turn_instance_id=turn), cwd=project, + ) + assert code == 0 and result["settlement_result"]["ok"] is True, result + run = json.loads(Path(result["json_path"]).read_text(encoding="utf-8")) + semantic = run["autonomous_replan_ack"]["semantic_delta"] + assert semantic["accepted"] is True + assert semantic["observation_fingerprint"] == run["progress_observation"]["fingerprint"] + readback = read_heartbeat_settlement( + runtime, goal_id=GOAL_ID, agent_id=AGENT_ID, + todo_id=SELECTED_REPLAN_TODO_ID, turn_instance_id=turn, + registry_path=registry, + ) + assert readback is not None + assert readback.prepared_lease is None + assert readback.completion_event is None + assert readback.writeback_run == run with pytest.raises(RefreshTaskLeaseUnavailable): - run_foreign( - requested_proof=None, - settlement=make_foreign( - prepared_lease=None, - prepared_lease_required=False, - semantic_replan_guard={"scope": "turn_guard", "selected_obligation_id": "replan-1"}, - heartbeat_receipt={"status": "autonomous_replan_required", "details": {"delivery_allowed": False}}, - ), + read_heartbeat_settlement( + runtime, goal_id=GOAL_ID, agent_id=AGENT_ID, + todo_id=SELECTED_REPLAN_TODO_ID, turn_instance_id=turn, + registry_path=registry, + resume_lease_candidate={"requested_proof": None}, ) diff --git a/tests/control_plane_ts/heartbeat_resume_lease.test.ts b/tests/control_plane_ts/heartbeat_resume_lease.test.ts index 646f91c413..a294900e5d 100644 --- a/tests/control_plane_ts/heartbeat_resume_lease.test.ts +++ b/tests/control_plane_ts/heartbeat_resume_lease.test.ts @@ -25,6 +25,9 @@ import { readQuotaSettlementForAdmittedOwnerFromSnapshot, readQuotaSettlementSnapshot, } from "../../loopx/control_plane/quota/settlement_readback.ts"; +import {buildVisionCheckpoint, VISION_REFRESH_REQUEST_SCHEMA, visionAuthoringContract} + from "../../loopx/control_plane/goals/vision_checkpoint.ts"; +import {decodeRefreshRetry, refreshRecovery} from "../../loopx/control_plane/quota/refresh_recovery.ts"; import {projection, todo} from "./shadow_file_fixture.ts"; const identity = settlementIdentity({goal_id: "goal-a", agent_id: "agent-a", @@ -295,17 +298,28 @@ function request(root: string): JsonObject { allow_unbound_binding: false}; } -async function receiptFixture(events: JsonObject[], handoffMode = "hard_lease", seedLease = currentLease) { +async function receiptFixture(events: JsonObject[], handoffMode = "hard_lease", seedLease = currentLease, + options: {provider?: "file" | "sqlite"; seedTodo?: JsonObject} = {}) { const root = await mkdtemp(join(tmpdir(), "prepared-source-")); const log = join(root, "goals", identity.goal_id, "rollout-event-log.jsonl"); await mkdir(join(root, "goals", identity.goal_id), {recursive: true}); await writeFile(log, events.map(row => JSON.stringify(row)).join("\n") + "\n"); - const store = new FileAuthorityStore(join(root, "authority", "file-v0"), identity.goal_id); + const store = options.provider === "sqlite" + ? new (await import("../../loopx/control_plane/coordination/sqlite_authority_store.ts")).SqliteAuthorityStore( + join(root, "authority", "sqlite-v0"), identity.goal_id) + : new FileAuthorityStore(join(root, "authority", "file-v0"), identity.goal_id); const committed = await store.commitAuthority({operation_id: "seed-prepared-source", expected_provider_revision: null, next_projection: projection([ - {...todo(identity.todo_id!), claimed_by: identity.agent_id}], [seedLease], handoffMode), + options.seedTodo ?? {...todo(identity.todo_id!), claimed_by: identity.agent_id}], [seedLease], handoffMode), events: [], receipts: []}); assert.equal(committed.status, "applied", JSON.stringify(committed)); + if (options.provider === "sqlite") { + const storeIdentity = await store.storeIdentity(); + assert.equal(storeIdentity.status, "available"); + if (storeIdentity.status !== "available") throw new Error("Fixture authority identity unavailable"); + await (await import("../../loopx/control_plane/coordination/local_authority_provider.ts")) + .publishLocalAuthoritySelection(root, identity.goal_id, "sqlite", storeIdentity.store_identity); + } return {root, log, store}; } @@ -543,3 +557,153 @@ test("legacy identity upgrade retains prepared source that predated the effect i assert.equal((broken.identity as any).result.failure.kind, "receipt_missing"); } finally {await rm(f.root, {recursive: true, force: true});} }); + + +function semanticWritebackCandidate(requestedProof: JsonObject | null = null): JsonObject { + const packet = visionAuthoringContract().minimal_example as JsonObject; + const vision = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "prepare", + goal_id: identity.goal_id, agent_id: identity.agent_id, agent_vision_packet: packet, + existing_agent_vision: null, merge_patch: false, require_path_delta_for_durable_change: true, + allow_terminal_stride_path: false}).agent_vision; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: identity.agent_id, agent_vision: vision, existing_agent_vision: null, + vision_unchanged_reason: null, delivery_outcome: "outcome_progress", + active_state_next_action_would_update: false, delivery_boundary: "semantic_closeout", + todo_id: identity.todo_id, completion_todo_id: null, autonomous_replan_recorded: false, blocked_retry: null}); + const retry = {vision: packet, unchanged_reason: null, merge_patch: false, + workspace_requested: false, mutation: {}, delivery_outcome: "outcome_progress", delivery_batch_scale: null, + delivery_boundary: "semantic_closeout", progress_observation: null}; + return {refresh_retry: retry, + writeback_candidate: {requested_proof: requestedProof, effect_ref: `${identity.effect_id}#durable_writeback`, + completion_todo_id: null, completion_turn_key: null, + run: {goal_id: identity.goal_id, agent_id: identity.agent_id, todo_id: identity.todo_id, + turn_instance_id: identity.turn_instance_id, settlement_identity: settlementIdentityPayload(identity), + classification: "turn_run_once", delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", + progress_observation: null, agent_vision: vision, vision_checkpoint: checkpoint, autonomous_replan_ack: null, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: resume hints retain exact CAS and original-generation authority`, async () => { + const f = await receiptFixture([preparedReceipt], "hard_lease", currentLease, {provider}); + try { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + for (const renewal of [true, false]) { + const result = await readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}, + allow_same_generation_lease_renewal: renewal, validation_recovery_turn_instance_id: identity.turn_instance_id}}); + assert.deepEqual((result.resume_lease_qualification as JsonObject).original_prepared, originalPrepared); + assert.equal((result.resume_lease_qualification as JsonObject).state, "ready"); + } + for (const candidate of [ + {requested_proof: {idempotency_key: originalKey, expected_version: 119}, allow_same_generation_lease_renewal: true}, + {requested_proof: {idempotency_key: originalKey, expected_version: 120}, + validation_recovery_turn_instance_id: "foreign-turn"}, + ]) await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: candidate}), + {code: "refresh_task_lease_unavailable"}); + for (const hint of [{allow_same_generation_lease_renewal: "true"}, + {validation_recovery_turn_instance_id: " turn-original "}]) { + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: { + requested_proof: {idempotency_key: originalKey, expected_version: 120}, ...hint}})); + } + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: active prepared authority still requalifies actual semantic inputs`, async () => { + const f = await receiptFixture([preparedReceipt], "hard_lease", currentLease, {provider}); + try { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const input = {...request(f.root), ...semanticWritebackCandidate({idempotency_key: originalKey, expected_version: 120})}; + const candidate = input.writeback_candidate as JsonObject, run = candidate.run as JsonObject; + const retry = input.refresh_retry as JsonObject; + const result = await readQuotaSettlement({...input, writeback_candidate: {...candidate, + run: {...run, classification: "caller-supplied-opaque-classification"}}}); + assert.deepEqual((result.writeback_qualification as any).result.value.original_prepared, originalPrepared); + for (const patch of [ + {vision_checkpoint: {...run.vision_checkpoint as JsonObject, satisfied: false}}, + {vision_checkpoint: {...run.vision_checkpoint as JsonObject, triggers: []}}, + {agent_vision: {...run.agent_vision as JsonObject, state: "invalid-vision-state"}}, + {agent_vision: null}, + ]) await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, run: {...run, ...patch}}})); + const withoutVision = {...retry, vision: null}; + await assert.rejects(readQuotaSettlement({...input, refresh_retry: withoutVision, + writeback_candidate: {...candidate, run: {...run, agent_vision: null, + refresh_recovery: refreshRecovery(decodeRefreshRetry(withoutVision)!, null, false, undefined, false)}}}), + {code: "refresh_task_lease_unavailable"}, "A forged satisfied checkpoint cannot replace actual Vision"); + const progress = {schema_version: "typed_progress_observation_v0", result_class: "advanced", + work_item_id: identity.todo_id, fingerprint: "abcdef0123456789", surface_id: "surface:current", + evidence_ids: ["evidence:current"]}; + for (const observation of [{...progress, work_item_id: "todo_foreign", evidence_ids: ["evidence:foreign"]}, + {...progress, evidence_ids: ["untyped evidence"]}]) { + await assert.rejects(readQuotaSettlement({...input, refresh_retry: {...retry, progress_observation: observation}, + writeback_candidate: {...candidate, run: {...run, progress_observation: observation}}})); + } + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: same-Turn supersede qualifies only first semantic writeback`, async () => { + const supersede: JsonObject = {...completed, event_kind: "todo_supersede", status: "done", event_id: "supersede-original"}; + const f = await receiptFixture([preparedReceipt, supersede], "hard_lease", {...currentLease, status: "released"}, + {provider, seedTodo: {...todo(identity.todo_id!, "done"), claimed_by: null}}); + try { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const input = {...request(f.root), ...semanticWritebackCandidate()}; + const result = await readQuotaSettlement(input); + assert.deepEqual((result.writeback_qualification as any).result.value, { + step_kind: "durable_writeback", effect_ref: `${identity.effect_id}#durable_writeback`, + delivery_boundary: "semantic_closeout", original_prepared: originalPrepared, selected_obligation_id: null}); + assert.equal((result.writeback_qualification as any).payload.ok, true); + assert.equal(result.resume_lease_qualification, undefined); + assert.equal(result.completion_event, null); + assert.equal(result.original_prepared_receipt_sha256, canonicalAuthoritySha256(preparedReceipt)); + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: {requested_proof: null}}), + {code: "refresh_task_lease_unavailable"}, "Supersede does not grant business resume"); + const candidate = input.writeback_candidate as JsonObject; + for (const patch of [ + {effect_ref: `${identity.effect_id}#terminal_closeout`}, + {completion_todo_id: identity.todo_id}, {completion_turn_key: identity.turn_instance_id}, + {requested_proof: {idempotency_key: originalKey, expected_version: 119}, allow_same_generation_lease_renewal: true}, + {validation_recovery_turn_instance_id: "foreign-turn"}, + {run: {...candidate.run as JsonObject, turn_instance_id: "foreign-turn"}}, + {run: {...candidate.run as JsonObject, delivery_outcome: "outcome_achieved"}}, + {run: {...candidate.run as JsonObject, vision_checkpoint: { + ...(candidate.run as any).vision_checkpoint, satisfied: false}}}, + ]) await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, ...patch}}), + {code: "refresh_task_lease_unavailable"}); + const snapshot = await readQuotaSettlementSnapshot(f.root, identity.goal_id); + assert.throws(() => readQuotaSettlementFromSnapshot(input, snapshot), /live canonical authority/); + for (const patch of [{run_id: "foreign-turn"}, {agent_id: "agent-b"}, {todo_id: "todo_foreign"}, + {goal_id: "goal-b"}, {status: "failed"}, {details: {settlement_effect_id: "foreign-effect"}}]) { + await writeFile(f.log, [preparedReceipt, {...supersede, ...patch}].map(row => JSON.stringify(row)).join("\n") + "\n"); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}); + } + await writeFile(f.log, [preparedReceipt, supersede, completed].map(row => JSON.stringify(row)).join("\n") + "\n"); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}, + "Supersede qualification cannot mix a business completion"); + await writeFile(f.log, before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: supersede rejects replacement generation and epochless renewal`, async () => { + const supersede = {...completed, event_kind: "todo_supersede", status: "done", event_id: "supersede-original"}; + for (const changed of [ + {receipt: preparedReceipt, lease: {...currentLease, idempotency_key: "replacement", lease_epoch: 51, version: 121}}, + {receipt: preparedReceipt, lease: {...currentLease, lease_epoch: 51}}, + {receipt: {...preparedReceipt, details: {...preparedReceipt.details as JsonObject, + prepared_lease_epoch: undefined}}, lease: currentLease}, + ]) { + const f = await receiptFixture([changed.receipt, supersede], "hard_lease", {...changed.lease, status: "released"}, + {provider, seedTodo: {...todo(identity.todo_id!, "done"), claimed_by: null}}); + try { + await assert.rejects(readQuotaSettlement({...request(f.root), ...semanticWritebackCandidate({ + idempotency_key: changed.lease.idempotency_key, expected_version: changed.lease.version})}), + {code: "refresh_task_lease_unavailable"}); + } finally {await rm(f.root, {recursive: true, force: true});} + } + }); +} diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index bf4fef5987..bcb5003305 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -37,6 +37,10 @@ import { PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, preflightPriorHostTurnCloseout, } from "../../loopx/control_plane/quota/unsettled_host_turn_recovery.ts"; +import {heartbeatSemanticAdmissionEvent} from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; +import {buildVisionCheckpoint, VISION_REFRESH_REQUEST_SCHEMA, visionAuthoringContract} + from "../../loopx/control_plane/goals/vision_checkpoint.ts"; +import {decodeRefreshRetry, refreshRecovery} from "../../loopx/control_plane/quota/refresh_recovery.ts"; import { requireJsonObject } from "../../loopx/control_plane/runtime_decode.ts"; import { AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, @@ -2871,3 +2875,192 @@ test("readback exposes matching supersede_event when records are included", asyn await rm(runtimeRoot, {recursive: true, force: true}); } }); + + +const selectedWritebackObligation = "replan-1111111111111111"; +function todoSemanticAdmission(): JsonObject { + return {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", event_id: "semantic-admitted", + goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", recorded_at: "2026-01-01T00:00:00Z", + details: {todo_id: todoId, selected_todo_id: todoId, settlement_effect_id: identity.effect_id, + closeout_required: true, ok: true, should_run: true, + must_attempt_work: true, delivery_allowed: false, quiet_noop_allowed: false, interaction_mode: "autonomous_replan", + semantic_replan_obligation_id: selectedWritebackObligation, semantic_replan_review_kind: "goal_outcome_uncovered", + semantic_replan_review_stop_condition: "Stop at owner-only decisions."}}; +} + +function todoSemanticWriteback(): JsonObject { + const packet = visionAuthoringContract().minimal_example as JsonObject; + const vision = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "prepare", + goal_id: goalId, agent_id: agentId, agent_vision_packet: packet, existing_agent_vision: null, + merge_patch: false, require_path_delta_for_durable_change: true, allow_terminal_stride_path: false}).agent_vision; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: agentId, agent_vision: vision, existing_agent_vision: null, vision_unchanged_reason: null, + delivery_outcome: "outcome_progress", active_state_next_action_would_update: false, + delivery_boundary: "semantic_closeout", todo_id: todoId, completion_todo_id: null, + autonomous_replan_recorded: true, blocked_retry: null}); + const progress = {schema_version: "typed_progress_observation_v0", result_class: "advanced", work_item_id: todoId, + fingerprint: "1234567890abcdef", surface_id: "surface:verified", evidence_ids: ["evidence:verified"]}; + // Independent expected semantic outcomes: the typed observation advances a + // new surface; the evidenced Vision retains the route. The selected scalar + // outcome review accepts the surface, not the Vision decision by itself. + const semantic = {schema_version: "replan_semantic_delta_v0", accepted: true, + outcomes: ["new_surface", "fresh_vision_path_outcome"], satisfying_outcomes: ["new_surface"], + required_any_of: ["new_surface", "new_hypothesis", "new_probe_family", "new_runnable_successor", + "coverage_backed_exploration_exhausted", "new_concrete_blocker", "coverage_backed_no_followup"], + observation_fingerprint: progress.fingerprint}; + const retry = {vision: packet, unchanged_reason: null, merge_patch: false, workspace_requested: false, + mutation: {autonomous_replan_recorded: true}, delivery_outcome: "outcome_progress", delivery_batch_scale: null, + delivery_boundary: "semantic_closeout", progress_observation: progress}; + return {refresh_retry: retry, + writeback_candidate: {requested_proof: null, effect_ref: `${identity.effect_id}#durable_writeback`, + completion_todo_id: null, completion_turn_key: null, + run: {goal_id: goalId, agent_id: agentId, todo_id: todoId, turn_instance_id: turnId, + settlement_identity: settlementIdentityPayload(identity), classification: "turn_run_once", + delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", progress_observation: progress, + agent_vision: vision, vision_checkpoint: checkpoint, + autonomous_replan_ack: {schema_version: "autonomous_replan_ack_v0", recorded: true, semantic_delta: semantic}, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; +} + +async function todoSemanticFixture(provider: "file" | "sqlite", seedTodo?: JsonObject, leases: JsonObject[] = []) { + const root = await fixture({guard: false}); + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const admission = todoSemanticAdmission(); + await writeFile(log, JSON.stringify(admission) + "\n"); + const {projection, todo} = await import("./shadow_file_fixture.ts"); + const store = provider === "sqlite" + ? new (await import("../../loopx/control_plane/coordination/sqlite_authority_store.ts")).SqliteAuthorityStore( + join(root, "authority", "sqlite-v0"), goalId) + : new (await import("../../loopx/control_plane/coordination/file_authority_store.ts")).FileAuthorityStore( + join(root, "authority", "file-v0"), goalId); + assert.equal((await store.commitAuthority({operation_id: "seed-semantic-authority", expected_provider_revision: null, + next_projection: {...projection([seedTodo ?? {...todo(todoId), claimed_by: agentId}], leases), goal_id: goalId}, + events: [], receipts: []})).status, "applied"); + if (provider === "sqlite") { + const observed = await store.storeIdentity(); + assert.equal(observed.status, "available"); + if (observed.status !== "available") throw new Error("Fixture authority unavailable"); + await (await import("../../loopx/control_plane/coordination/local_authority_provider.ts")) + .publishLocalAuthoritySelection(root, goalId, "sqlite", observed.store_identity); + } + return {root, log, store, admission}; +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: Todo-bound semantic writeback without business lease remains operation scoped`, async () => { + const f = await todoSemanticFixture(provider); + try { + const logBefore = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + const result = await readQuotaSettlement(input); + assert.deepEqual((result.writeback_qualification as any).result.value, { + step_kind: "durable_writeback", effect_ref: `${identity.effect_id}#durable_writeback`, + delivery_boundary: "semantic_closeout", original_prepared: null, + selected_obligation_id: selectedWritebackObligation}); + assert.equal((result.writeback_qualification as any).payload.ok, true); + assert.equal(result.resume_lease_qualification, undefined); + assert.equal(result.prepared_lease, undefined); + assert.equal(Object.hasOwn(result, "prepared_lease"), false); + assert.equal(result.completion_event, null); + await assert.rejects(readQuotaSettlement({...request(f.root), resume_lease_candidate: {requested_proof: null}}), + {code: "refresh_task_lease_unavailable"}); + const candidate = input.writeback_candidate as JsonObject, run = candidate.run as JsonObject; + for (const patch of [ + {requested_proof: {idempotency_key: "invented-business-proof", expected_version: 1}}, + {completion_todo_id: todoId}, {completion_turn_key: turnId}, + {run: {...run, autonomous_replan_ack: {...run.autonomous_replan_ack as JsonObject, + semantic_delta: {...(run.autonomous_replan_ack as any).semantic_delta, outcomes: ["new_concrete_blocker"]}}}}, + {run: {...run, settlement_identity: {...settlementIdentityPayload(identity), todo_id: "todo_foreign"}}}, + ]) await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, ...patch}}), + {code: "refresh_task_lease_unavailable"}); + const retry = input.refresh_retry as JsonObject; + const progress = run.progress_observation as JsonObject; + for (const observation of [{...progress, work_item_id: "todo_foreign", evidence_ids: ["evidence:foreign"]}, + {...progress, work_item_id: undefined}, {...progress, evidence_ids: ["untyped evidence"]}]) { + await assert.rejects(readQuotaSettlement({...input, refresh_retry: {...retry, progress_observation: observation}, + writeback_candidate: {...candidate, run: {...run, progress_observation: observation}}})); + } + // All real retry metadata must bind the actual Run, including mutation + // values under the existing recovery digest owner. + for (const patch of [{delivery_batch_scale: "multi_surface"}, + {mutation: {autonomous_replan_recorded: true, next_action: "Different next step"}}]) { + await assert.rejects(readQuotaSettlement({...input, refresh_retry: {...retry, ...patch}}), + {code: "refresh_task_lease_unavailable"}); + } + await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, run: {...run, + refresh_recovery: {...run.refresh_recovery as JsonObject, mutation_digest: "0".repeat(64)}}}}), + {code: "refresh_task_lease_unavailable"}); + const scaled = {...retry, delivery_batch_scale: "single_surface"}; + const scaledResult = await readQuotaSettlement({...input, refresh_retry: scaled, + writeback_candidate: {...candidate, run: {...run, delivery_batch_scale: "single_surface"}}}); + assert.equal((scaledResult.writeback_qualification as any).payload.ok, true); + for (const patch of [{external_delivery: {suppress: true, resume_key: null}}, + {workspace_requested: true}, {checkpoint_read_context_id: "checkpoint:foreign"}]) { + const changedRetry = {...retry, ...patch}; + await assert.rejects(readQuotaSettlement({...input, refresh_retry: changedRetry, + writeback_candidate: {...candidate, run: {...run, + refresh_recovery: refreshRecovery(decodeRefreshRetry(changedRetry)!, null, false, undefined, false)}}}), + {code: "refresh_task_lease_unavailable"}); + } + await assert.rejects(readQuotaSettlement({...input, refresh_retry: {...retry, unsupported_operation: true}}), + /unsupported operation field/); + const noProgress = {...input, refresh_retry: {...input.refresh_retry as JsonObject, progress_observation: null}, + writeback_candidate: {...candidate, run: {...run, progress_observation: null}}}; + await assert.rejects(readQuotaSettlement(noProgress), {code: "refresh_task_lease_unavailable"}, + "Proposed accepted=true cannot replace the typed observation"); + for (const patch of [{refresh_retry: undefined}, {resume_lease_candidate: {requested_proof: null}}, + {infer_turn_instance_id: true}, {allow_unbound_binding: true}, {resolve_original_binding: true}, + {pending_closeout: true}, {spend_precheck: true}, {tolerate_malformed_history: true}, + {replan_obligation_id: selectedWritebackObligation}, {native_task_lease_cleanup: cleanupRequest}]) { + await assert.rejects(readQuotaSettlement({...input, ...patch})); + } + assert.deepEqual(await readFile(f.log), logBefore); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: semantic writeback requires actual selected admission and current Todo lifetime`, async () => { + const f = await todoSemanticFixture(provider); + try { + const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + for (const patch of [{must_attempt_work: false}, {should_run: false}, {ok: false}, + {quiet_noop_allowed: true}, {closeout_required: false}, {delivery_allowed: true}, + {interaction_mode: "control_plane_self_repair"}, {semantic_replan_obligation_id: ""}, + {semantic_replan_review_kind: undefined}, {prepared_lease_required: true}, + {prepared_lease_key_sha256: "a".repeat(64)}, {prepared_lease_version: 119}, {prepared_lease_epoch: 50}]) { + await writeFile(f.log, JSON.stringify({...f.admission, details: {...f.admission.details as JsonObject, ...patch}}) + "\n"); + await assert.rejects(readQuotaSettlement(input), JSON.stringify(patch)); + } + } finally {await rm(f.root, {recursive: true, force: true});} + const {todo} = await import("./shadow_file_fixture.ts"); + for (const selected of [{...todo(todoId), claimed_by: "foreign-agent"}, + {...todo(todoId, "done"), claimed_by: null}, {...todo(todoId, "deferred"), claimed_by: agentId}]) { + const denied = await todoSemanticFixture(provider, selected); + try { + await assert.rejects(readQuotaSettlement({...request(denied.root), ...todoSemanticWriteback()}), + {code: "refresh_task_lease_unavailable"}); + } finally {await rm(denied.root, {recursive: true, force: true});} + } + }); +} + +test("semantic admission follows one causal source and cannot substitute another obligation", () => { + const admitted = todoSemanticAdmission(); + const reservation = {...admitted, event_id: "semantic-reserved", details: {...admitted.details as JsonObject, + closeout_required: false, should_run: false, must_attempt_work: false, quiet_noop_allowed: true}}; + const causal: JsonObject = {...admitted, details: {...admitted.details as JsonObject, settlement_receipt_revision: "work_admission"}, + causality: {source_event_id: reservation.event_id, caused_by: reservation.event_id}}; + const renewed = {...causal, event_id: "semantic-readiness", details: {...causal.details as JsonObject, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: causal.event_id, caused_by: causal.event_id}}; + assert.deepEqual(heartbeatSemanticAdmissionEvent([reservation, causal, renewed], renewed, identity), causal); + assert.equal(heartbeatSemanticAdmissionEvent([reservation], reservation, identity), null); + for (const patch of [{source_event_id: "missing", caused_by: "missing"}, + {source_event_id: causal.event_id, caused_by: "another-parent"}]) { + const invalid = {...renewed, causality: patch}; + assert.equal(heartbeatSemanticAdmissionEvent([reservation, causal, invalid], invalid, identity), null); + } + const changed = {...causal, event_id: "semantic-replacement", details: {...causal.details as JsonObject, + semantic_replan_obligation_id: "replan-2222222222222222"}}; + assert.throws(() => heartbeatSemanticAdmissionEvent([causal, changed], changed, identity), + {code: "heartbeat_receipt_identity_conflict"}); +}); From 09a6c5097f36318252627acb8279cc5ceb91a5e4 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 10:59:21 +0800 Subject: [PATCH 08/12] fix(quota): preserve the original semantic admission source Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 2 + .../monorepo-distribution-split-v0.zh-CN.md | 2 + docs/reference/protocols/turn-envelope-v0.md | 5 + .../quota/heartbeat_receipt_identity.ts | 26 ++-- .../quota/semantic_replan_guard.ts | 121 ++++++++++++++++++ .../quota/settlement_readback.ts | 101 +-------------- .../work_items/replan_semantics.ts | 13 +- .../content_digest_single_owner.test.ts | 2 +- .../quota_settlement_readback.test.ts | 85 ++++++++++++ 9 files changed, 238 insertions(+), 119 deletions(-) create mode 100644 loopx/control_plane/quota/semantic_replan_guard.ts diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index d64a4f8a00..f6e869350e 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4830,6 +4830,8 @@ view shows its independently checked result in the same conversation. shared maintenance marker and retains Goal/source ownership through JSON, Markdown and index durability; direct provider writes bypassing the marker remain outside that guarantee. + Uncaused repeats preserve the complete typed review/capability source and + earliest admission; the same obligation id cannot authorize a source rewrite. - **Source evidence:** CLI tests cover both local providers, stale/foreign proofs, ordinary parity and zero completion/debit. Recovery tests inject interruption after JSON, Markdown, index and receipt durability. SourceSession diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index 9e1a0e76f6..06a68dd3aa 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3730,6 +3730,8 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 lease。受支持 writer 在共享 maintenance marker 内重新读取,并持有 Goal/source ownership 直到 JSON、Markdown、index 持久化;绕过 marker 的直接 provider 写仍不属于此保证。 + 无因果链接的重复记录保留完整 typed review/capability 来源与最早准入;相同 + obligation id 不能授权来源改写。 - **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index d23bc945c5..4514522fc1 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -128,6 +128,9 @@ effect scope, never an execution lease. The supported writer rereads inside the shared maintenance marker and holds Goal/source ownership and the marker through Run JSON, Markdown and index durability. Direct provider writes bypassing that marker are outside this canonical writer guarantee. +Uncaused repeated semantic admissions must retain the complete typed review or +capability source. Exact repeats retain the earliest admission; a later receipt +cannot change that source merely by retaining the same obligation id. `refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / @@ -179,6 +182,8 @@ resume 仍拒绝。没有业务 lease 的 Todo-bound replan 须证明原 selecte maintenance marker 内重新读取,并持有 Goal/source ownership 与 marker 直到 Run JSON、Markdown、index 持久化完成。绕过 marker 的直接 provider 写不属于此 canonical writer 保证。 +无因果链接的重复 semantic admission 须保留完整 typed review/capability 来源。 +完全相同的重复记录保留最早准入;后续收据不能仅凭相同 obligation id 改写来源。 Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. diff --git a/loopx/control_plane/quota/heartbeat_receipt_identity.ts b/loopx/control_plane/quota/heartbeat_receipt_identity.ts index 8491632a1b..ec34f54219 100644 --- a/loopx/control_plane/quota/heartbeat_receipt_identity.ts +++ b/loopx/control_plane/quota/heartbeat_receipt_identity.ts @@ -20,9 +20,10 @@ import { jsonObject, requireJsonObject, requireNonEmptyString } from "../runtime import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; import type {ReceiptBoundReplayPhase} from "./settlement_phase.ts"; import type {ProviderEffect} from "../turn_driver/settlement_provider.ts"; +import {normalizeHeartbeatReplanObligationId, projectSemanticReplanGuard} from "./semantic_replan_guard.ts"; +export {normalizeHeartbeatReplanObligationId} from "./semantic_replan_guard.ts"; const TODO_ID_PATTERN = /^todo_[a-z0-9_-]{3,64}$/; -const REPLAN_OBLIGATION_ID_PATTERN = /^replan-[a-f0-9]{16}$/; /** Public diagnostic preserved from the retired Python identity rule. */ const IDENTITY_CONFLICT_CODE = "heartbeat_receipt_identity_conflict"; @@ -44,15 +45,6 @@ export function normalizeHeartbeatTodoId(value: unknown): string | null { return candidate && TODO_ID_PATTERN.test(candidate) ? candidate : null; } -export function normalizeHeartbeatReplanObligationId( - value: unknown, -): string | null { - const candidate = String(value ?? "").trim(); - return candidate && REPLAN_OBLIGATION_ID_PATTERN.test(candidate) - ? candidate - : null; -} - export interface HeartbeatReceiptBinding extends JsonObject { binding_kind: "todo" | "autonomous_replan" | "request"; /** The Todo id or autonomous replan obligation id, per `binding_kind`. */ @@ -475,6 +467,20 @@ function selectHeartbeatValidationEvent( } return first ?? validation; } + if (semanticAdmission && positive && !Object.hasOwn(validationDetails, "settlement_receipt_revision") + && !Object.hasOwn(validation, "causality")) { + const originalGuard = projectSemanticReplanGuard(validationDetails); + let first: JsonObject | null = null; + for (const [index, prior] of scoped.entries()) { + if (index >= position) break; + if (heartbeatSemanticWorkAdmission(prior, identity) === null) continue; + if (!isDeepStrictEqual(projectSemanticReplanGuard(heartbeatReceiptDetails(prior)), originalGuard)) { + throw receiptIdentityConflict("an uncaused semantic receipt cannot redefine this Turn's original guard"); + } + first ??= prior; + } + return first ?? validation; + } return preparedAdmission ?? validation; } const lineage = jsonObject(validation.causality); diff --git a/loopx/control_plane/quota/semantic_replan_guard.ts b/loopx/control_plane/quota/semantic_replan_guard.ts new file mode 100644 index 0000000000..980473a07d --- /dev/null +++ b/loopx/control_plane/quota/semantic_replan_guard.ts @@ -0,0 +1,121 @@ +/** Shared typed semantic receipt guard; no settlement or replan decision imports. */ +import type { JsonObject } from "../effect_program.ts"; +import { createHash } from "node:crypto"; +import { BARE_SHA256_PATTERN } from "../content_digest.ts"; +import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { jsonObject, requireNonEmptyString } from "../runtime_decode.ts"; + +const REPLAN_OBLIGATION_ID_PATTERN = /^replan-[a-f0-9]{16}$/; +export const SEMANTIC_REPLAN_GUARD_SCHEMA = "semantic_replan_guard_v0"; + +export const AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID = "autonomous_replan_default_v0"; +export const AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION = + "stop only the affected path when required access, destructive git, production actions, " + + "or owner-only decisions exceed existing user authority. Keep authorized private reads " + + "within their storage and publication boundaries; do not expose credentials or disclose " + + "private material beyond that authority. Continue independent authorized replan work " + + "while honoring explicit stops"; +export const AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION = + "stop this replan path until a compatible reader validates its digest-bound review guard"; + +export function normalizeHeartbeatReplanObligationId( + value: unknown, +): string | null { + const candidate = String(value ?? "").trim(); + return candidate && REPLAN_OBLIGATION_ID_PATTERN.test(candidate) + ? candidate + : null; +} + +const normalizeReplanObligationId = normalizeHeartbeatReplanObligationId; + +export function projectSemanticReplanGuard( + receiptDetails: JsonObject, +): JsonObject { + const hasCapability = ["semantic_replan_capability_id", "semantic_replan_gap_id", "semantic_replan_frontier_revision"] + .some(field => Object.hasOwn(receiptDetails, field)); + const hasReview = ["semantic_replan_review_kind", "semantic_replan_review_stop_condition", + "semantic_replan_review_stop_condition_sha256", "semantic_replan_review_stop_contract_id"] + .some(field => Object.hasOwn(receiptDetails, field)); + if (hasCapability && hasReview) throw new EffectRuntimeRequestError( + "a review source cannot replace a capability guard", "malformed_settlement_state"); + if (!Object.hasOwn(receiptDetails, "semantic_replan_obligation_id")) { + if (hasCapability) throw new EffectRuntimeRequestError("a capability guard requires a selected obligation", "malformed_settlement_state"); + if (hasReview) throw new EffectRuntimeRequestError("a review source requires a selected obligation", "malformed_settlement_state"); + return { + schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, + scope: "legacy_unscoped", + selected_obligation_id: null, + }; + } + const rawObligationId = receiptDetails.semantic_replan_obligation_id; + if (rawObligationId === "" || rawObligationId === null) { + if (hasCapability) throw new EffectRuntimeRequestError("a capability guard requires a selected obligation", "malformed_settlement_state"); + if (hasReview) throw new EffectRuntimeRequestError("a review source requires a selected obligation", "malformed_settlement_state"); + return { + schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, + scope: "turn_guard", + selected_obligation_id: null, + }; + } + const selectedObligationId = normalizeReplanObligationId(rawObligationId); + if (selectedObligationId === null) { + throw new EffectRuntimeRequestError( + "heartbeat receipt semantic replan guard is malformed", + "malformed_settlement_state", + ); + } + const reviewKind = hasReview ? requireNonEmptyString(receiptDetails.semantic_replan_review_kind, "receipt review kind") : null; + const stop = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_condition") + ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_condition, "receipt review stop condition") : null; + const stopDigest = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_condition_sha256") + ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_condition_sha256, + "receipt review stop condition digest") : null; + const stopContractId = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_contract_id") + ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_contract_id, + "receipt review stop contract id") : null; + if (hasReview && stop === null && stopDigest === null) throw new EffectRuntimeRequestError( + "a receipt review requires a stop condition or digest", "malformed_settlement_state"); + if (stop && stop.length > 240) throw new EffectRuntimeRequestError( + "receipt review stop condition exceeds scalar metadata bounds", "malformed_settlement_state"); + if (stopDigest !== null && !BARE_SHA256_PATTERN.test(stopDigest)) throw new EffectRuntimeRequestError( + "receipt review stop condition digest is malformed", "malformed_settlement_state"); + if (stopContractId !== null && + (stopContractId !== AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID || stopDigest === null || + createHash("sha256").update(AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, "utf8").digest("hex") !== stopDigest)) { + throw new EffectRuntimeRequestError( + "receipt review stop contract does not match its digest", "malformed_settlement_state"); + } + if (stop !== null && stopDigest !== null && + stop !== AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION && + createHash("sha256").update(stop, "utf8").digest("hex") !== stopDigest) { + throw new EffectRuntimeRequestError( + "receipt review stop condition digest does not match its value", "malformed_settlement_state"); + } + return { + schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, + scope: "turn_guard", + selected_obligation_id: selectedObligationId, + ...(hasReview ? {selected_review_source: {kind: reviewKind, + ...(stop === null ? {} : {stop_condition: stop}), + ...(stopDigest === null ? {} : {stop_condition_sha256: stopDigest}), + ...(stopContractId === null ? {} : {stop_contract_id: stopContractId})}} : {}), + ...(hasCapability ? { + selected_capability_guard: decodeCapabilityGuard({schema_version: "semantic_replan_capability_guard_v0", + capability_id: receiptDetails.semantic_replan_capability_id, gap_id: receiptDetails.semantic_replan_gap_id, + frontier_revision: receiptDetails.semantic_replan_frontier_revision}), + } : {}), + }; +} + +function decodeCapabilityGuard(value: unknown): JsonObject { + const guard = jsonObject(value); + if (!guard || guard.schema_version !== "semantic_replan_capability_guard_v0" + || typeof guard.capability_id !== "string" || !/^[a-z][a-z0-9-]{0,63}$/.test(guard.capability_id) + || typeof guard.gap_id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(guard.gap_id) + || typeof guard.frontier_revision !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(guard.frontier_revision) + || Object.keys(guard).some(key => !["schema_version", "capability_id", "gap_id", "frontier_revision"].includes(key))) { + throw new EffectRuntimeRequestError("heartbeat receipt capability guard is malformed", "malformed_settlement_state"); + } + return guard; +} diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 621c851620..8d7d1d540b 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { isAbsolute, join } from "node:path"; import { readReceiptLogSnapshot } from "../runtime/receipt_log_snapshot.ts"; @@ -24,7 +23,6 @@ import { type SettlementResult, } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; -import { BARE_SHA256_PATTERN } from "../content_digest.ts"; import { hasMonitorPollTransaction } from "./monitor_poll_commit.ts"; import { readQuotaSpendEventFromRun } from "./accounting_artifact_transaction.ts"; import { @@ -88,11 +86,8 @@ import { import { refreshExternalDelivery } from "./refresh_external_delivery.ts"; import { BLOCKED_WAIT_REQUEST_SCHEMA, prepareBlockedWait, RECEIPT_BOUND_WAIT_REQUEST_SCHEMA, projectReceiptBoundWait, isMonitorUnavailableWait } from "./blocked_wait.ts"; import {nativeChildReportAdmission} from "../capabilities/native_child_admission.ts"; -import { - AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, - AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID, - AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION, -} from "../work_items/replan_semantics.ts"; +import {projectSemanticReplanGuard} from "./semantic_replan_guard.ts"; +export {projectSemanticReplanGuard, SEMANTIC_REPLAN_GUARD_SCHEMA} from "./semantic_replan_guard.ts"; import { parseQuotaAccountingOwner, quotaGoalRef, @@ -106,7 +101,6 @@ export const QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA = "loopx_quota_settlement_readback_request_v0"; export const QUOTA_SETTLEMENT_READBACK_RESULT_SCHEMA = "loopx_quota_settlement_readback_result_v0"; -export const SEMANTIC_REPLAN_GUARD_SCHEMA = "semantic_replan_guard_v0"; export const NATIVE_TASK_LEASE_CLEANUP_REQUEST_SCHEMA = "native_task_lease_cleanup_request_v0"; export const NATIVE_TASK_LEASE_CLEANUP_SCHEMA = "native_task_lease_cleanup_v0"; @@ -756,85 +750,6 @@ function runEffectMatches( quotaSpendMetadataMatches(run.quota_spend_commit, effectRef, expectedEffectRef)); } -export function projectSemanticReplanGuard( - receiptDetails: JsonObject, -): JsonObject { - const hasCapability = ["semantic_replan_capability_id", "semantic_replan_gap_id", "semantic_replan_frontier_revision"] - .some(field => Object.hasOwn(receiptDetails, field)); - const hasReview = ["semantic_replan_review_kind", "semantic_replan_review_stop_condition", - "semantic_replan_review_stop_condition_sha256", "semantic_replan_review_stop_contract_id"] - .some(field => Object.hasOwn(receiptDetails, field)); - if (hasCapability && hasReview) throw new EffectRuntimeRequestError( - "a review source cannot replace a capability guard", "malformed_settlement_state"); - if (!Object.hasOwn(receiptDetails, "semantic_replan_obligation_id")) { - if (hasCapability) throw new EffectRuntimeRequestError("a capability guard requires a selected obligation", "malformed_settlement_state"); - if (hasReview) throw new EffectRuntimeRequestError("a review source requires a selected obligation", "malformed_settlement_state"); - return { - schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, - scope: "legacy_unscoped", - selected_obligation_id: null, - }; - } - const rawObligationId = receiptDetails.semantic_replan_obligation_id; - if (rawObligationId === "" || rawObligationId === null) { - if (hasCapability) throw new EffectRuntimeRequestError("a capability guard requires a selected obligation", "malformed_settlement_state"); - if (hasReview) throw new EffectRuntimeRequestError("a review source requires a selected obligation", "malformed_settlement_state"); - return { - schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, - scope: "turn_guard", - selected_obligation_id: null, - }; - } - const selectedObligationId = normalizeReplanObligationId(rawObligationId); - if (selectedObligationId === null) { - throw new EffectRuntimeRequestError( - "heartbeat receipt semantic replan guard is malformed", - "malformed_settlement_state", - ); - } - const reviewKind = hasReview ? requireNonEmptyString(receiptDetails.semantic_replan_review_kind, "receipt review kind") : null; - const stop = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_condition") - ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_condition, "receipt review stop condition") : null; - const stopDigest = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_condition_sha256") - ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_condition_sha256, - "receipt review stop condition digest") : null; - const stopContractId = Object.hasOwn(receiptDetails, "semantic_replan_review_stop_contract_id") - ? requireNonEmptyString(receiptDetails.semantic_replan_review_stop_contract_id, - "receipt review stop contract id") : null; - if (hasReview && stop === null && stopDigest === null) throw new EffectRuntimeRequestError( - "a receipt review requires a stop condition or digest", "malformed_settlement_state"); - if (stop && stop.length > 240) throw new EffectRuntimeRequestError( - "receipt review stop condition exceeds scalar metadata bounds", "malformed_settlement_state"); - if (stopDigest !== null && !BARE_SHA256_PATTERN.test(stopDigest)) throw new EffectRuntimeRequestError( - "receipt review stop condition digest is malformed", "malformed_settlement_state"); - if (stopContractId !== null && - (stopContractId !== AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID || stopDigest === null || - createHash("sha256").update(AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, "utf8").digest("hex") !== stopDigest)) { - throw new EffectRuntimeRequestError( - "receipt review stop contract does not match its digest", "malformed_settlement_state"); - } - if (stop !== null && stopDigest !== null && - stop !== AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION && - createHash("sha256").update(stop, "utf8").digest("hex") !== stopDigest) { - throw new EffectRuntimeRequestError( - "receipt review stop condition digest does not match its value", "malformed_settlement_state"); - } - return { - schema_version: SEMANTIC_REPLAN_GUARD_SCHEMA, - scope: "turn_guard", - selected_obligation_id: selectedObligationId, - ...(hasReview ? {selected_review_source: {kind: reviewKind, - ...(stop === null ? {} : {stop_condition: stop}), - ...(stopDigest === null ? {} : {stop_condition_sha256: stopDigest}), - ...(stopContractId === null ? {} : {stop_contract_id: stopContractId})}} : {}), - ...(hasCapability ? { - selected_capability_guard: decodeCapabilityGuard({schema_version: "semantic_replan_capability_guard_v0", - capability_id: receiptDetails.semantic_replan_capability_id, gap_id: receiptDetails.semantic_replan_gap_id, - frontier_revision: receiptDetails.semantic_replan_frontier_revision}), - } : {}), - }; -} - /** Upgrade a legacy bound review receipt without changing its Turn identity. * * Older receipts retained the selected replan obligation but not the review @@ -877,18 +792,6 @@ function requalifyHeartbeatReceipt( return {append: true, details: upgraded}; } -function decodeCapabilityGuard(value: unknown): JsonObject { - const guard = jsonObject(value); - if (!guard || guard.schema_version !== "semantic_replan_capability_guard_v0" - || typeof guard.capability_id !== "string" || !/^[a-z][a-z0-9-]{0,63}$/.test(guard.capability_id) - || typeof guard.gap_id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(guard.gap_id) - || typeof guard.frontier_revision !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(guard.frontier_revision) - || Object.keys(guard).some(key => !["schema_version", "capability_id", "gap_id", "frontier_revision"].includes(key))) { - throw new EffectRuntimeRequestError("heartbeat receipt capability guard is malformed", "malformed_settlement_state"); - } - return guard; -} - function runMatchesBinding(run: JsonObject, identity: SettlementIdentity): boolean { return normalizeTodoId(run.todo_id) === identity.todo_id && normalizeReplanObligationId(run.replan_obligation_id) === diff --git a/loopx/control_plane/work_items/replan_semantics.ts b/loopx/control_plane/work_items/replan_semantics.ts index 31e62ea64f..99fbc5d473 100644 --- a/loopx/control_plane/work_items/replan_semantics.ts +++ b/loopx/control_plane/work_items/replan_semantics.ts @@ -10,6 +10,10 @@ import { id as todoId, readResume, taskClass } from "../todos/read_codec.ts"; import { indexInferredSuccessors } from "../todos/succession.ts"; import {normalizeStrideDecision, normalizeStrideStrategy, STRIDE_DECISION_SCHEMA, strideControlContract, strideReviewMismatch} from "./stride_policy.ts"; import { projectProgressDelta } from "./progress_delta.ts"; +import {AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID} + from "../quota/semantic_replan_guard.ts"; +export {AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION, AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID, + AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION} from "../quota/semantic_replan_guard.ts"; const PROGRESS_OUTCOMES = [ "new_surface", "new_hypothesis", "new_probe_family", "new_runnable_successor", @@ -38,15 +42,6 @@ const ACCEPTANCE_HOLD_TRIGGERS = new Set(["goal_acceptance_stale", "goal_accepta // same scalar receipt can retain a mixed review after its work links change. const SCALAR_REVIEW_KINDS = new Set(["goal_outcome_uncovered", "vision_acceptance_gap", "monitor_no_change_streak"]); const RECONSTRUCTIBLE_REVIEW_KINDS = new Set([...SCALAR_REVIEW_KINDS, "periodic_review_due"]); -export const AUTONOMOUS_REPLAN_DEFAULT_STOP_CONTRACT_ID = "autonomous_replan_default_v0"; -export const AUTONOMOUS_REPLAN_DEFAULT_STOP_CONDITION = - "stop only the affected path when required access, destructive git, production actions, " + - "or owner-only decisions exceed existing user authority. Keep authorized private reads " + - "within their storage and publication boundaries; do not expose credentials or disclose " + - "private material beyond that authority. Continue independent authorized replan work " + - "while honoring explicit stops"; -export const AUTONOMOUS_REPLAN_DIGEST_COMPATIBILITY_STOP_CONDITION = - "stop this replan path until a compatible reader validates its digest-bound review guard"; const EXTERNAL_REVIEW_TRIGGERS = new Set(["external_progress_review_drift"]); export const FRESH_PATH_DISPOSITIONS = new Set(["continue", "no_change", "replan"]); const PROGRESS_CLI_ARGS = "--progress-result-class --progress-surface-id --progress-hypothesis-id --progress-probe-kind --progress-evidence-id "; diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index 1c418417c9..94d102be27 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -165,7 +165,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/quota/execution_loss_closeout_record.ts", "control_plane/quota/heartbeat_receipt_identity.ts", "control_plane/quota/refresh_external_delivery.ts", - "control_plane/quota/settlement_readback.ts", + "control_plane/quota/semantic_replan_guard.ts", "control_plane/quota/turn_envelope.ts", "control_plane/runtime/checkpoint_artifacts.ts", "control_plane/runtime/checkpoint_retention.ts", diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index bcb5003305..318af55ef5 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -3064,3 +3064,88 @@ test("semantic admission follows one causal source and cannot substitute another assert.throws(() => heartbeatSemanticAdmissionEvent([causal, changed], changed, identity), {code: "heartbeat_receipt_identity_conflict"}); }); + + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: uncaused scalar receipt cannot downgrade the original capability admission`, async () => { + const f = await todoSemanticFixture(provider); + try { + const originalDetails: JsonObject = {...f.admission.details as JsonObject}; + delete originalDetails.semantic_replan_review_kind; + delete originalDetails.semantic_replan_review_stop_condition; + // Use the existing capability fixture's typed transport shape. A and B + // retain the exact same Goal, Agent, Todo, Turn, effect and obligation; + // the later scalar receipt has no causal authority to replace A's duty. + Object.assign(originalDetails, {semantic_replan_capability_id: "explore", + semantic_replan_gap_id: "research-composition-0123456789abcdef", + semantic_replan_frontier_revision: "research-composition-v0:0123456789abcdef"}); + const original: JsonObject = {...f.admission, event_id: "original-capability-admission", details: originalDetails}; + const later: JsonObject = {...f.admission, event_id: "uncaused-scalar-admission", + recorded_at: "2026-01-01T00:00:01Z"}; + const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + const authorityBefore = await f.store.loadAuthority(); + await writeFile(f.log, JSON.stringify(original) + "\n"); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}, + "The original capability duty must use its capability owner"); + assert.equal(Object.hasOwn(later, "causality"), false); + await writeFile(f.log, [original, later].map(row => JSON.stringify(row)).join("\n") + "\n"); + await assert.rejects(readQuotaSettlement(input), (error: unknown) => error instanceof Error && "code" in error + && (error.code === "heartbeat_receipt_identity_conflict" || error.code === "refresh_task_lease_unavailable"), + "An uncaused scalar receipt must not turn an original capability duty into generic progress permission"); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} + + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: semantic guard retains identical uncaused and causal original admissions`, async () => { + const f = await todoSemanticFixture(provider); + try { + const original: JsonObject = {...f.admission, event_id: "original-scalar-source"}; + const repeated: JsonObject = {...f.admission, event_id: "identical-uncaused-source", + recorded_at: "2026-01-01T00:00:01Z"}; + const causal: JsonObject = {...repeated, event_id: "causal-scalar-source", + details: {...repeated.details as JsonObject, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: original.event_id, caused_by: original.event_id}}; + const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + for (const later of [repeated, causal]) { + const events = [original, later]; + assert.deepEqual(heartbeatSemanticAdmissionEvent(events, later, identity), original); + await writeFile(f.log, events.map(row => JSON.stringify(row)).join("\n") + "\n"); + const result = await readQuotaSettlement(input); + assert.equal((result.writeback_qualification as {payload: {ok: boolean}}).payload.ok, true); + } + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: uncaused semantic source changes reject under the same obligation`, async () => { + const f = await todoSemanticFixture(provider); + try { + const scalar: JsonObject = {...f.admission, event_id: "original-scalar-source"}; + const capDetails: JsonObject = {...f.admission.details as JsonObject}; + delete capDetails.semantic_replan_review_kind; + delete capDetails.semantic_replan_review_stop_condition; + Object.assign(capDetails, {semantic_replan_capability_id: "explore", + semantic_replan_gap_id: "research-composition-0123456789abcdef", + semantic_replan_frontier_revision: "research-composition-v0:0123456789abcdef"}); + const capability: JsonObject = {...f.admission, event_id: "original-capability-source", details: capDetails}; + const cases: {original: JsonObject; patch: JsonObject}[] = [ + {original: scalar, patch: {semantic_replan_review_kind: "vision_acceptance_gap"}}, + {original: scalar, patch: {semantic_replan_review_stop_condition: "Stop at destructive owner-only decisions."}}, + {original: capability, patch: {semantic_replan_capability_id: "research"}}, + {original: capability, patch: {semantic_replan_gap_id: "research-composition-fedcba9876543210"}}, + {original: capability, patch: {semantic_replan_frontier_revision: "research-composition-v0:fedcba9876543210"}}, + ]; + const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + for (const {original, patch} of cases) { + const later: JsonObject = {...original, event_id: "changed-uncaused-source", recorded_at: "2026-01-01T00:00:01Z", + details: {...original.details as JsonObject, ...patch}}; + await writeFile(f.log, [original, later].map(row => JSON.stringify(row)).join("\n") + "\n"); + assert.throws(() => heartbeatSemanticAdmissionEvent([original, later], later, identity), + {code: "heartbeat_receipt_identity_conflict"}, JSON.stringify(patch)); + await assert.rejects(readQuotaSettlement(input), {code: "heartbeat_receipt_identity_conflict"}, JSON.stringify(patch)); + } + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} From a2c303cf6759ff40def3c8e38969f20ce756f9c9 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 12:03:24 +0800 Subject: [PATCH 09/12] fix(quota): retain exact negative checkpoints and readonly recovery Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 16 + .../monorepo-distribution-split-v0.zh-CN.md | 13 +- docs/reference/protocols/turn-envelope-v0.md | 38 ++- .../native_chat/steward_monitor.py | 11 + .../native_chat/steward_reports.py | 91 +++++- .../capabilities/native_chat/steward_turn.py | 54 +++- loopx/cli_commands/turn_rendering.py | 31 +- loopx/control_plane/quota/settlement.py | 11 +- .../quota/settlement_readback.ts | 152 +++++++++- .../turn_driver/turn_journal_query.ts | 21 +- loopx/state_refresh.py | 15 +- .../test_quota_execution_preparation.py | 42 ++- .../heartbeat_resume_lease.test.ts | 86 +++++- .../quota_settlement_readback.test.ts | 275 +++++++++++++++++- .../turn_journal_query.test.ts | 29 +- ...t_steward_report_recovery_and_validator.py | 149 +++++++++- 16 files changed, 975 insertions(+), 59 deletions(-) diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index f6e869350e..40e03434fd 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4826,12 +4826,28 @@ view shows its independently checked result in the same conversation. original semantic writeback; business resume still rejects it. A Todo-bound replan without a business lease proves its selected obligation and semantic evidence. Required preparation without an original positive proof rejects. + An ordinary typed blocked gap with exact live preparation preserves the + Vision owner's genuine negative checkpoint, including `missing_required`; + it does not claim satisfied acceptance. Every builder-owned field and its + presence must match the actual inputs. Independently appended acceptance-gap + diagnostics remain read-only and grant no authority. Accountable delivery, + retired writeback and no-business obligation discharge retain their satisfied + checkpoint requirements. The scope grants no execution lease. The supported writer rereads inside the shared maintenance marker and retains Goal/source ownership through JSON, Markdown and index durability; direct provider writes bypassing the marker remain outside that guarantee. Uncaused repeats preserve the complete typed review/capability source and earliest admission; the same obligation id cannot authorize a source rewrite. +- **Recovery readback:** a frozen report retains the typed original-lease + refusal. An optional, default-off settlement attachment verifies its original + owner, identity and journal before providing journal and task-lease inspection + commands. Missing or conflicting evidence withholds the commands. The + receiver records an idempotent local notice and the same health packet; + inspection grants no execution or administrative write authority. Modern + positive prepared admission may explain the existing explicit administrative + closeout and its separate current-control-writer requirement; bounded legacy + evidence supports inspection only. - **Source evidence:** CLI tests cover both local providers, stale/foreign proofs, ordinary parity and zero completion/debit. Recovery tests inject interruption after JSON, Markdown, index and receipt durability. SourceSession diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index 06a68dd3aa..f68c883fdf 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3726,12 +3726,23 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 receipt/Goal/Todo/Turn、当前 generation、progress 与 Vision/checkpoint。 Exact supersede 只允许原 semantic writeback,business resume 仍拒绝。 无业务 lease 的 Todo-bound replan 须证明 selected obligation 与语义证据; - 要求 preparation 却缺少原 positive proof 时拒绝。该 scope 不授予 execution + 要求 preparation 却缺少原 positive proof 时拒绝。持有 exact live preparation + 的普通 typed blocked gap 保留 Vision owner 的真实 negative checkpoint, + 包括 `missing_required`,不宣称验收满足。所有 builder-owned 字段及其存在性 + 必须与实际输入一致;独立追加的 acceptance-gap 诊断仅供读取,不授予权限。 + Accountable delivery、retired writeback 与无业务 obligation discharge 保留 + satisfied checkpoint 要求。该 scope 不授予 execution lease。受支持 writer 在共享 maintenance marker 内重新读取,并持有 Goal/source ownership 直到 JSON、Markdown、index 持久化;绕过 marker 的直接 provider 写仍不属于此保证。 无因果链接的重复记录保留完整 typed review/capability 来源与最早准入;相同 obligation id 不能授权来源改写。 +- **恢复回读:**frozen report 保留 typed 原租约拒绝。默认关闭的可选 settlement + attachment 核验原 owner、identity 与 journal 后,仅提供 journal 与 task-lease + 检查命令;证据缺失或冲突时不提供命令。receiver 记录幂等的本地提示与同一 + health packet;检查不授予 execution 或行政写权限。现代 positive prepared + admission 可以说明既有显式行政收尾及其独立的当前 control-writer 要求; + 有限 legacy 证据仅支持检查。 - **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 4514522fc1..2793ed27f9 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -123,7 +123,13 @@ generation, typed progress, Vision and checkpoint. Exact supersession permits only that original Turn's semantic writeback; business resume still rejects it. A Todo-bound replan with no business lease must prove its original selected obligation and actual semantic evidence. Required preparation without a positive -original proof is rejected. Qualification returns only the durable-writeback +original proof is rejected. An ordinary typed blocked gap with exact live +preparation may record the Vision owner's actual `missing_required` negative +checkpoint. All builder-owned fields and their presence must match the actual +inputs; separately appended acceptance-gap diagnostics remain read-only and +grant no authority. Accountable delivery, retired writeback and no-business +obligation discharge retain their satisfied-checkpoint requirements. +Qualification returns only the durable-writeback effect scope, never an execution lease. The supported writer rereads inside the shared maintenance marker and holds Goal/source ownership and the marker through Run JSON, Markdown and index durability. Direct provider writes bypassing that @@ -132,6 +138,19 @@ Uncaused repeated semantic admissions must retain the complete typed review or capability source. Exact repeats retain the earliest admission; a later receipt cannot change that source merely by retaining the same obligation id. +`quota.settlement.read` may request `include_readonly_recovery: true`, disabled +by default and mutually exclusive with operation-qualification requests. It +requires the original Goal owner, Agent, Todo and Turn plus a verified journal +binding. The `settlement_recovery` attachment preserves the original settlement +readback and provides only `turn inspect-journal` and `task-lease inspect` argv. +Missing or conflicting evidence withholds these actions. It exposes no execution +proof, lease claim, reactivation or administrative-closeout argv. Only verified +modern positive prepared admission may explain the existing explicit closeout's +separate current-control-writer requirement; legacy evidence supports inspection +only. A frozen-report receiver retains its typed lease refusal, publishes an +idempotent local notice and exposes the same failure packet in health. Reading +the attachment or repeating the notice performs no business or accounting work. + `refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / semantic_closeout 证据及当前 control writer 的确切租约 key/version。TS owner 在真实 @@ -178,13 +197,28 @@ Todo/Turn/effect、Goal owner、当前 canonical generation、typed progress、V 与 checkpoint。Exact supersede 只允许原 Turn 的 semantic writeback,business resume 仍拒绝。没有业务 lease 的 Todo-bound replan 须证明原 selected obligation 与实际语义证据;要求 preparation 却缺少 positive original proof 时拒绝。返回值 -仅为 durable-writeback effect scope,不授予 execution lease。受支持 writer 在共享 +仅为 durable-writeback effect scope,不授予 execution lease。持有 exact live +preparation 的普通 typed blocked gap 可以记录 Vision owner 的真实 +`missing_required` negative checkpoint;所有 builder-owned 字段及其存在性须与 +实际输入一致。独立追加的 acceptance-gap 诊断仅供读取,不授予权限。 +Accountable delivery、retired writeback 与无业务 obligation discharge 保留 +satisfied checkpoint 要求。受支持 writer 在共享 maintenance marker 内重新读取,并持有 Goal/source ownership 与 marker 直到 Run JSON、Markdown、index 持久化完成。绕过 marker 的直接 provider 写不属于此 canonical writer 保证。 无因果链接的重复 semantic admission 须保留完整 typed review/capability 来源。 完全相同的重复记录保留最早准入;后续收据不能仅凭相同 obligation id 改写来源。 +`quota.settlement.read` 可请求默认关闭的 `include_readonly_recovery: true`, +与操作资格请求互斥。它要求原 Goal owner、Agent、Todo、Turn 及已验证的 journal +binding。`settlement_recovery` attachment 保留原结算回读,仅提供 +`turn inspect-journal` 与 `task-lease inspect` argv;证据缺失或冲突时不提供操作。 +它不输出 execution proof、lease claim、reactivation 或行政收尾 argv。只有已验证 +的现代 positive prepared admission 可以说明既有显式收尾所需的独立当前 +control-writer 要求;legacy 证据仅支持检查。frozen-report receiver 保留 typed +租约拒绝,记录幂等本地提示,并在 health 暴露同一 failure packet。读取 attachment +或重复提示不执行任何业务或记账。 + Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. User task time requirements remain task context. Caller-private deadlines diff --git a/loopx/capabilities/native_chat/steward_monitor.py b/loopx/capabilities/native_chat/steward_monitor.py index 4788daeb3b..b0ddaef5a2 100644 --- a/loopx/capabilities/native_chat/steward_monitor.py +++ b/loopx/capabilities/native_chat/steward_monitor.py @@ -409,6 +409,9 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d from .steward_turn import StewardTurnHost bindings = getattr(getattr(controller, "project_contexts", None), "conversation_bindings", None) + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from .steward_reports import present_report_settlement_recovery + if controller.closed.is_set(): return [] service, changed = StewardMonitor(controller), [] @@ -443,6 +446,9 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d if created: changed.append(accepted) continue + except RefreshTaskLeaseUnavailable as rejected: + if not present_report_settlement_recovery(controller, rejected): + _LOG.warning("An original steward report recovery could not be presented; its original window is retained") except (OSError, ValueError, KeyError, TypeError, RuntimeError): _LOG.warning("An original steward report is unavailable; its window and other work are retained") due = service.due_monitors(actor) @@ -510,6 +516,8 @@ def pump_managed_session_monitors(controller: Any, *, cancelled=lambda: False) - provider is needed, and no model or task is created by a due observation. """ from .steward_managed_mode import ManagedSessionTurn + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from .steward_reports import present_report_settlement_recovery changed, monitor = [], StewardMonitor(controller) cache = controller._steward_monitor_next_check @@ -547,6 +555,9 @@ def pump_managed_session_monitors(controller: Any, *, cancelled=lambda: False) - if created: changed.append(accepted) continue + except RefreshTaskLeaseUnavailable as rejected: + if not present_report_settlement_recovery(controller, rejected): + _LOG.warning("A managed Session report recovery could not be presented; its original window is retained") except (OSError, ValueError, KeyError, TypeError, RuntimeError): _LOG.warning("A managed Session report is unavailable; its original window and other work are retained") rows = [row for row in monitor.todos(request["actor"]) diff --git a/loopx/capabilities/native_chat/steward_reports.py b/loopx/capabilities/native_chat/steward_reports.py index 473b5e4ab7..98fc3fceef 100644 --- a/loopx/capabilities/native_chat/steward_reports.py +++ b/loopx/capabilities/native_chat/steward_reports.py @@ -9,6 +9,7 @@ from datetime import datetime, timezone from contextlib import contextmanager import json +import hashlib import sys from typing import Any @@ -31,6 +32,84 @@ class PendingStewardReportError(ValueError): pass +def _report_recovery_message_id(session_id: str, turn_id: str, payload: dict[str, Any]) -> str: + # The shared gate, rather than changing exception prose or read timestamps, + # identifies a notice revision. The typed refusal remains in the event. + revision = payload.get("settlement_recovery") or { + "error_code": payload.get("error_code"), "recovery_available": False} + encoded = json.dumps([session_id, turn_id, revision], sort_keys=True, ensure_ascii=False) + return "report-recovery." + hashlib.sha256(encoded.encode()).hexdigest()[:32] + + +def report_settlement_recovery_health(controller: Any, actor: dict[str, Any], window: dict[str, Any] | None) -> dict[str, Any]: + """Project a retained local refusal; this is no settlement or delivery proof.""" + from .steward_turn import report_turn + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + + original = report_turn(controller.store, actor, window) if window else None + if original is None: + return {} + request = original.get("loopx_request") or {} + if (request.get("actor") != actor or request.get("work_kind") != "report" + or request.get("report_window_id") != window["window_id"] + or request.get("report_subscription_revision") != window["subscription_revision"]): + return {} + for row in reversed(controller.store.events_after(actor["session_id"], original["turn_id"], None)): + if row.get("kind") != "native_loopx.result": + continue + payload = row.get("payload") or {} + if payload.get("status") == "committed": + return {} + if payload.get("error_code") != RefreshTaskLeaseUnavailable.code: + continue + message_id = _report_recovery_message_id(actor["session_id"], original["turn_id"], payload) + messages = controller.store.messages(actor["session_id"]) + return {"recovery_failure": payload, + "recovery_message_id": message_id if any(message.get("message_id") == message_id + and message.get("turn_id") == original["turn_id"] for message in messages) else None} + return {} + + +def present_report_settlement_recovery(controller: Any, rejected: Exception) -> bool: + """Show the shared read gate once in its original local Session only.""" + from ...control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from ...control_plane.goals.goal_ref_validation import exact_goal_ref + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from .steward_turn import StewardTurnHost, report_turn + + if not isinstance(rejected, RefreshTaskLeaseUnavailable): + return False + context, payload = getattr(rejected, "report_recovery_context", None), getattr(rejected, "payload", None) + if not isinstance(context, dict) or not isinstance(payload, dict): + return False + request, window = context["request"], context["window"] + actor, host = request["actor"], StewardTurnHost(controller) + host._observe(actor) + goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) + if actor.get("goal_instance_id") is not None else None) + admission = FirstPartyHostGoalAdmission.for_plan(registry_path=controller.registry_path, + goal_id=actor["goal_id"], planned_goal_ref=goal_ref, agent_id=actor["agent_id"]) + with controller.lock, host._report_fence(request): + admission.require_running(operation="report_settlement_recovery_notice") + reports, owner_key = report_service(controller, actor) + original = report_turn(controller.store, actor, window) + if (not reports.delivery_current(owner_key, window) or original is None + or original["turn_id"] != context["original_turn_id"] + or original.get("loopx_request") != request): + return False + recovery = payload.get("settlement_recovery") + if isinstance(recovery, dict): + text = "本期报告的原执行暂时无法继续,原结果和窗口已保留。\n\n" + recovery["instruction"] + for command in recovery["next_cli_actions"]: + text += "\n\n```sh\n" + command + "\n```" + else: + text = "本期报告的原执行暂时无法继续,原结果和窗口已保留。恢复指引暂时无法读取,请重新读取原报告状态。" + controller.store.append_message(actor["session_id"], role="error", text=text, + turn_id=original["turn_id"], origin="native_loopx", + message_id=_report_recovery_message_id(actor["session_id"], original["turn_id"], payload)) + return True + + class StewardReports: def __init__(self, controller: Any) -> None: self.controller = controller @@ -365,14 +444,16 @@ def health(self, binding_id: str) -> dict[str, Any]: elif self._known_generation_failure(actor, journal["window"]): delivery = "generation_failed" active = resolve_active_goal_periodic_report_subscription(goal) + recovery = report_settlement_recovery_health(self.controller, actor, journal["window"] if journal else None) return {"available": True, "configured": True, "enabled": bool(active), "schedule": config.get("schedule"), "next_due_at": report_cadence_next_due_at( config.get("schedule"), now=datetime.now(timezone.utc)) if config.get("enabled") else None, "retained_window_due_at": journal["window"]["due_at"] if journal else None, "generation_id": generation, "delivery_status": delivery or "not_generated", - "execution_failed": bool(journal and self._execution_failed(actor, journal["window"])), + "execution_failed": bool(recovery or journal and self._execution_failed(actor, journal["window"])), "publication_verified": bool(journal and self._published_window(journal["window"])), - "destination": "original_owner_private_chat_full_body"} + "destination": "original_owner_private_chat_full_body", + **recovery} except (OSError, ValueError, TypeError, KeyError, StopIteration): return {"available": False, "error_code": "steward_report_unavailable"} @@ -723,14 +804,16 @@ def health(self, session_id: str) -> dict[str, Any]: returned = [row for row in self.controller.store.messages(session_id) if row["role"] == "agent" and row["turn_id"] == turn["turn_id"] and row["text"] == body] verified = published and len(returned) == 1 + recovery = report_settlement_recovery_health(self.controller, actor, window) return {"available": True, "configured": True, "enabled": bool(self.subscription(session_id)), "scheduled": bool(config.get("schedule")), "schedule": config.get("schedule"), "next_due_at": report_cadence_next_due_at( config.get("schedule"), now=datetime.now(timezone.utc)) if config.get("enabled") else None, "retained_window_due_at": window["due_at"] if window else None, "generation_complete": bool(folder and (folder / "private-generation.json").exists()), - "execution_failed": failed, + "execution_failed": bool(failed or recovery), "publication_verified": verified, - "report_message_id": returned[0]["message_id"] if verified else None} + "report_message_id": returned[0]["message_id"] if verified else None, + **recovery} except (OSError, ValueError, KeyError, TypeError): return {"available": False, "error_code": "managed_session_report_unavailable"} diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 8a1a9b9783..689bb34dbc 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -794,8 +794,57 @@ def replay_frozen_candidate(native_request: Mapping[str, Any], _launch_scope: An "--host", "generic-cli", "--resume-turn-key", key, "--retry-failed-turn", "--execute"]) args.host, args.session_scope = "codex-app-server", "agent" result: dict[str, Any] = {} - with (self._task_execution_lease(request, original["turn_id"], recovery=True) - if todo["status"] == "open" else nullcontext(None)) as lease: + from ...cli_commands.turn_rendering import build_turn_error_payload, render_readonly_settlement_recovery + from ...control_plane.goals.goal_ref_validation import exact_goal_ref + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from ...control_plane.quota.effect_program import SettlementIdentity + from ...control_plane.quota.settlement import read_heartbeat_settlement + from ...control_plane.turn_driver.journal_store import load_loopx_turn_plan_from_journal + + with ExitStack() as recovery_leases: + try: + lease = (recovery_leases.enter_context(self._task_execution_lease( + request, original["turn_id"], recovery=True)) if todo["status"] == "open" else None) + except RefreshTaskLeaseUnavailable as rejected: + planned = {"transaction": {"turn_key": key}} + # This invocation stopped before the native executor. Historical + # effects are carried separately by the original journal. + planned = {**planned, "effects": {name: False for name in + ("host_invoked", "state_written", "quota_spent", "scheduler_acknowledged")}} + failure = build_turn_error_payload(planned, rejected, + turn_command="run-once", journal_readback=inspection) + goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) + if actor.get("goal_instance_id") is not None else None) + try: + original_plan = load_loopx_turn_plan_from_journal(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=key) + failure = build_turn_error_payload({**original_plan, "effects": planned["effects"]}, rejected, + turn_command="run-once", journal_readback=inspection) + identity = SettlementIdentity.from_runtime_payload( + original_plan["transaction"]["settlement_plan"]["identity"]) + readback = read_heartbeat_settlement(reports.root, + goal_id=identity.goal_id, agent_id=identity.agent_id, todo_id=identity.todo_id, + turn_instance_id=identity.turn_instance_id, replan_obligation_id=identity.replan_obligation_id, + registry_path=self.controller.registry_path, + goal_ref=goal_ref, include_readonly_recovery=True) + recovery = readback.settlement_recovery if readback is not None else None + if not isinstance(recovery, Mapping): + raise ValueError("the original read-only settlement recovery could not be read") + failure["settlement_recovery"] = render_readonly_settlement_recovery(recovery, + registry_path=self.controller.registry_path, runtime_root=reports.root) + except (OSError, ValueError, RuntimeError, KeyError, TypeError) as read_error: + # Keep the original typed refusal authoritative. Missing + # recovery evidence grants no guessed command or permission. + failure["settlement_recovery_error"] = str(read_error) + failure["status"] = "failed" + rejected.payload = failure + rejected.report_recovery_context = {"request": request, "window": window, + "original_turn_id": original["turn_id"]} + if not any(row.get("kind") == "native_loopx.result" and row.get("payload") == failure + for row in self.store.events_after(actor["session_id"], original["turn_id"], None)): + self.store.append_event(actor["session_id"], original["turn_id"], + kind="native_loopx.result", payload=failure) + raise handle_turn_command(args, registry_path=self.controller.registry_path, runtime_root_arg=str(reports.root), output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), native_host_runner=host_runner, @@ -803,6 +852,7 @@ def replay_frozen_candidate(native_request: Mapping[str, Any], _launch_scope: An "goal_id": actor["goal_id"], "agent_id": actor["agent_id"]}, native_task_validator=self._report_validator(request), **({"native_completion_lease_resolver": lambda: self._task_lease_proof(request, lease)} if lease else {})) + # Retain the historical outer failure. The actual native owner # supplies the new event; no status or publication proof is forged. self.store.append_event(actor["session_id"], original["turn_id"], kind="native_loopx.result", payload=result) diff --git a/loopx/cli_commands/turn_rendering.py b/loopx/cli_commands/turn_rendering.py index f298def76c..e34883fd31 100644 --- a/loopx/cli_commands/turn_rendering.py +++ b/loopx/cli_commands/turn_rendering.py @@ -103,7 +103,7 @@ def build_turn_error_payload( "journal_consistent", "journal_status", "completed_phases", "recorded_effects", "recovery_decision", )} if journal_readback is not None else {}), - }} if execution_started else {}), + }} if execution_started or journal_readback is not None else {}), **({ "resume_turn_key": turn_key, "journal_ref": f"turn:{turn_key.removeprefix('sha256:')[:16]}", @@ -113,6 +113,35 @@ def build_turn_error_payload( } +def render_readonly_settlement_recovery( + recovery: Mapping[str, Any], *, registry_path: Path, runtime_root: Path, +) -> dict[str, Any]: + """Add private invocation addresses to the shared owner's read-only argv.""" + from ..command_invocation import loopx_cli_argv + from ..install_contract import render_host_cli_command + + actions = recovery.get("next_cli_actions_argv") + allowed = {"required", "settlement_identity", "goal_ref", "instruction", + "next_cli_actions_argv", "original_settlement_readback"} + if (set(recovery) - allowed or recovery.get("required") is not True + or not isinstance(recovery.get("instruction"), str) or not recovery["instruction"] + or not isinstance(recovery.get("settlement_identity"), Mapping) + or not isinstance(recovery.get("original_settlement_readback"), Mapping) + or not isinstance(actions, list) + or len(actions) != 2 or any(not isinstance(argv, list) or not argv + or any(not isinstance(word, str) or not word for word in argv) for argv in actions) + or actions[0][:2] != ["turn", "inspect-journal"] + or len(actions[0]) != 8 or actions[0][2::2] != ["--goal-id", "--agent-id", "--turn-key"] + or actions[1][:2] != ["task-lease", "inspect"] + or len(actions[1]) != 6 or actions[1][2::2] != ["--goal-id", "--todo-id"]): + raise ValueError("shared read-only settlement recovery actions are unavailable") + prefix = [*loopx_cli_argv(), "--registry", str(registry_path), + "--runtime-root", str(runtime_root), "--format", "json"] + return {**dict(recovery), "next_cli_actions": [ + render_host_cli_command([*prefix, *argv]) for argv in actions], + } + + def _turn_plan_readback_lines(payload: Mapping[str, object]) -> list[str]: """Render the existing inspection source; never recompute admission here.""" source = payload.get("work_contract") diff --git a/loopx/control_plane/quota/settlement.py b/loopx/control_plane/quota/settlement.py index 396624e98e..67a624b100 100644 --- a/loopx/control_plane/quota/settlement.py +++ b/loopx/control_plane/quota/settlement.py @@ -193,6 +193,7 @@ class QuotaSettlementReadback: original_prepared_receipt_sha256: str | None = None native_task_lease_cleanup: dict[str, Any] | None = None writeback_qualification: SettlementResult[dict[str, Any]] | None = None + settlement_recovery: dict[str, Any] | None = None def render_owed_host_command(argv: list[str]) -> str: @@ -409,7 +410,12 @@ def read(admission: Mapping[str, Any] | None) -> Any: if exc.diagnostic_code == "refresh_task_lease_unavailable": from .execution_preparation import RefreshTaskLeaseUnavailable - raise RefreshTaskLeaseUnavailable(str(exc)) from None + rejected = RefreshTaskLeaseUnavailable(str(exc)) + rejected.payload = {"original_ts_refusal": { + "kind": exc.error_kind, "code": exc.diagnostic_code, + "message": str(exc), "details": list(exc.details), + }} + raise rejected from exc raise ValueError(str(exc)) from None if source_admission is not None: @@ -512,6 +518,7 @@ def read_heartbeat_settlement( writeback_candidate: Mapping[str, Any] | None = None, include_record_bodies: bool = True, tolerate_malformed_history: bool = False, + include_readonly_recovery: bool = False, ) -> QuotaSettlementReadback | None: """Read one complete heartbeat settlement through the TS domain owner.""" @@ -539,6 +546,7 @@ def read_heartbeat_settlement( **({"pending_closeout": True} if pending_closeout else {}), **({"refresh_retry": refresh_retry} if refresh_retry is not None else {}), **({"borrow_source_admission": True} if borrow_source_admission else {}), + **({"include_readonly_recovery": True} if include_readonly_recovery else {}), **( {"heartbeat_reentry_guard": heartbeat_reentry_guard} if heartbeat_reentry_guard is not None @@ -628,4 +636,5 @@ def read_heartbeat_settlement( _readback_result(payload["writeback_qualification"]) if "writeback_qualification" in payload else None ), + settlement_recovery=_optional_readback_record(payload.get("settlement_recovery")), ) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 8d7d1d540b..2e03ea69aa 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -201,6 +201,7 @@ interface ReadbackRequest { pending_closeout: boolean; borrow_source_admission: boolean; include_record_bodies: boolean; + include_readonly_recovery: boolean; tolerate_malformed_history: boolean; refresh_retry: RefreshRetryRequest | null; heartbeat_reentry_guard: JsonObject | null; @@ -302,7 +303,8 @@ function decodeWritebackCandidate(value: unknown): WritebackCandidate | null { const run = requireJsonObject(candidate.run, "writeback_candidate.run"); if (Object.keys(run).some(key => !["goal_id", "agent_id", "todo_id", "turn_instance_id", "settlement_identity", "goal_ref", "classification", "delivery_outcome", "delivery_batch_scale", "delivery_boundary", - "progress_observation", "agent_vision", "vision_checkpoint", "autonomous_replan_ack", "refresh_recovery"].includes(key))) { + "progress_observation", "agent_vision", "vision_checkpoint", "autonomous_replan_ack", "refresh_recovery", + "generated_at", "blocked_retry"].includes(key))) { throw new EffectRuntimeRequestError("writeback_candidate.run contains an unsupported payload field"); } return {...proof, effect_ref: requireNonEmptyString(candidate.effect_ref, "writeback_candidate.effect_ref"), @@ -501,6 +503,19 @@ function decodeRequest( || optionalRequestString(request.replan_obligation_id, "replan_obligation_id") !== null)) { throw new EffectRuntimeRequestError("native task lease cleanup requires only an explicit original Todo Turn read"); } + if (request.include_readonly_recovery !== undefined && typeof request.include_readonly_recovery !== "boolean") { + throw new EffectRuntimeRequestError("include_readonly_recovery must be a boolean"); + } + if (request.include_readonly_recovery === true && (resumeCandidate !== null || writebackCandidate !== null + || nativeCleanup !== null || request.infer_turn_instance_id || request.allow_unbound_binding + || request.resolve_original_binding === true || request.pending_closeout === true || request.refresh_retry != null + || request.heartbeat_reentry_guard != null || request.spend_precheck === true || request.spend_request != null + || request.tolerate_malformed_history === true || request.replan_obligation_id != null + || ["agent_id", "todo_id", "turn_instance_id"].some(key => typeof request[key] !== "string" + || !request[key] || request[key] !== (request[key] as string).trim()) + || normalizeAgentId(request.agent_id) !== request.agent_id || normalizeTodoId(request.todo_id) !== request.todo_id)) { + throw new EffectRuntimeRequestError("read-only recovery requires only an exact original Agent/Todo/Turn readback"); + } return { runtime_root: runtimeRoot, goal_id: goalId, @@ -520,6 +535,7 @@ function decodeRequest( pending_closeout: request.pending_closeout === true, borrow_source_admission: request.borrow_source_admission === true, include_record_bodies: request.include_record_bodies !== false, + include_readonly_recovery: request.include_readonly_recovery === true, tolerate_malformed_history: request.tolerate_malformed_history === true, refresh_retry: decodeRefreshRetry(request.refresh_retry), heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null @@ -1779,6 +1795,7 @@ export function readQuotaSettlementFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value); + if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); if (request.owner.kind !== "alias") { @@ -1796,6 +1813,7 @@ export function readAdmittedQuotaSettlementFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value); + if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); @@ -1808,13 +1826,15 @@ export function readQuotaSettlementForAdmittedOwnerFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, ): JsonObject { const request = decodeRequest(value, owner); + if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); } async function readSettlementExecutionAuthority(request: ReadbackRequest, result: JsonObject): Promise<{ - identity: SettlementIdentity; todo: JsonObject; lease: JsonObject | undefined; handoffMode: unknown; + identity: SettlementIdentity; todo: JsonObject; todos: JsonObject[]; + lease: JsonObject | undefined; handoffMode: unknown; }> { function reject(message: string): never { throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); @@ -1858,7 +1878,8 @@ async function readSettlementExecutionAuthority(request: ReadbackRequest, result if (observed.status !== "available" || observed.store_identity !== snapshot.identity.store_identity) { reject("Canonical authority changed during resume qualification."); } - return {identity, todo, lease, handoffMode: authority.head.handoff_mode}; + return {identity, todo, todos: collection.projection.todo_ids.map(id => collection.projection.todos.get(id)!), + lease, handoffMode: authority.head.handoff_mode}; } async function readResumeLeaseQualification(request: ReadbackRequest, result: JsonObject): Promise { @@ -1914,12 +1935,13 @@ async function readWritebackQualification(request: ReadbackRequest, result: Json throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); } const candidate = request.writeback_candidate!, run = candidate.run; - const {identity, todo, lease, handoffMode} = await readSettlementExecutionAuthority(request, result); + const {identity, todo, todos, lease, handoffMode} = await readSettlementExecutionAuthority(request, result); if (identity.binding_kind !== "todo" || candidate.effect_ref !== `${identity.effect_id}#durable_writeback` || candidate.completion_todo_id !== null || candidate.completion_turn_key !== null || run.goal_id !== identity.goal_id || run.agent_id !== identity.agent_id || run.todo_id !== identity.todo_id || run.turn_instance_id !== identity.turn_instance_id || run.delivery_boundary !== "semantic_closeout" + || typeof run.generated_at !== "string" || !run.generated_at || run.generated_at !== run.generated_at.trim() || typeof run.classification !== "string" || !run.classification.trim()) { reject("Writeback candidate does not describe this original Todo's semantic operation."); } @@ -1935,7 +1957,9 @@ async function readWritebackQualification(request: ReadbackRequest, result: Json reject("Writeback progress must describe the original Todo, without substituting another work item."); } const recovery = jsonObject(result.refresh_recovery); - if (retry.external_delivery != null || retry.workspace_requested || retry.checkpoint_read_context_id != null + // The existing external-delivery owner consumes the actual default/suppress + // policy separately. A resume acknowledgement is another operation. + if (retry.external_delivery?.resume_key != null || retry.workspace_requested || retry.checkpoint_read_context_id != null || retry.delivery_boundary !== "semantic_closeout" || retry.delivery_outcome !== run.delivery_outcome || retry.delivery_batch_scale !== (run.delivery_batch_scale ?? null) || !isDeepStrictEqual(retry.progress_observation, run.progress_observation ?? null) @@ -1981,6 +2005,42 @@ async function readWritebackQualification(request: ReadbackRequest, result: Json // Reuse the typed observation codec for every semantic branch. Its outcome // qualification remains with the existing replan owner below. if (progressSource !== null) projectProgressDelta(progressSource); + const blockedGap = run.delivery_outcome === "outcome_gap" && progress?.result_class === "blocked"; + let blockedRetry: JsonObject | null = null; + if (!blockedGap && run.blocked_retry != null) reject("A blocked wait cannot accompany a different semantic outcome."); + if (blockedGap) { + try { + const {evaluateTodoResumeConditions, TODO_RESUME_EVALUATION_REQUEST_SCHEMA_VERSION} = await import("../todos/resume_condition.ts"); + const rawTodos = todos.map(row => { + const raw = {...row}; + delete raw.resume_condition; + delete raw.resume_ready; + return raw; + }); + const evaluated = evaluateTodoResumeConditions({schema_version: TODO_RESUME_EVALUATION_REQUEST_SCHEMA_VERSION, + items: rawTodos, source_items: rawTodos, evaluated_at: run.generated_at, + rollout_events: snapshot.events.filter(event => quotaOwnerOwnsProjection(request.owner, event.goal_ref)), + kinds: ["resume_at", "todo_done", "monitor_changed"]}); + const conditions = new Map((evaluated.conditions as JsonObject[]).map(row => [row.todo_id, row.condition])); + const currentTodos = rawTodos.map(row => conditions.has(row.todo_id) + ? {...row, resume_condition: conditions.get(row.todo_id), resume_ready: jsonObject(conditions.get(row.todo_id))?.satisfied === true} + : row); + let allowTurnSettlementRetry = false; + if (!todo.resume_when && todo.status === "open") { + const {loadLegacyCoordinationWriterFence} = await import("../coordination/legacy_writer_fence.ts"); + const mode = await loadLegacyCoordinationWriterFence(request.runtime_root, identity.goal_id); + if (mode.status === "failed") reject(mode.reason); + allowTurnSettlementRetry = mode.status === "loaded"; + } + blockedRetry = prepareBlockedWait({schema_version: BLOCKED_WAIT_REQUEST_SCHEMA, todo_id: identity.todo_id, + todos: currentTodos, observed_at: run.generated_at, allow_turn_settlement_retry: allowTurnSettlementRetry}); + if (!isDeepStrictEqual(run.blocked_retry, blockedRetry)) { + reject("Writeback blocked retry differs from the actual Run clock or current canonical Todo wait."); + } + } catch (error) { + reject(error instanceof Error ? error.message : "The actual blocked writeback could not be qualified."); + } + } const existingVision = [...ownerRuns].reverse().map(row => jsonObject(row.agent_vision)).find(value => value !== null) ?? null; let vision: JsonObject | null = null; if (run.agent_vision != null) { @@ -2003,21 +2063,33 @@ async function readWritebackQualification(request: ReadbackRequest, result: Json const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", agent_id: identity.agent_id, agent_vision: vision, existing_agent_vision: existingVision, vision_unchanged_reason: retry.unchanged_reason, delivery_outcome: run.delivery_outcome, - active_state_next_action_would_update: retry.mutation.next_action != null, + active_state_next_action_would_update: false, delivery_boundary: "semantic_closeout", todo_id: identity.todo_id, completion_todo_id: null, - autonomous_replan_recorded: retry.mutation.autonomous_replan_recorded === true, blocked_retry: null}); + autonomous_replan_recorded: retry.mutation.autonomous_replan_recorded === true, blocked_retry: blockedRetry}); const offeredCheckpoint = jsonObject(run.vision_checkpoint); - if (checkpoint.satisfied !== true || offeredCheckpoint === null - || ["schema_version", "agent_id", "required", "satisfied", "decision", "triggers", - "delivery_boundary", "agent_vision_state"].some(key => !isDeepStrictEqual(offeredCheckpoint[key], checkpoint[key]))) { + if (offeredCheckpoint === null) reject("Writeback checkpoint is missing from its actual Run."); + const offeredBuilderCheckpoint = {...offeredCheckpoint}; + // outcome_continuity.py appends this read-only diagnostic after the Vision + // builder. Preserve its actual facts without duplicating that owner or + // consuming the annotation as checkpoint acceptance or write authority. + delete offeredBuilderCheckpoint.acceptance_gaps; + if (!isDeepStrictEqual(offeredBuilderCheckpoint, checkpoint)) { reject("Writeback checkpoint was not qualified from its actual semantic inputs."); } const qualifiedRun = {...run, settlement_identity: settlementIdentityPayload(identity), agent_vision: vision, - vision_checkpoint: checkpoint}; - // Prepared execution authority alone cannot validate semantic writeback. - // The same actual Vision, progress identity and checkpoint checks above - // apply before returning this operation's scope. - if (activePrepared !== null) return scope(activePrepared, null); + vision_checkpoint: {...checkpoint, ...(Object.hasOwn(offeredCheckpoint, "acceptance_gaps") + ? {acceptance_gaps: offeredCheckpoint.acceptance_gaps} : {})}, blocked_retry: blockedRetry}; + // An exact active prepared Turn records the Vision owner's negative decision + // as its original writeback. The scope neither passes the checkpoint nor + // grants completion, spend, a lease, or another Turn's execution. + if (activePrepared !== null) { + if (checkpoint.satisfied !== true && !(blockedGap && blockedRetry !== null + && checkpoint.decision === "missing_required" && checkpoint.satisfied === false)) { + reject("Only an original typed blocked outcome-gap writeback can retain a negative checkpoint."); + } + return scope(activePrepared, null); + } + if (checkpoint.satisfied !== true) reject("This semantic writeback requires a satisfied Vision checkpoint."); const supersede = jsonObject(result.supersede_event); if (originalReceipt !== null && supersede !== null && result.completion_event == null && todo.status === "done" && lease?.status === "released") { @@ -2106,6 +2178,53 @@ async function readWritebackQualification(request: ReadbackRequest, result: Json return scope(null, selected); } +/** Read-only recovery is an attachment to verified receipt/journal evidence, + * not execution readiness or a substitute control-writer proof. */ +async function readReadonlySettlementRecovery(request: ReadbackRequest, result: JsonObject): Promise { + const identityResult = jsonObject(jsonObject(result.identity)?.result); + const failure = jsonObject(identityResult?.failure); + if (failure !== null) { + throw new EffectRuntimeRequestError(requireNonEmptyString(failure.reason, "original settlement failure reason"), + requireNonEmptyString(failure.kind, "original settlement failure kind")); + } + const value = jsonObject(identityResult?.value); + if (result.found !== true || value === null) { + throw new EffectRuntimeRequestError("The original settlement readback is unavailable.", "refresh_task_lease_unavailable"); + } + const parsed = settlementIdentityFromPlan({settlement_plan: {identity: value}}); + if (parsed.failure !== null) throw new EffectRuntimeRequestError(parsed.failure.reason, parsed.failure.kind); + const identity = parsed.value; + if (identity.binding_kind !== "todo" || identity.goal_id !== request.goal_id || identity.agent_id !== request.agent_id + || identity.todo_id !== request.todo_id || identity.turn_instance_id !== request.turn_instance_id) { + throw new EffectRuntimeRequestError("Read-only recovery does not match the original Todo Turn.", "refresh_task_lease_unavailable"); + } + const {readTurnJournalRecoveryLocator} = await import("../turn_driver/turn_journal_query.ts"); + const turnKey = await readTurnJournalRecoveryLocator(request.runtime_root, identity, request.owner); + if (turnKey === null) { + throw new EffectRuntimeRequestError("The original Turn journal could not be verified for recovery inspection.", "refresh_task_lease_unavailable"); + } + const original = jsonObject(result.original_prepared_receipt); + const prepared = original === null ? null : heartbeatPreparedWorkAdmission(original, identity); + const originalDetails = details(original); + const modern = original !== null && prepared?.prepared_lease_epoch !== undefined + && heartbeatReceiptFactFromEvent(original).settlement_effect_id === identity.effect_id + && (originalDetails.semantic_replan_obligation_id == null || originalDetails.semantic_replan_obligation_id === ""); + const instruction = "Inspect the original journal and this Todo's current lease before retrying. Keep the original Goal, Agent, Todo and Turn; " + + "these read-only observations grant no execution, lease, claim, reactivation, completion or spend authority. " + + (modern + ? "The original receipt contains modern positive epoch-bound preparation. If that execution was lost, the existing refresh-state --execution-loss-closeout entry remains a separate administrative operation: it requires the current registered control writer and exact current active lease/CAS, the original admission, and its own fenced qualification. This attachment supplies none of that permission or proof." + : "This receipt does not establish modern epoch-bound administrative admission. Retain read-only inspection; do not infer a recovery grant or reconstruct missing preparation."); + const goalRef = quotaGoalRef(request.owner); + const originalReadback = Object.fromEntries(["identity", "settlement", "progress", "replay_phase", + "prepared_lease_required", "prepared_lease", "original_prepared_receipt", "original_prepared_event_id", + "original_prepared_receipt_sha256"].filter(field => Object.hasOwn(result, field)).map(field => [field, result[field]])); + return {required: true, settlement_identity: settlementIdentityPayload(identity), ...(goalRef === null ? {} : {goal_ref: goalRef}), + instruction, next_cli_actions_argv: [ + ["turn", "inspect-journal", "--goal-id", identity.goal_id, "--agent-id", identity.agent_id, "--turn-key", turnKey], + ["task-lease", "inspect", "--goal-id", identity.goal_id, "--todo-id", identity.todo_id!], + ], original_settlement_readback: structuredClone(originalReadback)}; +} + export async function readQuotaSettlement(value: unknown): Promise { if (jsonObject(value)?.schema_version === "loopx_quota_history_qualification_request_v0") { const input = jsonObject(value)!, runs = input.runs, observedAt = input.observed_at; @@ -2132,7 +2251,7 @@ export async function readQuotaSettlement(value: unknown): Promise { !Number.isSafeInteger(spendRequest.slots) || typeof spendRequest.source !== "string")) { throw new EffectRuntimeRequestError("spend_request requires spend_precheck, integer slots and a source"); } - if (spendPrecheck) request.include_record_bodies = true; + if (spendPrecheck || request.include_readonly_recovery) request.include_record_bodies = true; if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) request.include_record_bodies = true; const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner @@ -2188,6 +2307,7 @@ export async function readQuotaSettlement(value: unknown): Promise { "a stored Monitor poll may already have committed its observation; recover the exact monitor-poll before an unavailable closeout", ); } + if (request.include_readonly_recovery) result.settlement_recovery = await readReadonlySettlementRecovery(request, result); if (!spendPrecheck || result.found !== true) return result; // Preview still needs a fresh full read. The earlier replay check needs // only its receipts and whether a spend Run already needs receipt repair. diff --git a/loopx/control_plane/turn_driver/turn_journal_query.ts b/loopx/control_plane/turn_driver/turn_journal_query.ts index 620c3a75fe..a2587ab159 100644 --- a/loopx/control_plane/turn_driver/turn_journal_query.ts +++ b/loopx/control_plane/turn_driver/turn_journal_query.ts @@ -207,6 +207,7 @@ async function queryTurnJournal( runtimeRootValue: unknown, identity: JournalAddress | BoundSettlementIdentity, goalRef: ExactGoalRef | null, + legacyOnly = false, ): Promise { const runtimeRoot = requireNonEmptyString(runtimeRootValue, "runtime_root"); if (!isAbsolute(runtimeRoot) || runtimeRoot.includes("\0")) { @@ -250,7 +251,7 @@ async function queryTurnJournal( goalRef, parsed.numbers, ); - if (evidence === null) continue; + if (evidence === null || (legacyOnly && parseTurnJournalGoalBinding(evidence.journal).kind !== "legacy")) continue; if (matched !== null) journalConflict("LoopX Turn settlement identity matched multiple journals"); matched = evidence; } @@ -265,10 +266,26 @@ export async function findTurnJournalBySettlement(params: JsonObject): Promise<{ request_subject: params.request_subject as import("../effect_program.ts").ExecutionRequestSubject | undefined, turn_instance_id: requireNonEmptyString(params.turn_instance_id, "turn_instance_id"), })); - const evidence = await queryTurnJournal(params.runtime_root, identity, null); + const goalRef = Object.hasOwn(params, "goal_ref") ? decodeGoalRef(params.goal_ref) : null; + if (goalRef !== null && goalRef.goalId.value !== identity.goal_id) { + throw new EffectRuntimeRequestError("Turn journal query GoalRef conflicts with its settlement identity"); + } + const evidence = await queryTurnJournal(params.runtime_root, identity, goalRef); return { turn_key: evidence?.turn_key ?? null }; } +/** A recovery inspection uses its original parsed owner. Historical lookup + * keeps its wildcard default; this locator never lends another Goal lifetime. */ +export async function readTurnJournalRecoveryLocator( + runtimeRoot: string, + identity: SettlementIdentity, + owner: QuotaAccountingOwner, +): Promise { + const evidence = await queryTurnJournal(runtimeRoot, identity, + owner.kind === "exact_source" ? owner.goalRef : null, owner.kind === "alias"); + return evidence?.turn_key ?? null; +} + export async function findTurnJournalByInstance(params: JsonObject): Promise<{ turn_key: string | null }> { const address = { goal_id: requireNonEmptyString(params.goal_id, "goal_id"), diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index 13ff647431..e6f4d84d95 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -1,5 +1,7 @@ from __future__ import annotations +from collections.abc import Mapping + import hashlib import json import re @@ -1619,17 +1621,24 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): ) ): writeback_candidate = None - if normalized_delivery_boundary == "semantic_closeout": + if ( + normalized_delivery_boundary == "semantic_closeout" + and completion_todo_id is None + and completion_turn_key is None + and not isinstance(record.get("delivery_workspace"), Mapping) + ): # Project the final Run without replacing its observed # work identity, semantic delta or recovery metadata. + # Completion and workspace writebacks retain their + # existing business operation owner. writeback_run = { field: record[field] for field in ( - "goal_id", "agent_id", "todo_id", "turn_instance_id", + "generated_at", "goal_id", "agent_id", "todo_id", "turn_instance_id", "settlement_identity", "classification", "delivery_outcome", "delivery_batch_scale", "delivery_boundary", "progress_observation", "agent_vision", "vision_checkpoint", "autonomous_replan_ack", - "refresh_recovery", + "refresh_recovery", "blocked_retry", ) if field in record } diff --git a/tests/control_plane/test_quota_execution_preparation.py b/tests/control_plane/test_quota_execution_preparation.py index 2dcad022a4..c31b13d319 100644 --- a/tests/control_plane/test_quota_execution_preparation.py +++ b/tests/control_plane/test_quota_execution_preparation.py @@ -1331,6 +1331,10 @@ def _writeback_guard_fixture( state_path.write_text("# Fixture\n", encoding="utf-8") runtime = tmp_path / "runtime" registry = tmp_path / "registry.json" + fixture_todo = { + **_todo(), + "continuation_policy": "same_agent_non_delivery", + } write_fixture_registry( project=project, runtime_root=runtime, @@ -1351,7 +1355,7 @@ def _writeback_guard_fixture( build_todo_runtime_shadow_projection( goal_id=GOAL, handoff_mode="hard_lease", - todos=[_todo()], + todos=[fixture_todo], leases=[lease], ), state_path=state_path, @@ -1382,18 +1386,29 @@ def _writeback_guard_fixture( "quota", "should-run", "--codex-app", "--context-delivery", "off", "--goal-id", GOAL, "--agent-id", AGENT, "--todo-id", TODO, "--turn-instance-id", turn_instance_id, - "--task-lease-idempotency-key", str(proof["idempotency_key"]), - "--task-lease-expected-version", str(proof["expected_version"]), registry_path=registry, runtime_root=runtime, cwd=project, ) - assert code == 0 and packet["execution_preparation"]["state"] == "ready", packet + assert code == 0 and packet["execution_preparation"]["state"] == "ready", json.dumps( + packet, ensure_ascii=False, indent=2, + ) settlement = read_heartbeat_settlement( runtime, goal_id=GOAL, agent_id=AGENT, todo_id=TODO, turn_instance_id=turn_instance_id, registry_path=registry, ) assert settlement is not None assert settlement.identity.value is not None + assert settlement.identity.value.goal_id == GOAL + assert settlement.identity.value.agent_id == AGENT + assert settlement.identity.value.todo_id == TODO assert settlement.identity.value.turn_instance_id == turn_instance_id + if not legacy_no_epoch_receipt: + assert settlement.prepared_lease == { + "prepared_lease_key_sha256": hashlib.sha256( + str(lease["idempotency_key"]).encode("utf-8") + ).hexdigest(), + "prepared_lease_version": lease["version"], + "prepared_lease_epoch": lease["lease_epoch"], + } return { "project": project, "runtime": runtime, "registry": registry, "state": state_path, "turn": turn_instance_id, "proof": proof, @@ -1437,6 +1452,7 @@ def _run_real_semantic_writeback( def _writeback_side_effect_snapshot(context: dict[str, object]) -> dict[str, object]: + from loopx.file_lock import lock_holder_path from loopx.control_plane.quota.settlement import read_heartbeat_settlement from loopx.rollout_event_log import rollout_event_log_path @@ -1450,7 +1466,20 @@ def _writeback_side_effect_snapshot(context: dict[str, object]) -> dict[str, obj ) assert settlement is not None runs = context["runtime"] / "goals" / goal_id / "runs" + index_holder = lock_holder_path(runs / "index.jsonl") receipt_log = rollout_event_log_path(context["runtime"], goal_id) + run_files: dict[str, bytes] = {} + if runs.exists(): + for path in runs.rglob("*"): + if not path.is_file(): + continue + if path == index_holder: + released_holder = json.loads(path.read_text(encoding="utf-8")) + assert released_holder["schema_version"] == "file_lock_holder_v0" + assert isinstance(released_holder.get("released_at"), str) + assert released_holder["released_at"] + continue + run_files[str(path.relative_to(runs))] = path.read_bytes() return { "authority": read_canonical_todos_if_promoted( runtime_root=context["runtime"], goal_id=goal_id, include_leases=True, @@ -1460,10 +1489,7 @@ def _writeback_side_effect_snapshot(context: dict[str, object]) -> dict[str, obj "original_sha256": settlement.original_prepared_receipt_sha256, "goal_file": context["state"].read_bytes(), "receipt_log": receipt_log.read_bytes(), - "run_files": { - str(path.relative_to(runs)): path.read_bytes() - for path in runs.rglob("*") if path.is_file() - } if runs.exists() else {}, + "run_files": run_files, } diff --git a/tests/control_plane_ts/heartbeat_resume_lease.test.ts b/tests/control_plane_ts/heartbeat_resume_lease.test.ts index a294900e5d..67b5d3ffc7 100644 --- a/tests/control_plane_ts/heartbeat_resume_lease.test.ts +++ b/tests/control_plane_ts/heartbeat_resume_lease.test.ts @@ -578,8 +578,8 @@ function semanticWritebackCandidate(requestedProof: JsonObject | null = null): J completion_todo_id: null, completion_turn_key: null, run: {goal_id: identity.goal_id, agent_id: identity.agent_id, todo_id: identity.todo_id, turn_instance_id: identity.turn_instance_id, settlement_identity: settlementIdentityPayload(identity), - classification: "turn_run_once", delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", - progress_observation: null, agent_vision: vision, vision_checkpoint: checkpoint, autonomous_replan_ack: null, + classification: "turn_run_once", generated_at: "2026-01-01T00:01:00Z", blocked_retry: null, + delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", progress_observation: null, agent_vision: vision, vision_checkpoint: checkpoint, autonomous_replan_ack: null, refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; } @@ -707,3 +707,85 @@ for (const provider of ["file", "sqlite"] as const) { } }); } + + +function missingPreparedVisionWriteback(wait: JsonObject | null = null): JsonObject { + const original = semanticWritebackCandidate({idempotency_key: originalKey, expected_version: 120}); + const candidate = original.writeback_candidate as JsonObject, run = candidate.run as JsonObject; + const progress = wait === null ? null : {schema_version: "typed_progress_observation_v0", work_item_id: identity.todo_id, + result_class: "blocked", blocker_id: "blocker:verified", fingerprint: "abcdef0123456789", evidence_ids: ["evidence:verified-blocker"]}; + const outcome = wait === null ? "outcome_progress" : "outcome_gap"; + const retry = {...original.refresh_retry as JsonObject, vision: null, unchanged_reason: "Prior route remains.", + delivery_outcome: outcome, progress_observation: progress}; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: identity.agent_id, agent_vision: null, existing_agent_vision: null, + vision_unchanged_reason: retry.unchanged_reason, delivery_outcome: outcome, + active_state_next_action_would_update: false, delivery_boundary: "semantic_closeout", todo_id: identity.todo_id, + completion_todo_id: null, autonomous_replan_recorded: false, blocked_retry: wait}); + return {refresh_retry: retry, writeback_candidate: {...candidate, run: {...run, agent_vision: null, + delivery_outcome: outcome, progress_observation: progress, blocked_retry: wait, vision_checkpoint: checkpoint, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: active prepared writeback preserves the owner's negative checkpoint and obligations`, async () => { + const wait = {schema_version: "quota_blocked_retry_v0", source: "todo", todo_id: identity.todo_id, + resume_when: "resume_at:2026-01-01T00:06:00Z", observed_at: "2026-01-01T00:01:00Z", due_at: "2026-01-01T00:06:00Z"}; + const f = await receiptFixture([preparedReceipt], "hard_lease", currentLease, {provider, + seedTodo: {...todo(identity.todo_id!), claimed_by: identity.agent_id, task_class: "advancement_task", resume_when: wait.resume_when}}); + try { + const payload = missingPreparedVisionWriteback(wait), candidate = payload.writeback_candidate as JsonObject; + const run = candidate.run as JsonObject, checkpoint = run.vision_checkpoint as JsonObject; + const input = {...request(f.root), ...payload}, before = structuredClone(payload); + const logBefore = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + assert.deepEqual(checkpoint, {schema_version: "vision_checkpoint_v0", agent_id: identity.agent_id, + required: true, satisfied: false, decision: "missing_required", triggers: [], delivery_boundary: "semantic_closeout", + missing_baseline: true, rejected_unchanged_reason: "Prior route remains.", required_resolution: ["write_vision_patch"]}); + const result = await readQuotaSettlement(input); + assert.deepEqual((result.writeback_qualification as {result: {value: JsonObject}}).result.value, { + step_kind: "durable_writeback", effect_ref: `${identity.effect_id}#durable_writeback`, delivery_boundary: "semantic_closeout", + original_prepared: originalPrepared, selected_obligation_id: null}); + assert.equal(checkpoint.satisfied, false); + assert.equal(result.resume_lease_qualification, undefined); + for (const patch of [{required: false}, {satisfied: true}, {decision: "patched"}, {missing_baseline: false}, + {rejected_unchanged_reason: "Different decision"}, {required_resolution: []}, + {required_resolution: ["record_unchanged_reason"]}, {unqualified_components: []}, + {continuity_basis: {kind: "existing_vision_unchanged", vision_generated_at: "foreign-baseline"}}]) { + await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, + run: {...run, vision_checkpoint: {...checkpoint, ...patch}}}}), {code: "refresh_task_lease_unavailable"}, JSON.stringify(patch)); + } + for (const field of ["satisfied", "required", "triggers", "missing_baseline", "rejected_unchanged_reason", "required_resolution"]) { + const omitted = {...checkpoint}; delete omitted[field]; + await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, + run: {...run, vision_checkpoint: omitted}}}), {code: "refresh_task_lease_unavailable"}, `missing ${field}`); + } + await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, + run: {...run, vision_checkpoint: null}}}), {code: "refresh_task_lease_unavailable"}); + // The independent frontier annotation is retained as data. It cannot + // change the builder's false decision or the five-field operation scope. + const annotated = {...checkpoint, acceptance_gaps: [{kind: "vision_checkpoint_missing", source: "latest_vision_checkpoint", + agent_id: identity.agent_id, decision: "missing_required", missing_baseline: true}]}; + const annotatedInput = {...input, writeback_candidate: {...candidate, run: {...run, vision_checkpoint: annotated}}}; + const annotatedBefore = structuredClone(annotatedInput); + const annotatedResult = await readQuotaSettlement(annotatedInput); + assert.deepEqual((annotatedResult.writeback_qualification as {result: {value: JsonObject}}).result.value, + (result.writeback_qualification as {result: {value: JsonObject}}).result.value); + assert.deepEqual(annotatedInput, annotatedBefore); + assert.deepEqual(payload, before); + assert.deepEqual(await readFile(f.log), logBefore); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: retired writeback still rejects the same negative checkpoint`, async () => { + for (const retirement of [completed, {...completed, event_kind: "todo_supersede", event_id: "supersede-original", status: "done"}]) { + const f = await receiptFixture([preparedReceipt, retirement], "hard_lease", {...currentLease, status: "released"}, + {provider, seedTodo: {...todo(identity.todo_id!, "done"), claimed_by: null}}); + try { + const payload = missingPreparedVisionWriteback(), candidate = payload.writeback_candidate as JsonObject; + await assert.rejects(readQuotaSettlement({...request(f.root), ...payload, + writeback_candidate: {...candidate, requested_proof: null}}), {code: "refresh_task_lease_unavailable"}); + } finally {await rm(f.root, {recursive: true, force: true});} + } + }); +} diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index 318af55ef5..c63359d7b8 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -2916,25 +2916,29 @@ function todoSemanticWriteback(): JsonObject { completion_todo_id: null, completion_turn_key: null, run: {goal_id: goalId, agent_id: agentId, todo_id: todoId, turn_instance_id: turnId, settlement_identity: settlementIdentityPayload(identity), classification: "turn_run_once", - delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", progress_observation: progress, + generated_at: "2026-01-01T00:01:00Z", blocked_retry: null, delivery_outcome: "outcome_progress", delivery_boundary: "semantic_closeout", progress_observation: progress, agent_vision: vision, vision_checkpoint: checkpoint, autonomous_replan_ack: {schema_version: "autonomous_replan_ack_v0", recorded: true, semantic_delta: semantic}, refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; } -async function todoSemanticFixture(provider: "file" | "sqlite", seedTodo?: JsonObject, leases: JsonObject[] = []) { +async function todoSemanticFixture(provider: "file" | "sqlite", seedTodo?: JsonObject, leases: JsonObject[] = [], + additionalTodos: JsonObject[] = []) { const root = await fixture({guard: false}); const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); const admission = todoSemanticAdmission(); await writeFile(log, JSON.stringify(admission) + "\n"); const {projection, todo} = await import("./shadow_file_fixture.ts"); + const {authorityUnicodeCompare} = await import("../../loopx/control_plane/coordination/authority_store_codec.ts"); + const allTodos = [seedTodo ?? {...todo(todoId), claimed_by: agentId}, ...additionalTodos] + .sort((left, right) => authorityUnicodeCompare(String(left.todo_id), String(right.todo_id))); const store = provider === "sqlite" ? new (await import("../../loopx/control_plane/coordination/sqlite_authority_store.ts")).SqliteAuthorityStore( join(root, "authority", "sqlite-v0"), goalId) : new (await import("../../loopx/control_plane/coordination/file_authority_store.ts")).FileAuthorityStore( join(root, "authority", "file-v0"), goalId); assert.equal((await store.commitAuthority({operation_id: "seed-semantic-authority", expected_provider_revision: null, - next_projection: {...projection([seedTodo ?? {...todo(todoId), claimed_by: agentId}], leases), goal_id: goalId}, + next_projection: {...projection(allTodos, leases), goal_id: goalId}, events: [], receipts: []})).status, "applied"); if (provider === "sqlite") { const observed = await store.storeIdentity(); @@ -2994,7 +2998,7 @@ for (const provider of ["file", "sqlite"] as const) { const scaledResult = await readQuotaSettlement({...input, refresh_retry: scaled, writeback_candidate: {...candidate, run: {...run, delivery_batch_scale: "single_surface"}}}); assert.equal((scaledResult.writeback_qualification as any).payload.ok, true); - for (const patch of [{external_delivery: {suppress: true, resume_key: null}}, + for (const patch of [{external_delivery: {suppress: false, resume_key: "a".repeat(64)}}, {workspace_requested: true}, {checkpoint_read_context_id: "checkpoint:foreign"}]) { const changedRetry = {...retry, ...patch}; await assert.rejects(readQuotaSettlement({...input, refresh_retry: changedRetry, @@ -3149,3 +3153,266 @@ for (const provider of ["file", "sqlite"] as const) { } finally {await rm(f.root, {recursive: true, force: true});} }); } + + +function todoBlockedSemanticWriteback(wait: JsonObject): JsonObject { + const input = todoSemanticWriteback(), candidate = input.writeback_candidate as JsonObject; + const run = candidate.run as JsonObject; + const progress = {schema_version: "typed_progress_observation_v0", work_item_id: todoId, result_class: "blocked", + blocker_id: "blocker:verified", fingerprint: "abcdef0123456789", evidence_ids: ["evidence:verified-blocker"]}; + const retry = {...input.refresh_retry as JsonObject, vision: null, delivery_outcome: "outcome_gap", progress_observation: progress}; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: agentId, agent_vision: null, existing_agent_vision: null, vision_unchanged_reason: null, + delivery_outcome: "outcome_gap", active_state_next_action_would_update: false, + delivery_boundary: "semantic_closeout", todo_id: todoId, completion_todo_id: null, + autonomous_replan_recorded: true, blocked_retry: wait}); + return {refresh_retry: retry, writeback_candidate: {...candidate, run: {...run, agent_vision: null, + delivery_outcome: "outcome_gap", progress_observation: progress, blocked_retry: wait, vision_checkpoint: checkpoint, + autonomous_replan_ack: {schema_version: "autonomous_replan_ack_v0", recorded: true, semantic_delta: { + schema_version: "replan_semantic_delta_v0", accepted: true, + outcomes: ["new_concrete_blocker"], satisfying_outcomes: ["new_concrete_blocker"], + required_any_of: ["new_surface", "new_hypothesis", "new_probe_family", "new_runnable_successor", + "coverage_backed_exploration_exhausted", "new_concrete_blocker", "coverage_backed_no_followup"], + observation_fingerprint: progress.fingerprint}}, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}; +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: first writeback keeps external policy separate and NextAction journal only`, async () => { + const f = await todoSemanticFixture(provider); + try { + const original = todoSemanticWriteback(), candidate = original.writeback_candidate as JsonObject; + const run = candidate.run as JsonObject; + const authorityBefore = await f.store.loadAuthority(), logBefore = await readFile(f.log); + for (const suppress of [false, true]) { + const retry = {...original.refresh_retry as JsonObject, external_delivery: {suppress, resume_key: null}, + mutation: {autonomous_replan_recorded: true, next_action: "Record the next useful step", next_action_basis: "verified-result"}}; + const result = await readQuotaSettlement({...request(f.root), refresh_retry: retry, + writeback_candidate: {...candidate, run: {...run, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}); + const qualified = result.writeback_qualification as {result: {value: JsonObject}; payload: {ok: boolean}}; + assert.equal(qualified.payload.ok, true); + assert.deepEqual(Object.keys(qualified.result.value).sort(), ["delivery_boundary", "effect_ref", "original_prepared", + "selected_obligation_id", "step_kind"]); + assert.equal((result.external_delivery as JsonObject).authorized, !suppress); + assert.equal(Object.hasOwn(qualified.result.value, "authorized"), false); + assert.equal(Object.hasOwn(qualified.result.value, "current_task_lease_proof"), false); + } + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + assert.deepEqual(await readFile(f.log), logBefore); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: blocked semantic writeback uses the actual clock and raw canonical wait`, async () => { + const {todo} = await import("./shadow_file_fixture.ts"); + const selected = {...todo(todoId), task_class: "advancement_task", claimed_by: agentId, + resume_when: "resume_at:2026-01-01T00:06:00Z"}; + const f = await todoSemanticFixture(provider, selected); + try { + const wait = {schema_version: "quota_blocked_retry_v0", source: "todo", todo_id: todoId, + resume_when: selected.resume_when, observed_at: "2026-01-01T00:01:00Z", due_at: "2026-01-01T00:06:00Z"}; + const payload = todoBlockedSemanticWriteback(wait), candidate = payload.writeback_candidate as JsonObject; + const run = candidate.run as JsonObject; + const input = {...request(f.root), ...payload}; + const result = await readQuotaSettlement(input); + assert.equal((result.writeback_qualification as {payload: {ok: boolean}}).payload.ok, true); + assert.equal(Object.hasOwn(selected, "resume_condition"), false); + assert.equal(Object.hasOwn(selected, "resume_ready"), false); + for (const patch of [{generated_at: undefined}, {generated_at: "2026-01-01T00:02:00Z"}, + {generated_at: "2026-01-01T00:00:00Z"}, {generated_at: "2026-01-01T00:01:00"}, + {blocked_retry: null}, {blocked_retry: {...wait, todo_id: "todo_foreign"}}, + {blocked_retry: {...wait, observed_at: "2026-01-01T00:00:00Z"}}, + {blocked_retry: {...wait, due_at: "2026-01-01T00:07:00Z", resume_when: "resume_at:2026-01-01T00:07:00Z"}}]) { + await assert.rejects(readQuotaSettlement({...input, writeback_candidate: {...candidate, run: {...run, ...patch}}}), + {code: "refresh_task_lease_unavailable"}, JSON.stringify(patch)); + } + const advanced = todoSemanticWriteback(), advancedCandidate = advanced.writeback_candidate as JsonObject; + await assert.rejects(readQuotaSettlement({...request(f.root), ...advanced, writeback_candidate: {...advancedCandidate, + run: {...advancedCandidate.run as JsonObject, blocked_retry: wait}}}), {code: "refresh_task_lease_unavailable"}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + + test(`${provider}: blocked semantic writeback checks the complete canonical dependency graph`, async () => { + const {todo} = await import("./shadow_file_fixture.ts"); + const waiting = {...todo(todoId), task_class: "advancement_task", claimed_by: agentId, resume_when: "todo_done:todo_dependency"}; + const target = {...todo("todo_dependency"), task_class: "advancement_task", resume_when: "todo_done:todo_tail"}; + const wait = {schema_version: "quota_blocked_causal_wait_v0", source: "todo", todo_id: todoId, + resume_when: waiting.resume_when, observed_at: "2026-01-01T00:01:00Z", + waiting_todo: {todo_id: todoId, role: "agent", status: "open", task_class: "advancement_task", archive_state: "active", + resume_when: waiting.resume_when, resume_ready: false}, + target_todo: {todo_id: "todo_dependency", role: "agent", status: "open", task_class: "advancement_task", archive_state: "active", + resume_when: target.resume_when, resume_ready: false}}; + for (const cyclic of [false, true]) { + const tail = {...todo("todo_tail"), task_class: "advancement_task", + ...(cyclic ? {resume_when: "todo_done:todo_dependency"} : {})}; + const f = await todoSemanticFixture(provider, waiting, [], [target, tail]); + try { + const input = {...request(f.root), ...todoBlockedSemanticWriteback(wait)}; + if (cyclic) await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}, + "The waiting Todo must not qualify a cycle hidden beyond its direct target"); + else assert.equal(((await readQuotaSettlement(input)).writeback_qualification as {payload: {ok: boolean}}).payload.ok, true); + } finally {await rm(f.root, {recursive: true, force: true});} + } + }); +} + + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: no-business semantic writeback cannot discharge a negative checkpoint`, async () => { + const {todo} = await import("./shadow_file_fixture.ts"); + const waiting = {...todo(todoId), task_class: "advancement_task", claimed_by: agentId, + resume_when: "resume_at:2026-01-01T00:06:00Z"}; + const f = await todoSemanticFixture(provider, waiting); + try { + const wait = {schema_version: "quota_blocked_retry_v0", source: "todo", todo_id: todoId, + resume_when: waiting.resume_when, observed_at: "2026-01-01T00:01:00Z", due_at: "2026-01-01T00:06:00Z"}; + const original = todoBlockedSemanticWriteback(wait), candidate = original.writeback_candidate as JsonObject; + const run = candidate.run as JsonObject; + const retry = {...original.refresh_retry as JsonObject, unchanged_reason: "Prior route remains."}; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: agentId, agent_vision: null, existing_agent_vision: null, vision_unchanged_reason: retry.unchanged_reason, + delivery_outcome: "outcome_gap", active_state_next_action_would_update: false, delivery_boundary: "semantic_closeout", + todo_id: todoId, completion_todo_id: null, autonomous_replan_recorded: true, blocked_retry: wait}); + assert.equal(checkpoint.satisfied, false); + assert.equal(checkpoint.decision, "missing_required"); + await assert.rejects(readQuotaSettlement({...request(f.root), refresh_retry: retry, + writeback_candidate: {...candidate, run: {...run, vision_checkpoint: checkpoint, + refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}), + {code: "refresh_task_lease_unavailable"}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} + + +async function recoveryJournalFixture(root: string, goalRef?: JsonObject): Promise<{path: string; key: string}> { + const key = `sha256:${"c".repeat(64)}`; + const directory = join(root, "goals", goalId, "turns"); + await mkdir(directory, {recursive: true}); + const path = join(directory, `${"c".repeat(64)}.json`); + await writeFile(path, JSON.stringify({schema_version: "loopx_turn_journal_v0", goal_id: goalId, turn_key: key, + status: "in_progress", completed_phases: [], plan: {...(goalRef === undefined ? {} : {goal_ref: goalRef}), + turn_envelope: {goal_id: goalId, agent_id: agentId, action: {selected_todo: {todo_id: todoId}}}, + transaction: {turn_key: key, turn_instance_id: turnId, ...(goalRef === undefined ? {} : {goal_ref: goalRef}), + settlement_plan: {schema_version: "quota_settlement_plan_v1", identity: settlementIdentityPayload(identity)}}}})); + return {path, key}; +} + +test("read-only settlement recovery attaches only verified inspection addresses and original evidence", async () => { + const root = await fixture(); + try { + const journal = await recoveryJournalFixture(root); + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const before = await readFile(journal.path), logBefore = await readFile(log); + const ordinary = await readQuotaSettlement(request(root)); + assert.equal(Object.hasOwn(ordinary, "settlement_recovery"), false); + const disabled = await readQuotaSettlement(request(root, {include_readonly_recovery: false})); + assert.deepEqual(disabled, ordinary); + assert.equal(JSON.stringify(disabled).length, JSON.stringify(ordinary).length); + const result = await readQuotaSettlement(request(root, {include_readonly_recovery: true})); + const recovery = requireJsonObject(result.settlement_recovery, "fixture recovery"); + assert.deepEqual(Object.keys(recovery).sort(), ["instruction", "next_cli_actions_argv", "original_settlement_readback", "required", "settlement_identity"]); + assert.equal(recovery.required, true); + assert.deepEqual(recovery.settlement_identity, settlementIdentityPayload(identity)); + assert.deepEqual(recovery.original_settlement_readback, {identity: ordinary.identity, settlement: ordinary.settlement, + progress: ordinary.progress, replay_phase: ordinary.replay_phase, original_prepared_receipt: ordinary.original_prepared_receipt, + original_prepared_event_id: ordinary.original_prepared_event_id, original_prepared_receipt_sha256: ordinary.original_prepared_receipt_sha256}); + assert.ok(JSON.stringify(recovery.original_settlement_readback).length < JSON.stringify(ordinary).length); + assert.deepEqual(recovery.next_cli_actions_argv, [ + ["turn", "inspect-journal", "--goal-id", goalId, "--agent-id", agentId, "--turn-key", journal.key], + ["task-lease", "inspect", "--goal-id", goalId, "--todo-id", todoId]]); + assert.equal(String(recovery.instruction).includes("refresh-state --execution-loss-closeout"), false); + assert.deepEqual(await readFile(journal.path), before); + assert.deepEqual(await readFile(log), logBefore); + const snapshot = await readQuotaSettlementSnapshot(root, goalId); + assert.throws(() => readQuotaSettlementFromSnapshot(request(root, {include_readonly_recovery: true}), snapshot), /live journal read/); + for (const incompatible of [{resume_lease_candidate: {requested_proof: null}}, {writeback_candidate: {}}, + {native_task_lease_cleanup: cleanupRequest}, {infer_turn_instance_id: true}, {allow_unbound_binding: true}, + {resolve_original_binding: true}, {pending_closeout: true}, {refresh_retry: {}}, {heartbeat_reentry_guard: {}}, + {spend_precheck: true}, {tolerate_malformed_history: true}, {agent_id: null}, {todo_id: null}, + {turn_instance_id: null}, {replan_obligation_id: selectedWritebackObligation}, {todo_id: " todo_settlement "}]) { + await assert.rejects(readQuotaSettlement(request(root, {include_readonly_recovery: true, ...incompatible}))); + } + await assert.rejects(readQuotaSettlement(request(root, {include_readonly_recovery: "true"}))); + await writeFile(journal.path, "{truncated"); + assert.deepEqual(await readQuotaSettlement(request(root)), ordinary); + assert.deepEqual(await readQuotaSettlement(request(root, {include_readonly_recovery: false})), ordinary); + await assert.rejects(readQuotaSettlement(request(root, {include_readonly_recovery: true})), {code: "journal_readback_conflict"}); + await rm(journal.path); + await assert.rejects(readQuotaSettlement(request(root, {include_readonly_recovery: true})), {code: "refresh_task_lease_unavailable"}); + } finally {await rm(root, {recursive: true, force: true});} +}); + +test("read-only recovery describes administrative requirements only for original modern positive epoch admission", async () => { + for (const epoch of [50, undefined]) { + const root = await fixture(); + try { + await recoveryJournalFixture(root); + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const original: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", event_id: "original-modern", + goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", details: {todo_id: todoId, + selected_todo_id: todoId, settlement_effect_id: identity.effect_id, closeout_required: true, + ok: true, should_run: true, must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false, + prepared_lease_required: true, prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 119, + ...(epoch === undefined ? {} : {prepared_lease_epoch: epoch})}}; + await writeFile(log, JSON.stringify(original) + "\n"); + const recovery = requireJsonObject((await readQuotaSettlement(request(root, {include_readonly_recovery: true}))).settlement_recovery, + "fixture recovery"); + assert.equal(String(recovery.instruction).includes("refresh-state --execution-loss-closeout"), epoch !== undefined); + if (epoch !== undefined) assert.match(String(recovery.instruction), /current registered control writer and exact current active lease\/CAS/); + assert.deepEqual((recovery.next_cli_actions_argv as string[][]).map(argv => argv.slice(0, 2)), [["turn", "inspect-journal"], ["task-lease", "inspect"]]); + } finally {await rm(root, {recursive: true, force: true});} + } +}); + +test("read-only recovery preserves the original Source lifetime and never falls back to an alias journal", async () => { + const goalRef = {goal_id: goalId, goal_instance_id: instanceA}; + const root = await fixture({goalRef}); + try { + const journal = await recoveryJournalFixture(root, goalRef); + await withSourceAdmission(root, instanceA, instanceA, async binding => { + const input = request(root, {...binding, borrow_source_admission: true, include_readonly_recovery: true}); + const result = await readQuotaSettlement(input); + assert.deepEqual((result.settlement_recovery as JsonObject).goal_ref, goalRef); + await recoveryJournalFixture(root); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}); + await recoveryJournalFixture(root, {goal_id: goalId, goal_instance_id: instanceB}); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}); + }); + assert.ok(journal.path); + } finally {await rm(root, {recursive: true, force: true});} + const alias = await fixture(); + try { + await recoveryJournalFixture(alias, goalRef); + await assert.rejects(readQuotaSettlement(request(alias, {include_readonly_recovery: true})), {code: "refresh_task_lease_unavailable"}); + } finally {await rm(alias, {recursive: true, force: true});} +}); + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: exact active preparation cannot pass an accountable negative checkpoint`, async () => { + const {todo} = await import("./shadow_file_fixture.ts"); + const leaseKey = "exact-active-accountable"; + const lease: JsonObject = {schema_version: "task_lease_v0", goal_id: goalId, todo_id: todoId, owner: agentId, + status: "active", idempotency_key: leaseKey, version: 120, lease_epoch: 50, + acquired_at: "2026-01-01T00:00:00Z", updated_at: "2026-01-01T00:01:00Z", expires_at: "2099-01-01T00:00:00Z"}; + const f = await todoSemanticFixture(provider, {...todo(todoId), claimed_by: agentId, task_class: "advancement_task"}, [lease]); + try { + const positive = {...f.admission, details: {todo_id: todoId, selected_todo_id: todoId, settlement_effect_id: identity.effect_id, + closeout_required: true, prepared_lease_required: true, prepared_lease_key_sha256: createHash("sha256").update(leaseKey).digest("hex"), + prepared_lease_version: 119, prepared_lease_epoch: 50, ok: true, should_run: true, + must_attempt_work: true, delivery_allowed: true, quiet_noop_allowed: false}}; + await writeFile(f.log, JSON.stringify(positive) + "\n"); + const original = todoSemanticWriteback(), candidate = original.writeback_candidate as JsonObject, run = candidate.run as JsonObject; + const retry = {...original.refresh_retry as JsonObject, vision: null, unchanged_reason: "Prior route remains."}; + const checkpoint = buildVisionCheckpoint({schema_version: VISION_REFRESH_REQUEST_SCHEMA, phase: "finalize", + agent_id: agentId, agent_vision: null, existing_agent_vision: null, vision_unchanged_reason: retry.unchanged_reason, + delivery_outcome: "outcome_progress", active_state_next_action_would_update: false, delivery_boundary: "semantic_closeout", + todo_id: todoId, completion_todo_id: null, autonomous_replan_recorded: true, blocked_retry: null}); + assert.equal(checkpoint.decision, "missing_required"); + assert.equal(checkpoint.satisfied, false); + await assert.rejects(readQuotaSettlement({...request(f.root), refresh_retry: retry, writeback_candidate: {...candidate, + requested_proof: {idempotency_key: leaseKey, expected_version: 120}, run: {...run, agent_vision: null, + vision_checkpoint: checkpoint, refresh_recovery: refreshRecovery(decodeRefreshRetry(retry)!, null, false, undefined, false)}}}), + {code: "refresh_task_lease_unavailable"}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} diff --git a/tests/control_plane_ts/turn_journal_query.test.ts b/tests/control_plane_ts/turn_journal_query.test.ts index a4b396c655..b4089f76df 100644 --- a/tests/control_plane_ts/turn_journal_query.test.ts +++ b/tests/control_plane_ts/turn_journal_query.test.ts @@ -6,7 +6,7 @@ import test, { type TestContext } from "node:test"; import { EffectRuntimeConflictError, EffectRuntimeRequestError } from "../../loopx/control_plane/effect_runtime_errors.ts"; import { commitTurnJournal } from "../../loopx/control_plane/turn_driver/turn_journal_effects.ts"; -import { findTurnJournalBySettlement, readTurnJournalCapabilities, readTurnTerminalCloseoutProvider } from "../../loopx/control_plane/turn_driver/turn_journal_query.ts"; +import { findTurnJournalBySettlement, readTurnJournalCapabilities, readTurnTerminalCloseoutProvider, readTurnJournalRecoveryLocator } from "../../loopx/control_plane/turn_driver/turn_journal_query.ts"; import {settlementIdentity, type JsonObject} from "../../loopx/control_plane/effect_program.ts"; const phases = ["host_execute", "typed_result", "validation", "durable_writeback", "quota_spend", "scheduler_apply", "scheduler_ack"]; @@ -298,3 +298,30 @@ test("invalid UTF-8 cannot be normalized into readable history", async t => { await writeFile(path, Buffer.concat([Buffer.from('{"comment":"'), Buffer.from([0xff]), Buffer.from('"}') ])); await assert.rejects(lookup(root), /unreadable/); }); + + +test("settlement lookup optionally binds the original GoalRef without changing its historical default", async t => { + const {root, path} = await fixture(t); + await writeFile(path, JSON.stringify(journal(goalRefA))); + assert.deepEqual(await lookup(root), {turn_key: key}); + assert.deepEqual(await findTurnJournalBySettlement({runtime_root: root, ...selector, goal_ref: goalRefA}), {turn_key: key}); + assert.deepEqual(await findTurnJournalBySettlement({runtime_root: root, ...selector, goal_ref: goalRefB}), {turn_key: null}); + await writeFile(path, JSON.stringify(journal())); + assert.deepEqual(await findTurnJournalBySettlement({runtime_root: root, ...selector, goal_ref: goalRefA}), {turn_key: null}); + for (const ref of [null, {goal_id: "goal"}, {...goalRefA, goal_id: "other-goal"}]) { + await assert.rejects(findTurnJournalBySettlement({runtime_root: root, ...selector, goal_ref: ref}), EffectRuntimeRequestError); + } +}); + +test("read-only recovery locator does not lend a Source journal to the original alias owner", async t => { + const {root, path} = await fixture(t); + const original = settlementIdentity(selector); + await writeFile(path, JSON.stringify(journal())); + assert.equal(await readTurnJournalRecoveryLocator(root, original, {kind: "alias"}), key); + await writeFile(path, JSON.stringify(journal(goalRefA))); + assert.equal(await readTurnJournalRecoveryLocator(root, original, {kind: "alias"}), null); + await writeFile(path, JSON.stringify(journal(goalRefB))); + assert.equal(await readTurnJournalRecoveryLocator(root, original, {kind: "alias"}), null); + await writeFile(path, "{truncated"); + await assert.rejects(readTurnJournalRecoveryLocator(root, original, {kind: "alias"}), {code: "journal_readback_conflict"}); +}); diff --git a/tests/test_steward_report_recovery_and_validator.py b/tests/test_steward_report_recovery_and_validator.py index b97efff389..333909523d 100644 --- a/tests/test_steward_report_recovery_and_validator.py +++ b/tests/test_steward_report_recovery_and_validator.py @@ -363,18 +363,24 @@ def test_steward_report_normalizes_profile_minimum_scale_shorthand(steward, tmp_ assert reports.health(binding["binding_id"])["publication_verified"] -@pytest.mark.parametrize("replace_original_lease", [False, True], ids=["same-generation", "replacement-refused"]) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("lease_loss", ["none", "replacement", "released", "readonly_unavailable"], + ids=["same-generation", "replacement-refused", "released-read-required", "readonly-unavailable"]) def test_frozen_report_recovers_host_result_contract_failure_without_model_call( - steward, tmp_path, monkeypatch, caplog, replace_original_lease, + steward, tmp_path, monkeypatch, caplog, provider, lease_loss, ): from loopx.capabilities.native_chat.steward_monitor import StewardMonitor from loopx.capabilities.native_chat import steward_commands import loopx.capabilities.native_chat.steward_turn as steward_turn_mod from loopx.configure_goal import configure_goal from loopx.control_plane.quota import execution_preparation + from loopx.control_plane.quota import settlement from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.capabilities.periodic_report.cadence_journal import cadence_journal_path - _, runtime, _, transport, binding, _, _ = steward + store, runtime, _, transport, binding, _, _ = steward + replace_original_lease = lease_loss != "none" point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) class CalendarClock(datetime): @@ -384,7 +390,7 @@ def now(cls, tz=None): monkeypatch.setattr(report_host, "datetime", CalendarClock) monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) - sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, provider) owner = binding["binding_id"] assert configure_goal( registry_path=runtime.registry_path, goal_id=actor["goal_id"], @@ -473,20 +479,42 @@ def observe_qualification(**kwargs): monkeypatch.setattr(execution_preparation, "read_turn_execution_lease_qualification", observe_qualification) task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) model_calls = steward[5].read_bytes() + external_writes = list(steward[2].writes) replacement = None + rejected_packet = None + if lease_loss == "readonly_unavailable": + actual_readback = settlement.read_heartbeat_settlement + + def unavailable_readonly_gate(*args, **kwargs): + if kwargs.get("include_readonly_recovery") is True: + raise OSError("fixture read-only recovery transport unavailable") + return actual_readback(*args, **kwargs) + + monkeypatch.setattr(settlement, "read_heartbeat_settlement", unavailable_readonly_gate) if replace_original_lease: actual_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], "--idempotency-key", original_lease["idempotency_key"], "--expected-version", str(original_lease["version"])]) released = actual_command(runtime, ["task-lease", "inspect", *route])["lease"] - replacement = actual_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], - "--idempotency-key", request["turn_instance_id"] + ".replacement", "--ttl-seconds", "3600", - "--expected-version", str(released["version"])])["lease"] - assert replacement["idempotency_key"] != original_lease["idempotency_key"] - assert replacement["lease_epoch"] > original_lease["lease_epoch"] - assert replacement["version"] > original_lease["version"] - with pytest.raises(execution_preparation.RefreshTaskLeaseUnavailable, match="replacement lease"): + if lease_loss == "released": + replacement = released + else: + replacement = actual_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", request["turn_instance_id"] + ".replacement", "--ttl-seconds", "3600", + "--expected-version", str(released["version"])])["lease"] + assert replacement["idempotency_key"] != original_lease["idempotency_key"] + assert replacement["lease_epoch"] > original_lease["lease_epoch"] + assert replacement["version"] > original_lease["version"] + protected = [runtime.registry_path, generation_path, + cadence_journal_path(reports.root, actor["goal_id"]), + turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"])] + protected_bytes = {path: path.read_bytes() for path in protected} + with pytest.raises(execution_preparation.RefreshTaskLeaseUnavailable) as refused: StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + rejected_packet = refused.value.payload + assert rejected_packet["error_code"] == "refresh_task_lease_unavailable" + assert rejected_packet["original_ts_refusal"]["code"] == "refresh_task_lease_unavailable" + assert all(effect is False for effect in rejected_packet["effects"].values()) runtime._steward_monitor_next_check.clear() with caplog.at_level(logging.WARNING): @@ -504,7 +532,7 @@ def observe_qualification(**kwargs): if row["todo_id"] == resumed["loopx_request"]["todo_id"] ) if replace_original_lease: - assert "An original steward report is unavailable" in caplog.text + assert "An original steward report is unavailable" not in caplog.text assert commands == [] and qualifications == [] assert qualification_requests and all(proof == { "idempotency_key": replacement["idempotency_key"], "expected_version": replacement["version"], @@ -515,6 +543,36 @@ def observe_qualification(**kwargs): assert final_readback.spend_run is None and final_readback.spend_event is None assert final_readback.completion_event is None assert transport.steward_reports.reconcile() == 0 + health = reports.health(owner) + assert health["recovery_failure"] == rejected_packet + notice = [row for row in store.messages(sid) if row.get("message_id") == health["recovery_message_id"]] + assert len(notice) == 1 and notice[0]["role"] == "error" + assert notice[0]["turn_id"] == refused.value.report_recovery_context["original_turn_id"] + assert "ready" not in rejected_packet and "can_execute" not in rejected_packet + if lease_loss == "readonly_unavailable": + assert "settlement_recovery" not in rejected_packet + assert "fixture read-only recovery transport unavailable" in rejected_packet["settlement_recovery_error"] + assert "```sh" not in notice[0]["text"] + else: + gate = rejected_packet["settlement_recovery"] + assert gate["required"] is True + assert gate["settlement_identity"] == original_readback.identity.value.as_dict() + assert gate["original_settlement_readback"]["prepared_lease"] == original_prepared + assert gate["next_cli_actions_argv"][0][:2] == ["turn", "inspect-journal"] + assert gate["next_cli_actions_argv"][1][:2] == ["task-lease", "inspect"] + assert len(gate["next_cli_actions"]) == 2 + assert all("--execution-loss-closeout" not in command for command in gate["next_cli_actions"]) + assert all(command in notice[0]["text"] for command in gate["next_cli_actions"]) + assert not {"ready", "can_execute", "control_writer_proof", "claim", "reactivate"} & gate.keys() + messages = store.messages(sid) + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + assert store.messages(sid) == messages + assert reports.health(owner)["recovery_failure"] == rejected_packet + assert steward[5].read_bytes() == model_calls + assert steward[2].writes == external_writes + assert actual_command(runtime, ["task-lease", "inspect", *route])["lease"] == replacement + assert {path: path.read_bytes() for path in protected} == protected_bytes else: assert "An original steward report is unavailable" not in caplog.text assert commands and all(argv[:2] == ["task-lease", "renew"] for argv, _ in commands) @@ -533,6 +591,73 @@ def observe_qualification(**kwargs): assert reports.health(owner)["publication_verified"] +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_local_report_lost_execution_projects_one_readonly_recovery_notice( + steward, tmp_path, monkeypatch, provider, +): + from test_native_steward_reports import configured_local_report + from loopx.capabilities.native_chat import steward_turn as turn_host + from loopx.capabilities.native_chat.steward_commands import native_command + from loopx.control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.capabilities.periodic_report.cadence_journal import cadence_journal_path + + store, runtime, bot, *_ = steward + session, reports, _, _ = configured_local_report(steward, tmp_path, monkeypatch, provider) + sid = session["session_id"] + actor = reports.selected(sid)["actor"] + install_model(steward, tmp_path, sid) + runtime.project_contexts.conversation_bindings = None + with monkeypatch.context() as invalid_candidate: + invalid_candidate.setattr(turn_host, "_normalize_report_kernel_candidate", lambda candidate: { + **{key: value for key, value in candidate.items() if key != "editorial_response"}, + "result_kind": "validated_completion", "delivery_batch_scale": "unsupported", + }) + failed = finish(runtime, pump_steward_monitors(runtime)[0]) + native = failed["response"]["native_loopx"] + assert native["status"] == "failed" and native["validation_stage"] == "host_result_contract" + request = failed["loopx_request"] + _, window = reports.current_window(sid) + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + original_lease = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + native_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], + "--idempotency-key", original_lease["idempotency_key"], "--expected-version", str(original_lease["version"])]) + released = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + replacement = native_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", request["turn_instance_id"] + ".replacement", "--ttl-seconds", "3600", + "--expected-version", str(released["version"])])["lease"] + paths = [runtime.registry_path, store._turn_path(sid, failed["turn_id"]), + turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"]), + cadence_journal_path(reports.root, actor["goal_id"]), + attempt_directory(reports.root, window) / "private-generation.json"] + originals = {path: path.read_bytes() for path in paths} + model_calls, writes = steward[5].read_bytes(), list(bot.writes) + for _ in range(2): + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + health = reports.health(sid) + payload = health["recovery_failure"] + assert payload["error_code"] == RefreshTaskLeaseUnavailable.code + assert payload["original_ts_refusal"]["code"] == RefreshTaskLeaseUnavailable.code + gate = payload["settlement_recovery"] + assert gate["settlement_identity"]["turn_instance_id"] == request["turn_instance_id"] + assert gate["next_cli_actions_argv"][0][:2] == ["turn", "inspect-journal"] + assert gate["next_cli_actions_argv"][1][:2] == ["task-lease", "inspect"] + assert not {"ready", "can_execute", "control_writer_proof", "claim", "reactivate"} & gate.keys() + notices = [row for row in store.messages(sid) if row["message_id"] == health["recovery_message_id"]] + assert len(notices) == 1 and notices[0]["role"] == "error" and notices[0]["turn_id"] == failed["turn_id"] + assert all(command in notices[0]["text"] for command in gate["next_cli_actions"]) + assert {path: path.read_bytes() for path in paths} == originals + assert native_command(runtime, ["task-lease", "inspect", *route])["lease"] == replacement + assert steward[5].read_bytes() == model_calls and bot.writes == writes + readback = read_heartbeat_settlement(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"], registry_path=runtime.registry_path) + assert readback is not None and readback.writeback_run is None and readback.spend_run is None + assert readback.completion_event is None and not health["publication_verified"] + + @pytest.mark.parametrize("cleanup_boundary", ["normal", "rollout_occupied"]) def test_legacy_native_task_lease_cleanup_uses_fenced_release( steward, tmp_path, monkeypatch, cleanup_boundary, From 83812c4344c1f71845db86b32201684d1d67c5a5 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 17:12:04 +0800 Subject: [PATCH 10/12] fix(turn): retain native admission through exact recovery and writeback Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 19 + .../monorepo-distribution-split-v0.zh-CN.md | 15 + docs/reference/protocols/turn-envelope-v0.md | 25 + .../capabilities/native_chat/steward_turn.py | 229 +++-- loopx/cli_commands/quota_should_run.py | 132 ++- loopx/cli_commands/turn.py | 33 + loopx/cli_commands/turn_run_once.py | 47 +- .../goals/first_party_host_admission.py | 27 +- .../quota/accounting_admission.py | 159 +++- .../quota/execution_preparation.py | 23 +- .../quota/heartbeat_receipt_identity.ts | 86 ++ .../quota/settlement_readback.ts | 129 ++- loopx/control_plane/quota/source_admission.ts | 182 +++- loopx/control_plane/turn_driver/executor.py | 6 +- .../turn_driver/turn_journal_effects.ts | 32 +- .../work_items/recommendation_source_io.py | 26 +- .../work_items/replan_history_codec.py | 11 +- loopx/state_refresh.py | 11 +- .../test_quota_execution_preparation.py | 147 +++- .../content_digest_single_owner.test.ts | 1 + .../heartbeat_resume_lease.test.ts | 63 ++ .../quota_settlement_readback.test.ts | 229 +++++ .../turn_journal_effects.test.ts | 191 +++- tests/test_codex_turn_completion.py | 818 +++++++++++++++++- tests/test_native_steward_reports.py | 407 +++++++++ 25 files changed, 2902 insertions(+), 146 deletions(-) diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 40e03434fd..3030d10da1 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4791,6 +4791,7 @@ view shows its independently checked result in the same conversation. ### 2026-11-20 — Close a lost execution attempt without replacement delivery +- **Proposal:** [#9153](https://github.com/loopx-project/loopx/pull/9153). - **Gap:** a replacement task lease cannot authorize the original prepared Turn's business writeback. Repeating the same work guard can therefore leave the original receipt open without a legal recovery operation. @@ -4848,6 +4849,24 @@ view shows its independently checked result in the same conversation. positive prepared admission may explain the existing explicit administrative closeout and its separate current-control-writer requirement; bounded legacy evidence supports inspection only. +- **Fresh native admission:** a new native Turn records its own genuine typed + work admission before Host execution. The shared owner checks prior debt + before preparation and rereads it with current CAS at publication. A saved + candidate supplies content evidence; it cannot borrow the original Turn's + execution proof or clear its debt. Freeze and replay follow the new Turn's + admission, and later recovery and cleanup retain that Turn's identity. + An ordinary registry's exact GoalRef requires its actual persisted instance + and registry/resource witnesses; a nonempty ref does not activate the + SourceSession runtime. Callers without a ref retain their bounded alias + ownership. Historical readback preserves the original journal/receipt owner + rather than substituting the current actor's ref. + Ordinary exact journal checkpoints reuse the same typed native registry + witnesses, bound to the original settlement identity and canonical journal + path. Checkpoint persistence does not grant Host execution authority. + Nested read-only replan projections reuse held native metadata while retaining + original receipt and journal identities. Native writeback verifies the actual + borrowed registry/index scope inside the existing maintenance guard before + avoiding duplicate lock acquisition; source reread and Stop checks remain. - **Source evidence:** CLI tests cover both local providers, stale/foreign proofs, ordinary parity and zero completion/debit. Recovery tests inject interruption after JSON, Markdown, index and receipt durability. SourceSession diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index f68c883fdf..36eb144777 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3701,6 +3701,7 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 ### 2026-11-20 — 收尾失去的执行尝试,不借用替代执行交付 +- **提案:**[#9153](https://github.com/loopx-project/loopx/pull/9153)。 - **缺口:**替代 task lease 不能授权原 prepared Turn 的业务写回。重复同一个 work guard 会让原收据持续开放,却没有合法的恢复操作。 - **增量:**显式行政收尾复用 TS settlement、Goal lifetime 与 File/SQLite @@ -3743,6 +3744,20 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 health packet;检查不授予 execution 或行政写权限。现代 positive prepared admission 可以说明既有显式行政收尾及其独立的当前 control-writer 要求; 有限 legacy 证据仅支持检查。 +- **首次 native 准入:**新 native Turn 在 Host 执行前记录自己的真实 typed + work admission。共享 owner 在 preparation 前检查原债务,并在 publication + 时连同当前 CAS 复读。saved candidate 仅提供内容证据,不能借原 Turn 的 + 执行证明或消除原债务。freeze、replay 发生在新 Turn 准入后,后续恢复与 + cleanup 保持该 Turn 的身份。普通 registry 的 exact GoalRef 必须由真实 + 持久 instance 及 registry/resource witness 证明;非空 ref 不意味着激活 + SourceSession runtime。没有 ref 的 caller 保留有限 alias ownership。 + 历史回读沿原 journal/receipt owner,不以当前 actor 的 ref 替换。 + 普通 exact journal checkpoint 复用同一 typed native registry witness,绑定 + 原 settlement identity 与 canonical journal 路径;checkpoint 持久化不授予 + Host 执行权限。 + 嵌套的只读 replan projection 复用已持有的 native metadata,保留原 receipt + 与 journal 身份。native writeback 在既有 maintenance guard 内验证实际借用的 + registry/index scope 后,避免重复获取同一锁;source 重读和 Stop 检查保留。 - **源码证据:**CLI 测试覆盖两个本地 provider、stale/foreign proof、普通路径 parity 与零 completion/debit;恢复测试在 JSON、Markdown、index 与 receipt 持久化后注入中断。SourceSession 范围是 typed-owner history 加行政 CLI, diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 2793ed27f9..45381e5d0a 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -151,6 +151,20 @@ only. A frozen-report receiver retains its typed lease refusal, publishes an idempotent local notice and exposes the same failure packet in health. Reading the attachment or repeating the notice performs no business or accounting work. +A fresh runtime-owned native Turn must publish its own genuine typed work +admission before Host execution. The shared owner checks prior debt before +preparation and rereads current CAS and debt at publication. A saved candidate +provides content evidence, never the original Turn's execution authority; +freeze and replay follow the new admission. That new Turn's receipt and journal +own completion, subsequent recovery and cleanup. Ordinary registries may carry +an exact GoalRef only with a matching persisted instance and real registry and +resource witnesses. This does not activate generic SourceSession execution. +No-ref callers retain bounded alias ownership; historical readback and resume +retain the original journal/receipt ref without a current-actor substitution. +Ordinary exact journal writes reuse the same typed native registry witnesses +and bind them to the original settlement identity and canonical journal path; +persisting a checkpoint grants no Host execution authority. + `refresh-state --execution-loss-closeout` 显式关闭原 prepared Todo Turn 已失去的 执行尝试。使用相同 Goal、Agent、Todo、Turn,提供 typed blocked / outcome_gap / semantic_closeout 证据及当前 control writer 的确切租约 key/version。TS owner 在真实 @@ -219,6 +233,17 @@ control-writer 要求;legacy 证据仅支持检查。frozen-report receiver 租约拒绝,记录幂等本地提示,并在 health 暴露同一 failure packet。读取 attachment 或重复提示不执行任何业务或记账。 +fresh runtime-owned native Turn 必须在 Host 执行前发布自己的真实 typed work +admission。共享 owner 在 preparation 前检查原债务,在 publication 时复读当前 +CAS 与债务。saved candidate 只提供内容证据,不成为原 Turn 的执行权限; +freeze、replay 沿新准入。完成、后续恢复及 cleanup 由新 Turn 的 receipt/journal +负责。普通 registry 的 exact GoalRef 必须匹配真实持久 instance,并具有实际 +registry/resource witness;这不激活通用 SourceSession 执行。无 ref 的 caller +保留有限 alias ownership;历史回读与 resume 沿原 journal/receipt ref,不以 +当前 actor 替换。 +普通 exact journal 写入复用同一 typed native registry witness,绑定原 settlement +identity 与 canonical journal 路径;checkpoint 持久化不授予 Host 执行权限。 + Newly generated heartbeat and same-Turn commands use the default v1 entry. Historical benchmark runs use the recorded runner and contract version. User task time requirements remain task context. Caller-private deadlines diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 6c79cd5115..43fa557e32 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from contextlib import contextmanager, ExitStack, nullcontext from datetime import datetime, timezone import hashlib @@ -517,7 +517,8 @@ def _task_execution_lease(self, request: Mapping[str, Any], turn_id: str, *, rec actor = request["actor"] # The admitted actor retains its original Goal lifetime. Current # admission can check that reference, never supply a replacement. - goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) + goal_ref = (request["goal_ref"] if "goal_ref" in request else + exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) if actor.get("goal_instance_id") is not None else None) admission = FirstPartyHostGoalAdmission.for_plan( registry_path=self.controller.registry_path, goal_id=actor["goal_id"], @@ -548,7 +549,7 @@ def cleanup_readback(*, source_admission=None, fenced=False): turn_instance_id=request["turn_instance_id"], registry_path=self.controller.registry_path, goal_ref=goal_ref, source_admission=source_admission, borrow_source_admission=source_admission is not None, - native_task_lease_cleanup={"schema_version": "native_task_lease_cleanup_request_v0", + native_task_lease_cleanup={"schema_version": "native_task_lease_request_v0", "recovery": recovery, "admission_retry": admission_retry, "resume_turn_key_present": request.get("resume_turn_key") is not None, "original_journal_present": journal is not None}) @@ -828,7 +829,8 @@ def has_recoverable_stopped_report( ) is not None) @staticmethod - def _build_replay_runner(normalized_candidate: Mapping[str, Any]): + def _build_replay_runner(normalized_candidate: Mapping[str, Any], *, + before_replay: Callable[[Mapping[str, Any]], None] | None = None): from ...control_plane.turn_driver.host_candidate import build_result replayed = False @@ -838,10 +840,88 @@ def replay_frozen_candidate(native_request: Mapping[str, Any], _launch_scope: An if replayed: raise ValueError("frozen report candidate replay may execute at most once") replayed = True + if before_replay is not None: + before_replay(native_request) return build_result(native_request, dict(normalized_candidate), host_name="Codex app-server", allow_completion=True) return replay_frozen_candidate + def _retain_report_recovery_refusal( + self, rejected: Exception, *, request: dict[str, Any], window: Mapping[str, Any], + original: Mapping[str, Any], reports: Any, recovery_key: str | None, + ) -> None: + """Retain the refused native Run while keeping its original Chat audience.""" + from ...cli_commands.turn_rendering import build_turn_error_payload, render_readonly_settlement_recovery + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from ...control_plane.quota.effect_program import SettlementIdentity + from ...control_plane.quota.settlement import read_heartbeat_settlement + from ...control_plane.turn_driver import inspect_loopx_turn_journal + from ...control_plane.turn_driver.journal_store import load_loopx_turn_plan_from_journal + + if not isinstance(rejected, RefreshTaskLeaseUnavailable): + raise TypeError("report recovery requires the original typed lease refusal") + actor = request["actor"] + # This invocation stopped before freeze and the native executor. + # Historical effects belong to the verified journal of the refused Run. + effects = {name: False for name in + ("host_invoked", "state_written", "quota_spent", "scheduler_acknowledged")} + failure = build_turn_error_payload({"transaction": {"turn_key": recovery_key}, + "effects": effects}, rejected, turn_command="run-once") + try: + if recovery_key is None: + raise ValueError("the original report journal is unavailable") + inspection = inspect_loopx_turn_journal(reports.root, goal_id=actor["goal_id"], + agent_id=actor["agent_id"], turn_key=recovery_key, retry_failed=True) + failure = build_turn_error_payload({"transaction": {"turn_key": recovery_key}, + "effects": effects}, rejected, turn_command="run-once", journal_readback=inspection) + recovery_plan = load_loopx_turn_plan_from_journal(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=recovery_key) + failure = build_turn_error_payload({**recovery_plan, "effects": effects}, rejected, + turn_command="run-once", journal_readback=inspection) + identity = SettlementIdentity.from_runtime_payload( + recovery_plan["transaction"]["settlement_plan"]["identity"]) + readback = read_heartbeat_settlement(reports.root, + goal_id=identity.goal_id, agent_id=identity.agent_id, todo_id=identity.todo_id, + turn_instance_id=identity.turn_instance_id, replan_obligation_id=identity.replan_obligation_id, + registry_path=self.controller.registry_path, + goal_ref=recovery_plan.get("goal_ref"), include_readonly_recovery=True) + recovery = readback.settlement_recovery if readback is not None else None + if not isinstance(recovery, Mapping): + raise ValueError("the original read-only settlement recovery could not be read") + failure["settlement_recovery"] = render_readonly_settlement_recovery(recovery, + registry_path=self.controller.registry_path, runtime_root=reports.root) + except (OSError, ValueError, RuntimeError, KeyError, TypeError) as read_error: + # Missing recovery evidence grants no guessed command or permission. + failure["settlement_recovery_error"] = str(read_error) + failure["status"] = "failed" + rejected.payload = failure + rejected.report_recovery_context = {"request": request, "window": window, + "original_turn_id": original["turn_id"]} + if not any(row.get("kind") == "native_loopx.result" and row.get("payload") == failure + for row in self.store.events_after(actor["session_id"], original["turn_id"], None)): + self.store.append_event(actor["session_id"], original["turn_id"], + kind="native_loopx.result", payload=failure) + + @contextmanager + def _report_recovery_lease( + self, *, request: dict[str, Any], window: Mapping[str, Any], original: Mapping[str, Any], + reports: Any, recovery_key: str | None, lease_request: Mapping[str, Any] | None = None, + ): + """A resumed Run qualifies only its own recorded execution identity.""" + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + + with ExitStack() as recovery_leases: + try: + lease = recovery_leases.enter_context(self._task_execution_lease( + lease_request if lease_request is not None else request, + original["turn_id"], recovery=True)) + except RefreshTaskLeaseUnavailable as rejected: + self._retain_report_recovery_refusal(rejected, request=request, window=window, + original=original, reports=reports, recovery_key=recovery_key) + raise + # Later executor errors must retain their actual effect evidence. + yield lease + def _settle_stopped_report_candidate( self, *, @@ -859,8 +939,13 @@ def _settle_stopped_report_candidate( raw_candidate: dict[str, Any], ) -> dict[str, Any]: from ...cli import build_parser + from ...cli_commands.quota_should_run import check_fresh_native_heartbeat_prior_closeout from ...cli_commands.turn import handle_turn_command + from ...control_plane.goals.goal_ref_validation import exact_goal_ref + from ...control_plane.quota.effect_program import SettlementIdentity + from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable from ...control_plane.turn_driver import inspect_loopx_turn_journal + from ...control_plane.turn_driver.codex_execution_preparation import compact_execution_preparation_request from ..periodic_report.private_generation import validate_private_generation inspection = inspect_loopx_turn_journal( @@ -871,14 +956,23 @@ def _settle_stopped_report_candidate( request, observed=observed, settlement_recovery=inspection, frozen_contract_recovery=stopped_proof, ) - if not (folder / "private-generation.json").exists(): - reports.freeze(request, dict(raw_candidate)) - generation = validate_private_generation(folder) - if raw_candidate["editorial_response"]["request_digest"] != generation["request_digest"]: - raise ValueError("the recovered candidate digest does not match the frozen report") if not reports.delivery_current(owner_key, window): raise ValueError("the original report authorization changed before settlement recovery") saved_instance_id = request["turn_instance_id"] + ".saved-candidate" + saved_request = {**request, "turn_instance_id": saved_instance_id} + goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) + if actor.get("goal_instance_id") is not None else None) + # Saved content is evidence, not a grant from the original Run. The + # common owner must close prior debt before any fresh preparation. + try: + check_fresh_native_heartbeat_prior_closeout( + identity=SettlementIdentity(goal_id=actor["goal_id"], agent_id=actor["agent_id"], + todo_id=todo["todo_id"], turn_instance_id=saved_instance_id), + runtime_root=reports.root, registry_path=self.controller.registry_path, goal_ref=goal_ref) + except RefreshTaskLeaseUnavailable as rejected: + self._retain_report_recovery_refusal(rejected, request=request, window=window, + original=original, reports=reports, recovery_key=original_key) + raise argv = ["turn", "run-once", "--goal-id", actor["goal_id"], "--agent-id", actor["agent_id"], "--project", admitted["workspace_path"], "--host", "generic-cli", "--turn-instance-id", saved_instance_id, "--todo-id", todo["todo_id"], "--execute"] @@ -887,8 +981,34 @@ def _settle_stopped_report_candidate( args = build_parser().parse_args(argv) args.host, args.session_scope = "codex-app-server", "agent" result: dict[str, Any] = {} - host_runner = self._build_replay_runner(_normalize_report_kernel_candidate(raw_candidate)) - with self._task_execution_lease(request, original["turn_id"], recovery=True) as lease: + def freeze_saved_candidate(_native_request: Mapping[str, Any]) -> None: + # The driver enters this callback only after fresh C1 admission, + # its own modern receipt and the actual Host launch lifetime. + if not reports.delivery_current(owner_key, window): + raise ValueError("the original report authorization changed before candidate replay") + if not (folder / "private-generation.json").exists(): + reports.freeze(request, dict(raw_candidate)) + generation = validate_private_generation(folder) + if raw_candidate["editorial_response"]["request_digest"] != generation["request_digest"]: + raise ValueError("the recovered candidate digest does not match the frozen report") + if not reports.delivery_current(owner_key, window): + raise ValueError("the original report authorization changed before settlement recovery") + + host_runner = self._build_replay_runner(_normalize_report_kernel_candidate(raw_candidate), + before_replay=freeze_saved_candidate) + with ExitStack() as saved_leases: + def prepare_saved_task(decision: Mapping[str, Any]) -> str | None: + preparation = compact_execution_preparation_request(decision) + if preparation is None: + return None + if (preparation["goal_id"] != actor["goal_id"] or preparation["agent_id"] != actor["agent_id"] + or preparation["todo_id"] != todo["todo_id"] + or preparation.get("turn_instance_id", saved_instance_id) != saved_instance_id): + raise ValueError("saved report preparation belongs to another Turn or Todo") + saved_leases.enter_context(self._task_execution_lease( + saved_request, original["turn_id"], claim_args=preparation["claim_args"])) + return preparation["todo_id"] + handle_turn_command( args, registry_path=self.controller.registry_path, runtime_root_arg=str(reports.root), output_format=lambda *_a, **_kw: "json", @@ -897,8 +1017,8 @@ def _settle_stopped_report_candidate( "schema_version": "loopx_turn_session_binding_v0", "goal_id": actor["goal_id"], "agent_id": actor["agent_id"], }, - native_task_validator=self._report_validator(request), - native_completion_lease_resolver=lambda: self._task_lease_proof(request, lease), + native_execution_preparer=prepare_saved_task, + native_task_validator=self._report_validator(saved_request), native_goal_ref=goal_ref, ) return result @@ -912,7 +1032,9 @@ def recover_report_settlement(self, actor: dict[str, Any], window: dict[str, Any from ...cli import build_parser from ...cli_commands.turn import handle_turn_command from ...control_plane.turn_driver import inspect_loopx_turn_journal - from ...control_plane.turn_driver.journal_store import find_loopx_turn_key_by_settlement_identity + from ...control_plane.turn_driver.journal_store import ( + find_loopx_turn_key_by_settlement_identity, load_loopx_turn_plan_from_journal, + ) from ..periodic_report.private_generation import attempt_directory, validate_private_generation from .steward_reports import report_service observed = self._observe(actor) @@ -933,10 +1055,11 @@ def recover_report_settlement(self, actor: dict[str, Any], window: dict[str, Any reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=todo["todo_id"], turn_instance_id=request["turn_instance_id"] + ".saved-candidate", ) - key = saved_key or find_loopx_turn_key_by_settlement_identity( + original_key = find_loopx_turn_key_by_settlement_identity( reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=todo["todo_id"], turn_instance_id=request["turn_instance_id"], ) + key = saved_key or original_key if key is None: return if saved_key is None: @@ -981,64 +1104,27 @@ def no_model(*_args): "--host", "generic-cli", "--resume-turn-key", key, "--retry-failed-turn", "--execute"]) args.host, args.session_scope = "codex-app-server", "agent" result: dict[str, Any] = {} - from ...cli_commands.turn_rendering import build_turn_error_payload, render_readonly_settlement_recovery - from ...control_plane.goals.goal_ref_validation import exact_goal_ref - from ...control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable - from ...control_plane.quota.effect_program import SettlementIdentity - from ...control_plane.quota.settlement import read_heartbeat_settlement - from ...control_plane.turn_driver.journal_store import load_loopx_turn_plan_from_journal - - with ExitStack() as recovery_leases: - try: - lease = (recovery_leases.enter_context(self._task_execution_lease( - request, original["turn_id"], recovery=True)) if todo["status"] == "open" else None) - except RefreshTaskLeaseUnavailable as rejected: - planned = {"transaction": {"turn_key": key}} - # This invocation stopped before the native executor. Historical - # effects are carried separately by the original journal. - planned = {**planned, "effects": {name: False for name in - ("host_invoked", "state_written", "quota_spent", "scheduler_acknowledged")}} - failure = build_turn_error_payload(planned, rejected, - turn_command="run-once", journal_readback=inspection) - goal_ref = (exact_goal_ref(actor["goal_id"], actor["goal_instance_id"]) - if actor.get("goal_instance_id") is not None else None) - try: - original_plan = load_loopx_turn_plan_from_journal(reports.root, - goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=key) - failure = build_turn_error_payload({**original_plan, "effects": planned["effects"]}, rejected, - turn_command="run-once", journal_readback=inspection) - identity = SettlementIdentity.from_runtime_payload( - original_plan["transaction"]["settlement_plan"]["identity"]) - readback = read_heartbeat_settlement(reports.root, - goal_id=identity.goal_id, agent_id=identity.agent_id, todo_id=identity.todo_id, - turn_instance_id=identity.turn_instance_id, replan_obligation_id=identity.replan_obligation_id, - registry_path=self.controller.registry_path, - goal_ref=goal_ref, include_readonly_recovery=True) - recovery = readback.settlement_recovery if readback is not None else None - if not isinstance(recovery, Mapping): - raise ValueError("the original read-only settlement recovery could not be read") - failure["settlement_recovery"] = render_readonly_settlement_recovery(recovery, - registry_path=self.controller.registry_path, runtime_root=reports.root) - except (OSError, ValueError, RuntimeError, KeyError, TypeError) as read_error: - # Keep the original typed refusal authoritative. Missing - # recovery evidence grants no guessed command or permission. - failure["settlement_recovery_error"] = str(read_error) - failure["status"] = "failed" - rejected.payload = failure - rejected.report_recovery_context = {"request": request, "window": window, - "original_turn_id": original["turn_id"]} - if not any(row.get("kind") == "native_loopx.result" and row.get("payload") == failure - for row in self.store.events_after(actor["session_id"], original["turn_id"], None)): - self.store.append_event(actor["session_id"], original["turn_id"], - kind="native_loopx.result", payload=failure) - raise + lease_request = request + if saved_key is not None: + from ...control_plane.quota.effect_program import SettlementIdentity + saved_plan = load_loopx_turn_plan_from_journal(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=saved_key) + saved_identity = SettlementIdentity.from_runtime_payload( + saved_plan["transaction"]["settlement_plan"]["identity"]) + lease_request = {**request, "turn_instance_id": saved_identity.turn_instance_id, + "todo_id": saved_identity.todo_id, "resume_turn_key": saved_key, + "goal_ref": saved_plan.get("goal_ref")} + lease_context = (self._report_recovery_lease(request=request, window=window, original=original, + reports=reports, recovery_key=key, lease_request=lease_request) + if todo["status"] == "open" else nullcontext(None)) + with lease_context as lease: handle_turn_command(args, registry_path=self.controller.registry_path, runtime_root_arg=str(reports.root), output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), native_host_runner=host_runner, native_session_binding_resolver=lambda _envelope: {"schema_version": "loopx_turn_session_binding_v0", "goal_id": actor["goal_id"], "agent_id": actor["agent_id"]}, - native_task_validator=self._report_validator(request), - **({"native_completion_lease_resolver": lambda: self._task_lease_proof(request, lease)} if lease else {})) + native_task_validator=self._report_validator(lease_request), + **({"native_completion_lease_resolver": lambda: self._task_lease_proof(lease_request, lease)} if lease else {})) # Retain the historical outer failure. The actual native owner # supplies the new event; no status or publication proof is forged. @@ -1370,12 +1456,15 @@ def execute(self, *, session_id: str, turn_id: str, adapter: Any, """Run in the original Chat worker, through all native settlement owners.""" from ...cli import build_parser from ...cli_commands.turn import handle_turn_command + from ...control_plane.goals.goal_ref_validation import exact_goal_ref from ...control_plane.turn_driver.host_candidate import extract_turn_authority, render_prompt, parse_model_json, build_result from .antigravity_adapter import AntigravityAdapter from .codex_adapter import CodexAppServerAdapter turn = self.store.load_turn(session_id, turn_id) request = turn["loopx_request"] + native_goal_ref = (exact_goal_ref(request["actor"]["goal_id"], request["actor"]["goal_instance_id"]) + if request["actor"].get("goal_instance_id") is not None else None) original_client = ("managed." + request["turn_instance_id"].removeprefix("managed-session.") if request["actor"].get("schema_version") == "loopx_managed_session_actor_v0" else native_steward_client_id(request["actor"], request["turn_instance_id"])) @@ -1640,7 +1729,8 @@ def probe_governance(todo_id: str | None) -> dict[str, Any]: handle_turn_command(governance_args, registry_path=self.controller.registry_path, runtime_root_arg=str(self.controller.coordination_runtime_root), output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: observed.update(value), - native_host_runner=run, native_session_binding_resolver=session_binding) + native_host_runner=run, native_session_binding_resolver=session_binding, + native_goal_ref=native_goal_ref) return observed # Probe the original report first. Unbound selection can @@ -1695,6 +1785,7 @@ def steward_validator(_plan: Mapping[str, Any], candidate_result: Mapping[str, A output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), native_host_runner=run, native_session_binding_resolver=session_binding, + native_goal_ref=native_goal_ref, native_execution_preparer=prepare_native_task if not report_work else None, **({"native_completion_lease_resolver": report_lease_proof, "native_task_validator": self._report_validator(request)} if report_work else diff --git a/loopx/cli_commands/quota_should_run.py b/loopx/cli_commands/quota_should_run.py index 9b4bb4eeb9..0a2d96745f 100644 --- a/loopx/cli_commands/quota_should_run.py +++ b/loopx/cli_commands/quota_should_run.py @@ -3,7 +3,8 @@ from __future__ import annotations import argparse -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping +from contextlib import ExitStack, contextmanager, nullcontext from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any @@ -30,6 +31,135 @@ RolloutEventAppender = Callable[..., dict[str, object]] +@contextmanager +def _native_heartbeat_admission_request( + *, identity: SettlementIdentity, runtime_root: Path, registry_path: Path, + goal_ref: Mapping[str, object] | None, goal_admission: Any = None, + publication: bool = False, +) -> Iterator[dict[str, Any]]: + from ..control_plane.coordination.shadow_management import shadow_maintenance_lock_target + from ..control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from ..control_plane.quota.accounting_admission import QUOTA_NATIVE_ADMISSION_SCHEMA, quota_accounting_admission + from ..file_lock import exclusive_cross_runtime_file_lock, exclusive_mutation_file_lock + from ..rollout_event_log import rollout_event_log_path + + root = runtime_root.expanduser().resolve() + registry = registry_path.expanduser().resolve() + admission = goal_admission or FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id=identity.goal_id, planned_goal_ref=goal_ref, agent_id=identity.agent_id) + with quota_accounting_admission(runtime_root=root, registry_path=registry, + goal_id=identity.goal_id, goal_ref=goal_ref, operation="native-work-admission", + handoff_legacy_index=True, native_agent_id=identity.agent_id, + native_publication=publication) as source_admission, ExitStack() as locks: + locks.enter_context(nullcontext() if source_admission is not None + else admission.launch_lifetime(operation="native-work-admission")) + if publication and (source_admission is None + or source_admission.get("schema_version") != QUOTA_NATIVE_ADMISSION_SCHEMA): + locks.enter_context(exclusive_mutation_file_lock(shadow_maintenance_lock_target(root, identity.goal_id), + timeout_seconds=0, operation="native-work-admission")) + locks.enter_context(exclusive_cross_runtime_file_lock(rollout_event_log_path(root, identity.goal_id), + timeout_seconds=0, operation="native-work-admission")) + yield {"schema_version": "loopx_quota_settlement_readback_request_v0", + "runtime_root": str(root), "goal_id": identity.goal_id, "agent_id": identity.agent_id, + "todo_id": identity.todo_id, "turn_instance_id": identity.turn_instance_id, + "infer_turn_instance_id": False, "allow_unbound_binding": False, + **({"goal_ref": dict(goal_ref)} if goal_ref is not None else {}), + **({"source_admission": source_admission, "borrow_source_admission": True} + if source_admission is not None else {})} + + +def _read_native_heartbeat_admission(request: Mapping[str, Any]) -> dict[str, Any]: + from ..control_plane.effect_runtime import EffectRuntimeRemoteError, effect_runtime_result + from ..control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + + try: + result = effect_runtime_result("quota.settlement.read", dict(request), large_local_snapshot=True) + except EffectRuntimeRemoteError as exc: + rejected = RefreshTaskLeaseUnavailable(str(exc)) + rejected.code = exc.diagnostic_code + rejected.payload = {"original_ts_refusal": { + "kind": exc.error_kind, "code": exc.diagnostic_code, + "message": str(exc), "details": list(getattr(exc, "details", ())), + }} + raise rejected from exc + if not isinstance(result, dict) or not isinstance(result.get("native_work_admission"), dict): + raise RefreshTaskLeaseUnavailable("Native work admission returned no typed evidence.") + return result + + +def check_fresh_native_heartbeat_prior_closeout( + *, identity: SettlementIdentity, runtime_root: Path, registry_path: Path, + goal_ref: Mapping[str, object] | None, goal_admission: Any = None, +) -> dict[str, Any]: + """Read same-Todo prior debt before preparation; this grants no execution.""" + from ..control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + + with _native_heartbeat_admission_request(identity=identity, runtime_root=runtime_root, + registry_path=registry_path, goal_ref=goal_ref, goal_admission=goal_admission) as request: + evidence = _read_native_heartbeat_admission({**request, + "native_work_admission": {"check_prior_closeout_only": True}})["native_work_admission"] + if evidence.get("prior_closeout_checked") is not True or evidence.get("append") is not False: + raise RefreshTaskLeaseUnavailable("The original Todo's prior closeout could not be checked.") + return evidence + + +def record_fresh_native_heartbeat_receipt( + decision: Mapping[str, Any], *, identity: SettlementIdentity, + runtime_root: Path, registry_path: Path, goal_ref: Mapping[str, object] | None, + goal_admission: Any, +) -> dict[str, Any]: + """Publish actual native start facts under the existing receipt resources. + + The typed settlement owner alone decides first admission, current proof, + prior debt and immutable replay. No legacy guard is written before it. + """ + from ..control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from ..control_plane.quota.settlement_cli import quota_rollout_details + from ..rollout_event_log import _append_rollout_event_line, build_rollout_event, rollout_event_log_path + + root = runtime_root.expanduser().resolve() + details = quota_rollout_details(decision, + argparse.Namespace(quota_command="should-run", agent_id=identity.agent_id), + todo_id=identity.todo_id, replan_obligation_id=identity.replan_obligation_id) + details.update(turn_instance_id=identity.turn_instance_id, + settlement_effect_id=identity.effect_id, stall_observation="not_applicable") + log_path = rollout_event_log_path(root, identity.goal_id) + with _native_heartbeat_admission_request(identity=identity, runtime_root=root, + registry_path=registry_path, goal_ref=goal_ref, goal_admission=goal_admission, + publication=True) as base_request: + request = {**base_request, "native_work_admission": {"details": details}} + + def read() -> dict[str, Any]: + result = _read_native_heartbeat_admission(request) + if result["native_work_admission"].get("state") != "ready": + raise RefreshTaskLeaseUnavailable("Fresh native work admission could not be qualified.") + return result + + qualified = read()["native_work_admission"] + if qualified.get("append") is True: + offered_details = qualified.get("details") + parent = qualified.get("source_event_id") + if not isinstance(offered_details, dict) or (parent is not None and not isinstance(parent, str)): + raise RuntimeError("typed native admission omitted its publication facts") + event = build_rollout_event(goal_id=identity.goal_id, event_kind="quota_should_run", + goal_ref=goal_ref, agent_id=identity.agent_id, todo_id=identity.todo_id, + run_id=identity.turn_instance_id, status=str(decision.get("effective_action") or "should-run"), + summary=f"Native work admission recorded for turn={identity.turn_instance_id}", + source_event_id=parent, caused_by=parent, details=offered_details) + _append_rollout_event_line(log_path, event) + elif qualified.get("append") is not False: + raise RuntimeError("typed native admission omitted its append disposition") + committed = read() + disposition = committed["native_work_admission"] + receipt = committed.get("heartbeat_receipt") + if (disposition.get("append") is not False or not isinstance(receipt, dict) + or receipt.get("goal_id") != identity.goal_id or receipt.get("agent_id") != identity.agent_id + or receipt.get("run_id") != identity.turn_instance_id + or disposition.get("original_prepared") != qualified.get("original_prepared")): + raise RefreshTaskLeaseUnavailable("Fresh native work admission could not be read back after publication.") + return disposition + + @dataclass(slots=True) class ShouldRunExecutionState: payload: dict[str, object] | None = None diff --git a/loopx/cli_commands/turn.py b/loopx/cli_commands/turn.py index 8f5f220dfa..bb7460722d 100644 --- a/loopx/cli_commands/turn.py +++ b/loopx/cli_commands/turn.py @@ -52,6 +52,7 @@ def handle_turn_command( native_execution_preparer: Callable[[Mapping[str, Any]], str | None] | None = None, native_completion_lease_resolver: Callable[[], Mapping[str, Any]] | None = None, native_task_validator: Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]] | None = None, + native_goal_ref: Mapping[str, object] | None = None, ) -> int | None: if args.command != "turn": return None @@ -90,6 +91,8 @@ def project_preparation_effects(value: dict[str, Any]) -> None: raise ValueError("a native task validator requires its original host connection") if native_execution_preparer is not None and native_host_runner is None: raise ValueError("native task preparation requires its original host connection") + if native_goal_ref is not None and native_host_runner is None: + raise ValueError("native GoalRef requires its original host connection") runtime_root = resolve_status_projection_cache_runtime_root( registry_path=registry_path, runtime_root_override=runtime_root_arg, @@ -182,6 +185,10 @@ def execution_preparer(decision: Mapping[str, Any]) -> str | None: registry_path=registry_path, goal_id=args.goal_id, ) + if native_goal_ref is not None and not getattr(args, "resume_turn_key", None): + # This is caller transport, not a Goal identity inference. The + # native receipt owner verifies it against the locked registry. + goal_ref = dict(native_goal_ref) goal_admission = FirstPartyHostGoalAdmission.for_plan( registry_path=registry_path, goal_id=args.goal_id, @@ -192,6 +199,13 @@ def execution_preparer(decision: Mapping[str, Any]) -> str | None: capture_launch_authority=False, ) strict_goal_admission = goal_admission if goal_admission.enabled else None + fresh_native_start = (native_host_runner is not None and args.host == "codex-app-server" + and args.turn_command == "run-once" and args.execute and not getattr(args, "resume_turn_key", None)) + if fresh_native_start and args.turn_instance_id is None: + from ..turn_identity import mint_turn_instance_id + + args.turn_instance_id = mint_turn_instance_id(prefix="turn") + prior_native_closeout_checked = False if getattr(args, "codex_operation_tools", False) and args.host != "codex-cli": raise ValueError("--codex-operation-tools requires the codex-cli host") dsh_operation_module = getattr(args, "dsh_operation_module", None) @@ -329,9 +343,28 @@ def read_decision_owner(): ) decision_owner = read_decision_owner() + if fresh_native_start and getattr(args, "todo_id", None) and args.turn_instance_id: + from ..control_plane.quota.settlement import SettlementIdentity + from .quota_should_run import check_fresh_native_heartbeat_prior_closeout + + check_fresh_native_heartbeat_prior_closeout( + identity=SettlementIdentity(goal_id=args.goal_id, agent_id=args.agent_id, + todo_id=args.todo_id, turn_instance_id=args.turn_instance_id), + runtime_root=runtime_root, registry_path=registry_path, goal_ref=goal_ref) + prior_native_closeout_checked = True operator_inbox_urgency_projector = decision_owner.operator_inbox_urgency_projector scheduler_context = decision_owner.scheduler_execution_context decision = compose_turn_decision(decision_owner.resolve(), scheduler_execution_context=scheduler_context) + if fresh_native_start and not prior_native_closeout_checked and args.turn_instance_id: + from ..control_plane.quota.settlement import SettlementIdentity + from .quota_should_run import check_fresh_native_heartbeat_prior_closeout + + selected = decision.get("selected_todo") + if isinstance(selected, Mapping) and selected.get("todo_id"): + check_fresh_native_heartbeat_prior_closeout( + identity=SettlementIdentity(goal_id=args.goal_id, agent_id=args.agent_id, + todo_id=selected["todo_id"], turn_instance_id=args.turn_instance_id), + runtime_root=runtime_root, registry_path=registry_path, goal_ref=goal_ref) resume_requested, session_binding = resolve_turn_resume_session_binding(args) if execution_preparer is not None and args.turn_command == "run-once" and args.execute: prepared_todo = execution_preparer(decision) diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 6253b0b7a9..7837c86b73 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -782,11 +782,41 @@ def current_launch_decision(current_registry: Path) -> Mapping[str, Any]: if args.host == "codex-app-server": native_admission = (goal_admission or FirstPartyHostGoalAdmission.for_plan( registry_path=registry_path, goal_id=args.goal_id, planned_goal_ref=goal_ref, + agent_id=args.agent_id, )).with_launch_decision(current_launch_decision) + @contextmanager + def native_launch_lifetime(): + if native_admission.source_profile: + with native_admission.launch_lifetime(operation="native_conversation_turn_launch"): + yield + return + from ..control_plane.goals.first_party_host_admission import _legacy_goal_authority + from ..control_plane.projects.registry_codec import load_project_registry + from ..control_plane.quota.accounting_admission import QUOTA_NATIVE_ADMISSION_SCHEMA, quota_accounting_admission + + # Enter index -> registry first. Nested live reads borrow this + # same resource envelope instead of taking an index under a + # held registry. Both current decision and Goal witness are + # consumed by the existing typed launch owner before dispatch. + with quota_accounting_admission(runtime_root=runtime_root, registry_path=registry_path, + goal_id=args.goal_id, goal_ref=goal_ref, native_agent_id=args.agent_id, + operation="native_conversation_turn_launch") as admission: + if admission is None or admission.get("schema_version") != QUOTA_NATIVE_ADMISSION_SCHEMA: + raise RefreshTaskLeaseUnavailable("Native launch resource ownership changed before dispatch.") + witnesses = {Path(witness["path"]): witness for witness in admission["registry_sources"]} + source = native_admission + while source is not None: + source._decision("launch", + legacy_authority=_legacy_goal_authority(load_project_registry(source.registry_path), + source.goal_id, source.agent_id), + registry_witness=witnesses[source.registry_path], + launch_decision=current_launch_decision(source.registry_path)) + source = source.legacy_source_admission + yield + def run_native_host(request: Mapping[str, Any]) -> dict[str, Any]: - return native_host_runner(request, lambda: native_admission.launch_lifetime( - operation="native_conversation_turn_launch")) + return native_host_runner(request, native_launch_lifetime) host_runner = run_native_host session_binding_resolver = native_session_binding_resolver @@ -878,16 +908,25 @@ def bind_heartbeat_settlement_receipt() -> None: else None ), goal_ref=goal_ref, - task_lease_proof=None if resume_execution else task_lease_proof, - task_lease_epoch=None if resume_execution else task_lease_epoch, + task_lease_proof=None if resume_execution or fresh_native_start else task_lease_proof, + task_lease_epoch=None if resume_execution or fresh_native_start else task_lease_epoch, ) def on_managed_start_admitted() -> None: if settlement_identity.request_subject is not None: require_current_request_operation(request_locator, settlement_identity, registry_path=registry_path, runtime_root=runtime_root, workspace=project) + if fresh_native_start: + from .quota_should_run import record_fresh_native_heartbeat_receipt + + current = current_launch_decision(registry_path)["current"] + record_fresh_native_heartbeat_receipt(current, + identity=settlement_identity, runtime_root=runtime_root, + registry_path=registry_path, goal_ref=goal_ref, goal_admission=native_admission) bind_heartbeat_settlement_receipt() + fresh_native_start = (args.host == "codex-app-server" and native_host_runner is not None + and not getattr(args, "resume_turn_key", None) and settlement_identity.todo_id is not None) managed_cadence = managed_cadence_start( runtime_root=runtime_root, goal_id=args.goal_id, diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index 0189c55810..da11ebee5a 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -357,15 +357,12 @@ def for_plan( requested_registry = registry_path.expanduser().resolve() require_goal_id(goal_id) agent_id = normalize_todo_claimed_by(agent_id) - source_profile = planned_goal_ref is not None + source_profile = (requested_registry.is_file() + and load_project_registry(requested_registry).get("profile_id") == SOURCE_SESSION_PROFILE_ID) legacy_registry_witness = None legacy_source_admission = None legacy_running_authority_sha256 = None if not source_profile and requested_registry.is_file(): - source_profile = ( - load_project_registry(requested_registry).get("profile_id") - == SOURCE_SESSION_PROFILE_ID - ) if not source_profile and capture_launch_authority: with project_registry_transaction( requested_registry, operation="first_party_host_launch_capture", @@ -610,10 +607,26 @@ def source_journal_admission( *, runtime_root: Path, ) -> Iterator[dict[str, Any] | None]: - """Hand one journal mutation to the TS owner under the source guard.""" + """Hand one journal mutation to TS under its actual registry profile.""" if not self.source_profile: - yield None + if self.planned_goal_ref is None: + yield None + return + from ..quota.accounting_admission import quota_accounting_admission + + # Ordinary exact journals retain their native registry resources; + # an already-held launch scope is borrowed without another index. + with quota_accounting_admission( + runtime_root=runtime_root, registry_path=self.registry_path, + goal_id=self.goal_id, goal_ref=self.planned_goal_ref, + native_agent_id=self.agent_id, operation="first_party_host_journal_commit", + ) as admission: + with exclusive_cross_runtime_file_lock( + runtime_artifact_lock_target(runtime_root.expanduser().resolve(), self.goal_id), + operation="first_party_host_journal_runtime_artifact_guard", + ): + yield admission return target = guard_path(self.registry_path, self.goal_id) with exclusive_cross_runtime_file_lock( diff --git a/loopx/control_plane/quota/accounting_admission.py b/loopx/control_plane/quota/accounting_admission.py index 3ea97a9eea..93c71d9d25 100644 --- a/loopx/control_plane/quota/accounting_admission.py +++ b/loopx/control_plane/quota/accounting_admission.py @@ -1,7 +1,8 @@ from __future__ import annotations from collections.abc import Iterator, Mapping -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager +from contextvars import ContextVar from pathlib import Path from typing import Any @@ -9,12 +10,105 @@ cross_runtime_lock_witness, exclusive_cross_runtime_file_lock, exclusive_file_lock, + exclusive_mutation_file_lock, exclusive_run_index_lock, ) from ..projects.registry_codec import SOURCE_SESSION_PROFILE_ID QUOTA_SOURCE_ADMISSION_SCHEMA = "loopx_quota_source_admission_v0" +QUOTA_NATIVE_ADMISSION_SCHEMA = "loopx_quota_native_admission_v0" +_held_native_admission: ContextVar[tuple[str, dict[str, Any]] | None] = ContextVar( + "loopx_held_native_accounting_admission", default=None) + + +def _borrow_native_admission( + *, root: Path, registry: Path | None, goal_id: str, planned: dict[str, str] | None, + agent_id: str | None, publication: bool, +) -> dict[str, Any] | None: + held = _held_native_admission.get() + if held is None: + return None + held_goal, admission = held + sources = admission["registry_sources"] + if (held_goal != goal_id or admission["runtime_root"] != str(root) + or admission["planned_goal_ref"] != planned or registry is None + or str(registry) not in {source["path"] for source in sources} + or (agent_id is not None and admission.get("agent_id") != agent_id)): + raise RuntimeError("Nested native accounting does not match its held resource scope.") + if publication and not any(lock["role"] == "authority_maintenance" for lock in admission["locks"]): + raise RuntimeError("Nested native publication requires its maintenance and receipt resources.") + # A source launch read can borrow the already-held source subset of a + # mirror admission. TS still verifies its complete bytes, route and owner. + selected = sources if str(registry) == admission["registry_path"] else [ + source for source in sources if source["path"] == str(registry)] + paths = {source["path"] for source in selected} + return {**admission, "registry_path": str(registry), + "registry_sources": [dict(source) for source in selected], + "locks": [dict(lock) for lock in admission["locks"] + if lock["role"] != "registry_source" or lock["target"] in paths]} + + +def _borrow_native_read_admission( + *, runtime_root: Path, registry_path: Path | None, goal_id: str, + goal_ref: Mapping[str, Any] | None, agent_id: str | None, +) -> dict[str, Any] | None: + """Borrow actual exact native metadata for a read-only projection. + + Missing internal read context can use its enclosing resources. An explicit + context still has to match, and ordinary accounting never uses this seam. + """ + held = _held_native_admission.get() + if held is None: + return None + admission = held[1] + if admission["planned_goal_ref"] is None: + return None + root = runtime_root.expanduser().resolve() + registry = (registry_path or Path(admission["registry_path"])).expanduser().resolve() + planned = _planned_goal_ref( + goal_ref if goal_ref is not None else admission["planned_goal_ref"], goal_id=goal_id) + borrowed = _borrow_native_admission( + root=root, registry=registry, goal_id=goal_id, planned=planned, + agent_id=agent_id if agent_id is not None else admission.get("agent_id"), publication=False) + if borrowed is None: + return None + return {**borrowed, "locks": [dict(lock) for lock in borrowed["locks"] + if lock["role"] in {"run_index", "registry_source"}]} + + +def _require_held_native_registry_resources( + *, runtime_root: Path, registry_path: Path, goal_id: str, + goal_ref: Mapping[str, Any] | None, source_admission: Mapping[str, Any], + source_registry: Path, agent_id: str | None = None, +) -> None: + """Match offered registry/index facts to resources this caller still holds. + + Qualification and source-state checks stay with the existing owners. This + resource verifier cannot derive a business Ref or release its caller's locks. + """ + native_agent = source_admission.get("agent_id") + if not isinstance(goal_ref, Mapping) or (native_agent is not None and native_agent != agent_id): + raise RuntimeError("The native writer does not match its explicit GoalRef and Agent.") + root = runtime_root.expanduser().resolve() + invoked = registry_path.expanduser().resolve() + registries = list(dict.fromkeys((source_registry.expanduser().resolve(), invoked))) + borrowed = _borrow_native_admission( + root=root, registry=invoked, goal_id=goal_id, planned=dict(goal_ref), + agent_id=native_agent, publication=False) + targets = [root / "goals" / goal_id / "runs" / "index.jsonl", *registries] + if (borrowed is None or borrowed != dict(source_admission) + or [witness["path"] for witness in borrowed["registry_sources"]] != [str(path) for path in registries]): + raise RuntimeError("The native writer lost its enclosing resource scope.") + metadata_locks = [witness for witness in borrowed["locks"] + if witness["role"] in {"run_index", "registry_source"}] + if [(witness["role"], witness["target"]) for witness in metadata_locks] != [ + ("run_index" if index == 0 else "registry_source", str(path)) + for index, path in enumerate(targets)]: + raise RuntimeError("The native writer source and index resources changed.") + for path, witness in zip(targets, metadata_locks): + if cross_runtime_lock_witness(path) != {key: witness[key] for key in ("target", "pid", "token")}: + raise RuntimeError("The native writer resource owner changed.") def _planned_goal_ref( @@ -41,12 +135,43 @@ def _uses_source_profile( ) -> bool: if registry_path is None or not registry_path.is_file(): return False - from ..goals.first_party_host_admission import source_goal_authority + from ..projects.registry_codec import load_project_registry + + return load_project_registry(registry_path).get("profile_id") == SOURCE_SESSION_PROFILE_ID - return source_goal_authority(registry_path, goal_id).get("kind") in { - "present", - "absent", - } + +@contextmanager +def _native_accounting_admission( + *, root: Path, registry: Path, goal_id: str, planned: dict[str, str] | None, + operation: str, publication: bool, +) -> Iterator[dict[str, Any]]: + from ..coordination.shadow_management import shadow_maintenance_lock_target + from ..goals.first_party_host_admission import _legacy_source_registry + from ..projects.registry_codec import load_project_registry, project_registry_transaction + from ...rollout_event_log import rollout_event_log_path + + # Discover only resources here. The TS owner verifies the locked source + # route, actual registry role, exact Goal metadata, Agent and runtime. + source = _legacy_source_registry(load_project_registry(registry), goal_id, registry) + sources = list(dict.fromkeys([*([source] if source is not None else []), registry])) + index = root / "goals" / goal_id / "runs" / "index.jsonl" + maintenance = shadow_maintenance_lock_target(root, goal_id) + receipt = rollout_event_log_path(root, goal_id) + with exclusive_run_index_lock(index, operation=operation), ExitStack() as locks: + registry_sources = [] + for path in sources: + transaction = locks.enter_context(project_registry_transaction(path, operation=operation)) + registry_sources.append(transaction.source_read_witness()) + if publication: + locks.enter_context(exclusive_mutation_file_lock(maintenance, timeout_seconds=0, operation=operation)) + locks.enter_context(exclusive_cross_runtime_file_lock(receipt, timeout_seconds=0, operation=operation)) + yield {"schema_version": QUOTA_NATIVE_ADMISSION_SCHEMA, "registry_path": str(registry), + "planned_goal_ref": planned, "runtime_root": str(root), + "registry_sources": registry_sources, + "locks": [{"role": "run_index", **cross_runtime_lock_witness(index)}, + *({"role": "registry_source", **cross_runtime_lock_witness(path)} for path in sources), + *([{"role": "authority_maintenance", **cross_runtime_lock_witness(maintenance)}, + {"role": "heartbeat_receipt", **cross_runtime_lock_witness(receipt)}] if publication else [])]} @contextmanager @@ -59,16 +184,34 @@ def quota_accounting_admission( operation: str, lock_legacy_index: bool = True, handoff_legacy_index: bool = False, + native_agent_id: str | None = None, + native_publication: bool = False, ) -> Iterator[dict[str, Any] | None]: """Hold quota's index/source locks until the TypeScript owner adopts them.""" root = runtime_root.expanduser().resolve() registry = registry_path.expanduser().resolve() if registry_path else None planned = _planned_goal_ref(goal_ref, goal_id=goal_id) + borrowed = _borrow_native_admission(root=root, registry=registry, goal_id=goal_id, + planned=planned, agent_id=native_agent_id, publication=native_publication) + if borrowed is not None: + yield borrowed + return source_profile = _uses_source_profile(registry, goal_id) if not source_profile: - if planned is not None: - raise ValueError("quota GoalRef requires a source-session registry") + if planned is not None or native_agent_id is not None: + if registry is None: + raise ValueError("native quota GoalRef requires its registry") + with _native_accounting_admission(root=root, registry=registry, goal_id=goal_id, + planned=planned, operation=operation, publication=native_publication) as admission: + if native_agent_id is not None: + admission["agent_id"] = native_agent_id + token = _held_native_admission.set((goal_id, admission)) + try: + yield admission + finally: + _held_native_admission.reset(token) + return if not lock_legacy_index: yield None return diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 6ccc42b8d9..face4d9e9c 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -440,10 +440,25 @@ def guard_first_writeback_execution( if goal is None: source, _ = load_recommendation_source_goal(registry_path, goal_id) invoked = registry_path.resolve() - for registry in dict.fromkeys((source, invoked)): - locks.enter_context(exclusive_cross_runtime_file_lock( - registry, timeout_seconds=0, operation="refresh-state-execution", - )) + registries = list(dict.fromkeys((source, invoked))) + if source_admission is not None and source_admission.get("schema_version") == "loopx_quota_native_admission_v0": + from .accounting_admission import _require_held_native_registry_resources + + # This is resource reuse only. Business authority remains + # the original Turn's inside-M TS qualification above. + try: + _require_held_native_registry_resources( + runtime_root=runtime_root, registry_path=registry_path, goal_id=goal_id, + goal_ref=goal_ref, source_admission=source_admission, + source_registry=source, agent_id=agent_id, + ) + except RuntimeError as exc: + raise RefreshTaskLeaseUnavailable("The native writeback resources are no longer held.") from exc + else: + for registry in registries: + locks.enter_context(exclusive_cross_runtime_file_lock( + registry, timeout_seconds=0, operation="refresh-state-execution", + )) current_source, goal = load_recommendation_source_goal(registry_path, goal_id) if current_source != source or goal is None: raise RefreshTaskLeaseUnavailable("The Goal source changed before this writeback committed.") diff --git a/loopx/control_plane/quota/heartbeat_receipt_identity.ts b/loopx/control_plane/quota/heartbeat_receipt_identity.ts index ec34f54219..bcd474ab32 100644 --- a/loopx/control_plane/quota/heartbeat_receipt_identity.ts +++ b/loopx/control_plane/quota/heartbeat_receipt_identity.ts @@ -612,6 +612,92 @@ export function isHeartbeatLeaseReadiness(event: JsonObject): boolean { return heartbeatReceiptDetails(event).settlement_receipt_revision === "lease_readiness"; } +/** A runtime-owned fresh native Turn publishes its real prepared decision. + * Current CAS is checked here; an existing admission keeps its immutable proof. + * Status compatibility and another Turn's resume grant are never inputs. */ +export function qualifyHeartbeatNativeWorkAdmission(input: { + identity: SettlementIdentity; currentDetails: JsonObject; + existingReceipt: JsonObject | null; receiptHistory: readonly JsonObject[]; + todo: JsonObject; lease: JsonObject; handoffMode: unknown; + replayPhase: ReceiptBoundReplayPhase; closeoutStarted: boolean; observedAt: Date; +}): JsonObject { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } + const {identity, currentDetails, lease} = input; + if (identity.todo_id == null || identity.replan_obligation_id != null || identity.request_subject != null + || input.todo.role !== "agent" || lease.goal_id !== identity.goal_id + || currentDetails.todo_id !== identity.todo_id || currentDetails.selected_todo_id !== identity.todo_id + || currentDetails.turn_instance_id !== identity.turn_instance_id + || currentDetails.settlement_effect_id !== identity.effect_id + || currentDetails.closeout_required !== true || currentDetails.prepared_lease_required !== true + || currentDetails.delivery_allowed !== true + || Object.hasOwn(currentDetails, "settlement_receipt_revision") + || nativeChildReportAdmission(currentDetails, null, identity.effect_id, "open", false) + .report_permission !== "new_operation") { + reject("Fresh native admission requires the same Turn's complete explicit prepared work decision."); + } + const candidatePrepared = heartbeatPreparedLease(currentDetails); + if (candidatePrepared === null || candidatePrepared.prepared_lease_epoch === undefined + || candidatePrepared.prepared_lease_version !== lease.version) { + reject("Fresh native preparation must match the current canonical lease CAS and epoch."); + } + const candidate: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", + goal_id: identity.goal_id, agent_id: identity.agent_id, run_id: identity.turn_instance_id, + details: currentDetails}; + const currentProof = {idempotency_key: lease.idempotency_key, expected_version: lease.version}; + qualifyHeartbeatResumeLease({originalReceipt: candidate, identity, requestedProof: currentProof, + todo: input.todo, lease, handoffMode: input.handoffMode, exactSource: true, + completionEvent: null, replayPhase: input.replayPhase, closeoutStarted: input.closeoutStarted, + writebackPending: false, otherTerminalStarted: input.closeoutStarted, observedAt: input.observedAt}); + const existing = input.existingReceipt; + if (existing === null) { + qualifyHeartbeatPreparedPublication({schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: null, original_prepared: null, receipt_history: [], + settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields: {prepared_lease_required: true, ...candidatePrepared}}); + return {state: "ready", append: true, details: structuredClone(currentDetails), source_event_id: null, + original_prepared: {...candidatePrepared}, current_task_lease_proof: currentProof}; + } + const validation = heartbeatValidationEvent(input.receiptHistory, existing, identity, false); + if (validation === null) reject("The existing native Turn's receipt lineage is incomplete."); + const original = heartbeatPreparedWorkAdmission(validation, identity, false); + if (original !== null) { + const originalDetails = heartbeatReceiptDetails(validation); + if (!isDeepStrictEqual(projectSemanticReplanGuard(originalDetails), projectSemanticReplanGuard(currentDetails)) + || (originalDetails.quota_spend_source != null && originalDetails.quota_spend_source !== currentDetails.quota_spend_source)) { + reject("Fresh native replay cannot replace the original admission's semantic or spend source."); + } + qualifyHeartbeatResumeLease({originalReceipt: validation, identity, requestedProof: currentProof, + todo: input.todo, lease, handoffMode: input.handoffMode, exactSource: true, + completionEvent: null, replayPhase: input.replayPhase, closeoutStarted: input.closeoutStarted, + writebackPending: false, otherTerminalStarted: input.closeoutStarted, observedAt: input.observedAt}); + return {state: "ready", append: false, details: null, source_event_id: null, + original_prepared: {...original}, current_task_lease_proof: currentProof}; + } + // The existing reentry owner alone may qualify a genuine negative precursor. + // In particular, an ensure-only legacy-positive receipt is not promoted. + const previousDetails = heartbeatReceiptDetails(existing); + if (["ok", "should_run", "must_attempt_work", "delivery_allowed", "quiet_noop_allowed"] + .some(field => typeof previousDetails[field] !== "boolean") + || previousDetails.closeout_required !== false || previousDetails.prepared_lease_required !== true) { + reject("Only a complete explicit negative work receipt can precede native admission."); + } + qualifyHeartbeatPreparedPublication({schema_version: HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, + existing_receipt: existing, original_prepared: null, receipt_history: [...input.receiptHistory], + settlement_identity: settlementIdentityPayload(identity), + proposed_prepared_fields: {prepared_lease_required: true, ...candidatePrepared}}); + const qualified = heartbeatWorkRequalification(existing, currentDetails, identity, + input.replayPhase, input.closeoutStarted); + const qualifiedDetails = jsonObject(qualified.details); + if (qualified.append !== true || qualifiedDetails?.settlement_receipt_revision !== "work_admission" + || !optionalHeartbeatString(existing.event_id)) { + reject("The existing receipt does not admit a fresh native work qualification."); + } + return {state: "ready", append: true, details: qualifiedDetails, source_event_id: existing.event_id!, + original_prepared: {...candidatePrepared}, current_task_lease_proof: currentProof}; +} + /** Reentry can qualify work on the original binding, never replace its authority. * The caller commits this append-only projection under the receipt-log lock. */ export function heartbeatWorkRequalification( diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 2e03ea69aa..5da814411a 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -52,6 +52,7 @@ import { isTurnScopedSettlementOutcome } from "../work_items/delivery_outcome.ts import {EXECUTION_LOSS_CLOSEOUT_SCHEMA, isExecutionLossCloseout} from "./execution_loss_closeout_record.ts"; import {isDeepStrictEqual} from "node:util"; import {canonicalAuthoritySha256} from "../coordination/authority_store_codec.ts"; +import {parseExactGoalRef} from "../goals/goal_instance_identity.ts"; import { decodeRefreshRetry, isMaterialMonitorPoll, @@ -74,6 +75,7 @@ import { heartbeatSemanticAdmissionEvent, qualifyHeartbeatPreparedGeneration, qualifyHeartbeatResumeLease, + qualifyHeartbeatNativeWorkAdmission, type HeartbeatReceiptFact, } from "./heartbeat_receipt_identity.ts"; @@ -92,6 +94,7 @@ import { parseQuotaAccountingOwner, quotaGoalRef, quotaOwnerOwnsProjection, + requireNativeQuotaWorkAgent, withBorrowedQuotaAccountingOwner, withQuotaAccountingOwner, type QuotaAccountingOwner, @@ -153,6 +156,25 @@ export function projectNativeTaskLeaseCleanup(readback: JsonObject, value: unkno if (["work_admission", "lease_readiness", "identity_upgrade"].includes(String(revision)) && required === true) { return disposition("retain", "original_turn_owes_closeout"); } + // Primary positive receipts predate revision markers. Their explicit work + // facts and verified original prepared admission carry the debt themselves; + // status-only ensure and partial projections still remain unconfirmed. + const original = jsonObject(readback.original_prepared_receipt); + const identityValue = jsonObject(identity?.value); + if (!Object.hasOwn(receiptDetails, "settlement_receipt_revision") && required === true + && original !== null && identityValue !== null && typeof identityValue.goal_id === "string" + && typeof identityValue.agent_id === "string" && typeof identityValue.todo_id === "string" + && typeof identityValue.turn_instance_id === "string") { + const exactIdentity = settlementIdentity({goal_id: identityValue.goal_id, agent_id: identityValue.agent_id, + todo_id: identityValue.todo_id, turn_instance_id: identityValue.turn_instance_id}); + if (heartbeatPreparedWorkAdmission(original, exactIdentity, false) !== null + && nativeChildReportAdmission(details(original), null, exactIdentity.effect_id, "open", false) + .report_permission === "new_operation" + && nativeChildReportAdmission(receiptDetails, null, exactIdentity.effect_id, "open", false) + .report_permission === "new_operation") { + return disposition("retain", "original_turn_owes_closeout"); + } + } return disposition("unconfirmed", "original_closeout_requirement_unconfirmed"); } @@ -205,6 +227,7 @@ interface ReadbackRequest { tolerate_malformed_history: boolean; refresh_retry: RefreshRetryRequest | null; heartbeat_reentry_guard: JsonObject | null; + native_work_admission: JsonObject | null; resume_lease_candidate: ResumeLeaseCandidate | null; writeback_candidate: WritebackCandidate | null; native_task_lease_cleanup: NativeTaskLeaseCleanupRequest | null; @@ -452,19 +475,38 @@ function decodeRequest( "quota_source_admission_invalid", ); } + if (admittedOwner?.admission?.kind === "native_runtime" && admittedOwner.admission.goalId !== goalId) { + throw new EffectRuntimeRequestError("Native admitted owner does not match settlement goal_id", "quota_source_admission_invalid"); + } const owner = admittedOwner ?? parseQuotaAccountingOwner({ goalRefValue: request.goal_ref, sourceAdmissionValue: request.source_admission, runtimeRoot, goalId, }); - if (request.borrow_source_admission === true && owner.kind !== "exact_source") { + if (request.borrow_source_admission === true && owner.kind !== "exact_source" + && owner.admission?.kind !== "native_runtime") { throw new EffectRuntimeRequestError( "borrow_source_admission requires exact source admission", "quota_source_admission_invalid", ); } const resumeCandidate = decodeResumeLeaseCandidate(request.resume_lease_candidate); + const nativeAdmission = request.native_work_admission === undefined ? null + : requireJsonObject(request.native_work_admission, "native_work_admission"); + if (nativeAdmission !== null && (Object.keys(nativeAdmission).length !== 1 + || (jsonObject(nativeAdmission.details) === null && nativeAdmission.check_prior_closeout_only !== true) + || resumeCandidate !== null + || request.writeback_candidate != null || request.native_task_lease_cleanup != null + || request.include_readonly_recovery === true || request.infer_turn_instance_id || request.allow_unbound_binding + || request.resolve_original_binding === true || request.pending_closeout === true + || request.refresh_retry != null || request.heartbeat_reentry_guard != null || request.spend_precheck === true + || request.spend_request != null || request.tolerate_malformed_history === true + || !optionalRequestString(request.agent_id, "agent_id") || !optionalRequestString(request.todo_id, "todo_id") + || !optionalRequestString(request.turn_instance_id, "turn_instance_id") + || optionalRequestString(request.replan_obligation_id, "replan_obligation_id") !== null)) { + throw new EffectRuntimeRequestError("native work admission requires only its exact fresh Agent/Todo/Turn decision"); + } if (resumeCandidate !== null && (request.infer_turn_instance_id || request.allow_unbound_binding || request.pending_closeout === true || optionalRequestString(request.agent_id, "agent_id") === null @@ -540,6 +582,7 @@ function decodeRequest( refresh_retry: decodeRefreshRetry(request.refresh_retry), heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null : requireJsonObject(request.heartbeat_reentry_guard, "heartbeat_reentry_guard"), + native_work_admission: nativeAdmission, resume_lease_candidate: resumeCandidate, writeback_candidate: writebackCandidate, native_task_lease_cleanup: nativeCleanup, @@ -1796,9 +1839,9 @@ export function readQuotaSettlementFromSnapshot( ): JsonObject { const request = decodeRequest(value); if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); - if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null || request.native_work_admission !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); - if (request.owner.kind !== "alias") { + if (request.owner.kind !== "alias" || request.owner.admission !== undefined) { throw new EffectRuntimeRequestError( "exact source settlement readback requires live owner admission", "quota_source_admission_invalid", @@ -1814,7 +1857,7 @@ export function readAdmittedQuotaSettlementFromSnapshot( ): JsonObject { const request = decodeRequest(value); if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); - if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null || request.native_work_admission !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); } @@ -1827,7 +1870,7 @@ export function readQuotaSettlementForAdmittedOwnerFromSnapshot( ): JsonObject { const request = decodeRequest(value, owner); if (request.include_readonly_recovery) throw new EffectRuntimeRequestError("read-only recovery requires a live journal read"); - if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) throw new EffectRuntimeRequestError( + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null || request.native_work_admission !== null) throw new EffectRuntimeRequestError( "resume lease qualification requires a live canonical authority read"); return readQuotaSettlementFromRequest(request, snapshot); } @@ -1854,6 +1897,16 @@ async function readSettlementExecutionAuthority(request: ReadbackRequest, result && hint.validation_recovery_turn_instance_id !== identity.turn_instance_id) { reject("Recovery Turn hint differs from the original settlement identity."); } + return await readCanonicalExecutionAuthority(request, identity); +} + +async function readCanonicalExecutionAuthority(request: ReadbackRequest, identity: SettlementIdentity): Promise<{ + identity: SettlementIdentity; todo: JsonObject; todos: JsonObject[]; + lease: JsonObject | undefined; handoffMode: unknown; +}> { + function reject(message: string): never { + throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); + } // The ordinary reader does not load lease decision owners. Only this // explicitly requested ephemeral branch reads the real provider authority, // inside the enclosing quota/source admission and without claiming a lease. @@ -1882,6 +1935,67 @@ async function readSettlementExecutionAuthority(request: ReadbackRequest, result lease, handoffMode: authority.head.handoff_mode}; } +async function readNativeWorkAdmission(request: ReadbackRequest, result: JsonObject, + snapshot: QuotaSettlementReadbackSnapshot): Promise { + requireNativeQuotaWorkAgent(request.owner, request.agent_id!, request.native_work_admission!.check_prior_closeout_only !== true); + const identity = settlementIdentity({goal_id: request.goal_id, agent_id: request.agent_id!, + todo_id: request.todo_id, turn_instance_id: request.turn_instance_id!}); + const events = indexedEvents(snapshot, identity.goal_id, identity.agent_id, identity.turn_instance_id); + if (events.some(event => event.event_kind === "quota_should_run" + && !quotaOwnerOwnsProjection(request.owner, event.goal_ref))) { + throw new EffectRuntimeRequestError("Fresh native admission cannot upgrade another receipt owner or Goal instance.", + "heartbeat_receipt_identity_conflict"); + } + for (const event of snapshot.events) { + if (event.event_kind !== "quota_should_run") continue; + const fact = heartbeatReceiptFactFromEvent(event); + if (fact.todo_id !== identity.todo_id || !fact.closeout_required) continue; + const invalidOwner = event.goal_ref === undefined ? request.owner.kind === "exact_source" + : parseExactGoalRef(event.goal_ref).kind !== "parsed"; + if (invalidOwner || (quotaOwnerOwnsProjection(request.owner, event.goal_ref) + && (optionalString(event.agent_id) === null || fact.run_id === null))) { + throw new EffectRuntimeRequestError("The original Todo's admitted closeout subject or owner is unconfirmed.", + "heartbeat_receipt_identity_conflict"); + } + } + // The same prior-Turn owner fences publication as fresh selection. Restrict + // discovery to this Todo but retain its complete indexed settlement evidence. + const scoped = {...snapshot, events: snapshot.events.filter(event => + quotaOwnerOwnsProjection(request.owner, event.goal_ref) + && heartbeatReceiptFactFromEvent(event).todo_id === identity.todo_id)}; + const agents = new Set(scoped.events.map(event => optionalString(event.agent_id)) + .filter((agent): agent is string => agent !== null)); + for (const agent of agents) { + const debt = owedPriorTurnFromSnapshot({...request, native_work_admission: null}, scoped, agent, + agent === identity.agent_id ? identity.turn_instance_id : null); + if (debt !== null) { + throw new EffectRuntimeRequestError("The original Todo Turn must close its own admitted settlement before fresh native work.", + "refresh_task_lease_unavailable"); + } + } + const {todo, lease, handoffMode} = await readCanonicalExecutionAuthority(request, identity); + if (request.native_work_admission!.check_prior_closeout_only === true) { + return {prior_closeout_checked: true, append: false, details: null, source_event_id: null, + original_prepared: null, current_task_lease_proof: null}; + } + const existing = effectiveHeartbeatReceipt(events, identity); + const runs = indexedRuns(snapshot, identity).filter(run => quotaOwnerOwnsProjection(request.owner, run.goal_ref)); + if ((existing === null && runs.length !== 0) || (existing !== null && (result.found !== true + || jsonObject(jsonObject(result.identity)?.result)?.failure != null))) { + throw new EffectRuntimeRequestError("Fresh native work cannot replace an existing or unconfirmed Turn transaction.", + "refresh_task_lease_unavailable"); + } + if (lease === undefined) throw new EffectRuntimeRequestError("Fresh native work requires its current canonical task lease.", + "refresh_task_lease_unavailable"); + return qualifyHeartbeatNativeWorkAdmission({identity, currentDetails: requireJsonObject( + request.native_work_admission!.details, "native admission details"), existingReceipt: existing, + receiptHistory: events, todo, lease, handoffMode, + replayPhase: existing === null ? "open" : result.replay_phase as ReceiptBoundReplayPhase, + closeoutStarted: existing !== null && [result.writeback_run, result.writeback_event, + result.spend_run, result.spend_event, result.completion_event].some(record => record != null), + observedAt: new Date()}); +} + async function readResumeLeaseQualification(request: ReadbackRequest, result: JsonObject): Promise { function reject(message: string): never { throw new EffectRuntimeRequestError(message, "refresh_task_lease_unavailable"); @@ -2252,7 +2366,7 @@ export async function readQuotaSettlement(value: unknown): Promise { throw new EffectRuntimeRequestError("spend_request requires spend_precheck, integer slots and a source"); } if (spendPrecheck || request.include_readonly_recovery) request.include_record_bodies = true; - if (request.resume_lease_candidate !== null || request.writeback_candidate !== null) request.include_record_bodies = true; + if (request.resume_lease_candidate !== null || request.writeback_candidate !== null || request.native_work_admission !== null) request.include_record_bodies = true; const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner : withQuotaAccountingOwner; @@ -2267,6 +2381,9 @@ export async function readQuotaSettlement(value: unknown): Promise { request.tolerate_malformed_history, ); const result = readQuotaSettlementFromRequest(request, snapshot, true, matchedSpend); + if (request.native_work_admission !== null) { + result.native_work_admission = await readNativeWorkAdmission(request, result, snapshot); + } if (request.resume_lease_candidate !== null) { result.resume_lease_qualification = await readResumeLeaseQualification(request, result); } diff --git a/loopx/control_plane/quota/source_admission.ts b/loopx/control_plane/quota/source_admission.ts index 47526e2a75..b86d200a10 100644 --- a/loopx/control_plane/quota/source_admission.ts +++ b/loopx/control_plane/quota/source_admission.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { dirname, isAbsolute, join, resolve } from "node:path"; +import {readFileSync} from "node:fs"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { @@ -14,6 +15,10 @@ import { type FileMutationLockClaim, } from "../effect_runtime_io.ts"; import { decideFirstPartyHostRuntime } from "../goals/first_party_host_runtime.ts"; +import {legacyGoalRegistrationIsActive} from "../goals/first_party_host_runtime.ts"; +import {normalizeRegisteredTodoAgents} from "../coordination/todo_agents.ts"; +import {BARE_SHA256_PATTERN} from "../content_digest.ts"; +import {jsonObject} from "../runtime_decode.ts"; import { parseExactGoalRef, type ExactGoalRef, @@ -24,9 +29,10 @@ import { } from "../runtime_decode.ts"; const QUOTA_SOURCE_ADMISSION_SCHEMA = "loopx_quota_source_admission_v0"; +export const QUOTA_NATIVE_ADMISSION_SCHEMA = "loopx_quota_native_admission_v0"; const SOURCE_SESSION_PROFILE_ID = "source_session_v1"; -type LockRole = "run_index" | "source_guard"; +type LockRole = "run_index" | "source_guard" | "registry_source" | "authority_maintenance" | "heartbeat_receipt"; interface QuotaSourceLock { role: LockRole; @@ -36,18 +42,30 @@ interface QuotaSourceLock { } interface QuotaSourceAdmission { + kind?: "source_lifetime"; registryPath: string; plannedGoalRef: ExactGoalRef; authority: unknown; locks: readonly [QuotaSourceLock, QuotaSourceLock]; } +interface QuotaNativeAdmission { + kind: "native_runtime"; + registryPath: string; + plannedGoalRef: ExactGoalRef | null; + goalId: string; + runtimeRoot: string; + agentId: string | null; + registrySources: readonly {path: string; sha256: string}[]; + locks: readonly QuotaSourceLock[]; +} + export type QuotaAccountingOwner = - | Readonly<{ kind: "alias" }> + | Readonly<{ kind: "alias"; admission?: QuotaNativeAdmission }> | Readonly<{ kind: "exact_source"; goalRef: ExactGoalRef; - admission: QuotaSourceAdmission; + admission: QuotaSourceAdmission | QuotaNativeAdmission; }>; interface ClaimedLock { @@ -58,7 +76,7 @@ interface ClaimedLock { export function quotaAccountingOwnerLocks( owner: QuotaAccountingOwner, ): readonly Readonly[] { - return owner.kind === "alias" ? [] : owner.admission.locks; + return owner.admission?.locks ?? []; } function sha256(value: string): string { @@ -167,6 +185,130 @@ function quotaSourceLock( }; } +function parseNativeQuotaOwner(admission: JsonObject, goalRef: ExactGoalRef | null, + runtimeRoot: string, goalId: string): QuotaAccountingOwner { + const invalid = (message: string): never => { + throw new EffectRuntimeRequestError(message, "quota_source_admission_invalid"); + }; + if (Object.keys(admission).some(key => !["schema_version", "registry_path", "planned_goal_ref", + "runtime_root", "agent_id", "registry_sources", "locks"].includes(key))) { + invalid("Native quota admission has an unsupported resource field."); + } + const registryPath = requireNonEmptyString(admission.registry_path, "native registry_path"); + const parsed = admission.planned_goal_ref === null ? null : parseExactGoalRef(admission.planned_goal_ref); + if (goalRef === null ? parsed !== null : parsed?.kind !== "parsed" || !sameGoalRef(parsed.value, goalRef)) { + return invalid("Native quota admission must preserve its original alias or exact GoalRef boundary."); + } + const planned = parsed?.kind === "parsed" ? parsed.value : null; + if (!isAbsolute(registryPath) || resolve(registryPath) !== registryPath + || admission.runtime_root !== runtimeRoot || !Array.isArray(admission.registry_sources) + || admission.registry_sources.length < 1 || admission.registry_sources.length > 2) { + invalid("Native quota admission does not match its exact Goal and registered runtime resources."); + } + const registrySources = (admission.registry_sources as unknown[]).map(value => { + const witness = requireJsonObject(value, "native registry source witness"); + const path = requireNonEmptyString(witness.path, "native registry source path"); + const digest = requireNonEmptyString(witness.sha256, "native registry source digest"); + if (Object.keys(witness).length !== 2 || !isAbsolute(path) || resolve(path) !== path + || !BARE_SHA256_PATTERN.test(digest)) invalid("Native registry source witness is malformed."); + return {path, sha256: digest}; + }); + if (registrySources.at(-1)!.path !== registryPath + || new Set(registrySources.map(source => source.path)).size !== registrySources.length + || !Array.isArray(admission.locks) + || ![registrySources.length + 1, registrySources.length + 3].includes(admission.locks.length)) { + invalid("Native quota admission must retain its exact ordered registry and execution resources."); + } + const rawLocks = admission.locks as unknown[]; + const locks = [quotaSourceLock(rawLocks[0], "run_index", + resolve(join(runtimeRoot, "goals", goalId, "runs", "index.jsonl"))), + ...registrySources.map((source, index) => quotaSourceLock(rawLocks[index + 1], + "registry_source", source.path)), + ...(rawLocks.length === registrySources.length + 1 ? [] : [ + quotaSourceLock(rawLocks[registrySources.length + 1], "authority_maintenance", + requireNonEmptyString(requireJsonObject(rawLocks[registrySources.length + 1], "native maintenance witness").target, + "native maintenance target")), + quotaSourceLock(rawLocks[registrySources.length + 2], "heartbeat_receipt", + join(runtimeRoot, "goals", goalId, "rollout-event-log.jsonl"))])]; + const native: QuotaNativeAdmission = {kind: "native_runtime", registryPath, + plannedGoalRef: planned, goalId, runtimeRoot, agentId: admission.agent_id === undefined ? null + : requireNonEmptyString(admission.agent_id, "native agent_id"), + registrySources, locks}; + return goalRef === null ? {kind: "alias", admission: native} : {kind: "exact_source", goalRef, admission: native}; +} + +async function requireNativeQuotaResourceTargets(admission: QuotaNativeAdmission): Promise { + if (new Set(admission.locks.map(lock => lock.pid)).size !== 1) { + throw new EffectRuntimeRequestError("Native quota resources must belong to the same current host invocation.", + "quota_source_admission_invalid"); + } + if (admission.locks.length === admission.registrySources.length + 1) return; + // Keep the large maintenance owner outside ordinary settlement-read closure. + // Exact M validation still precedes adopting any native resource or proof. + const {shadowMaintenanceLockPath} = await import("../coordination/shadow_management.ts"); + if (admission.locks[admission.registrySources.length + 1]!.target + !== shadowMaintenanceLockPath(admission.runtimeRoot, admission.goalId)) { + throw new EffectRuntimeRequestError("Native quota maintenance lock does not protect its canonical Goal.", + "quota_source_admission_invalid"); + } +} + +function requireCurrentNativeQuotaOwner(goalRef: ExactGoalRef | null, admission: QuotaNativeAdmission): void { + function reject(message: string): never { + throw new EffectRuntimeConflictError(message, "goal_instance_conflict"); + } + const registrations = admission.registrySources.map((source, index) => { + const bytes = readFileSync(source.path); + if (createHash("sha256").update(bytes).digest("hex") !== source.sha256) { + reject("Native Goal registration changed during admission."); + } + const document: unknown = JSON.parse(bytes.toString("utf8")); + // The existing project codec decoded these locked bytes in the host. As + // canonical creation does, bind that exact source before inspecting its + // registration; this does not activate Source's generic runtime profile. + const registry = jsonObject(Array.isArray(document) && document.length === 2 ? document[1] : document); + const role = registry?.registry_role ?? registry?.role; + if (registry === null || registry.profile_id === SOURCE_SESSION_PROFILE_ID + || (role !== undefined && !["project-local", "global-local"].includes(String(role)))) { + reject("Native quota admission requires the existing ordinary runtime registry role."); + } + const matches = Array.isArray(registry.goals) ? registry.goals.map(jsonObject) + .filter(goal => goal?.id === admission.goalId) : []; + const goal = matches.length === 1 ? matches[0] : null; + const activation = jsonObject(goal?.activation); + if (goal === null || (goalRef !== null && goal.goal_instance_id !== goalRef.goalInstanceId.value) + || !legacyGoalRegistrationIsActive(goal.status) || Object.hasOwn(goal, "activation_state") + || (activation !== null && (activation.schema_version !== "loopx_goal_activation_v1" || activation.state !== "active"))) { + reject("Native quota admission lost its persisted active Goal instance."); + } + const registered = [jsonObject(goal.coordination)?.registered_agents, goal.registered_agents, + jsonObject(goal.spawn_policy)?.registered_agents].flatMap(value => Array.isArray(value) + ? normalizeRegisteredTodoAgents(value as string[]) : []); + if (admission.agentId !== null && !registered.includes(admission.agentId)) { + reject("Native admission Agent is not currently registered."); + } + const declaredRoot = registry.common_runtime_root; + const projectRoot = basename(dirname(source.path)) === ".loopx" ? dirname(dirname(source.path)) : dirname(source.path); + if (index === 0 && (typeof declaredRoot !== "string" || !declaredRoot + || resolve(projectRoot, declaredRoot) !== admission.runtimeRoot)) { + reject("Native admission does not own the registered canonical runtime root."); + } + return {path: source.path, goal}; + }); + const invoked = registrations.at(-1)!; + const sourceRoute = invoked.goal.source_registry; + if (registrations.length === 1 ? sourceRoute != null && sourceRoute !== "" + : typeof sourceRoute !== "string" || !isAbsolute(sourceRoute) + || resolve(sourceRoute) !== registrations[0]!.path + || registrations[0]!.goal.source_registry != null && registrations[0]!.goal.source_registry !== "") { + reject("Native admission registry sources differ from the existing owning route."); + } + if (goalRef === null ? admission.plannedGoalRef !== null + : admission.plannedGoalRef === null || !sameGoalRef(admission.plannedGoalRef, goalRef)) { + reject("Native GoalRef changed inside admission."); + } +} + export function parseQuotaAccountingOwner( { goalRefValue, @@ -182,6 +324,9 @@ export function parseQuotaAccountingOwner( ): QuotaAccountingOwner { const hasGoalRef = goalRefValue !== undefined; const hasAdmission = sourceAdmissionValue !== undefined; + if (!hasGoalRef && jsonObject(sourceAdmissionValue)?.schema_version === QUOTA_NATIVE_ADMISSION_SCHEMA) { + return parseNativeQuotaOwner(requireJsonObject(sourceAdmissionValue, "native quota admission"), null, runtimeRoot, goalId); + } if (!hasGoalRef && !hasAdmission) return { kind: "alias" }; if (!hasGoalRef || !hasAdmission) { throw new EffectRuntimeRequestError( @@ -204,6 +349,9 @@ export function parseQuotaAccountingOwner( sourceAdmissionValue, "quota source admission", ); + if (admission.schema_version === QUOTA_NATIVE_ADMISSION_SCHEMA) { + return parseNativeQuotaOwner(admission, goalRef.value, runtimeRoot, goalId); + } if ( admission.schema_version !== QUOTA_SOURCE_ADMISSION_SCHEMA || admission.profile_id !== SOURCE_SESSION_PROFILE_ID @@ -257,6 +405,7 @@ export function parseQuotaAccountingOwner( kind: "exact_source", goalRef: goalRef.value, admission: { + kind: "source_lifetime", registryPath, plannedGoalRef: plannedGoalRef.value, authority: admission.authority, @@ -269,6 +418,9 @@ export async function withQuotaAccountingOwner( owner: QuotaAccountingOwner, operation: (indexLockHeld: boolean) => Promise, ): Promise { + if (owner.admission?.kind === "native_runtime") { + return await withBorrowedQuotaAccountingOwner(owner, operation); + } if (owner.kind === "alias") return await operation(false); const claims: ClaimedLock[] = []; @@ -316,9 +468,23 @@ export async function withQuotaAccountingOwner( } } +export function requireNativeQuotaWorkAgent(owner: QuotaAccountingOwner, agentId: string, + publication: boolean): void { + if (owner.admission?.kind !== "native_runtime" || owner.admission.agentId !== agentId + || (publication && owner.admission.locks.length !== owner.admission.registrySources.length + 3)) { + throw new EffectRuntimeRequestError("Fresh native work requires its registered Agent and retained native resource admission.", + "quota_source_admission_invalid"); + } + requireCurrentNativeQuotaOwner(owner.kind === "alias" ? null : owner.goalRef, owner.admission); +} + export function requireCurrentQuotaAccountingOwner( owner: QuotaAccountingOwner, ): void { + if (owner.admission?.kind === "native_runtime") { + requireCurrentNativeQuotaOwner(owner.kind === "alias" ? null : owner.goalRef, owner.admission); + return; + } if (owner.kind === "alias") return; const decision = decideFirstPartyHostRuntime({ profile_id: SOURCE_SESSION_PROFILE_ID, @@ -350,11 +516,13 @@ export async function withBorrowedQuotaAccountingOwner( owner: QuotaAccountingOwner, operation: (indexLockHeld: boolean) => Promise, ): Promise { - if (owner.kind === "alias") return await operation(false); + const admission = owner.admission; + if (admission === undefined) return await operation(false); + if (admission.kind === "native_runtime") await requireNativeQuotaResourceTargets(admission); const claims: FileMutationLockClaim[] = []; try { - for (const witness of owner.admission.locks) { + for (const witness of admission.locks) { const claim = await claimFileMutationLock( witness.target, witness.token, diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 7782e575fc..b452f32f0c 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -1636,7 +1636,9 @@ def run_loopx_turn_once( journal_path = turn_journal_path(runtime_root, goal_id=goal_id, turn_key=turn_key) def persist_journal(snapshot: Mapping[str, Any]) -> None: - if goal_admission is None or not goal_admission.enabled: + if goal_admission is None or ( + not goal_admission.source_profile and goal_admission.planned_goal_ref is None + ): _write_journal(journal_path, snapshot) return try: @@ -1644,7 +1646,7 @@ def persist_journal(snapshot: Mapping[str, Any]) -> None: runtime_root=runtime_root, ) as source_admission: if source_admission is None: - raise RuntimeError("source journal admission was not produced") + raise RuntimeError("exact journal admission was not produced") _write_journal( journal_path, snapshot, diff --git a/loopx/control_plane/turn_driver/turn_journal_effects.ts b/loopx/control_plane/turn_driver/turn_journal_effects.ts index 4d5a5527fc..ab31c25048 100644 --- a/loopx/control_plane/turn_driver/turn_journal_effects.ts +++ b/loopx/control_plane/turn_driver/turn_journal_effects.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; -import { dirname, isAbsolute, join, resolve } from "node:path"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { settlementIdentityFromPlan } from "../effect_program.ts"; @@ -424,6 +424,36 @@ export async function commitTurnJournal( operation_id: incomingOperationId, }; }); + if (asObject(params.source_admission).schema_version === "loopx_quota_native_admission_v0") { + // Native exact registration is a selected transport variant, not Source's + // lifecycle profile or a fresh work grant. Keep its owner out of the + // dormant/default journal handler closure. + const {parseQuotaAccountingOwner, + withBorrowedQuotaAccountingOwner, requireNativeQuotaWorkAgent} = await import("../quota/source_admission.ts"); + if (goalBinding.kind !== "exact") { + throw new EffectRuntimeRequestError("Native exact journal admission cannot upgrade a legacy journal binding.", + "journal_source_admission_invalid"); + } + const admission = asObject(params.source_admission); + const runtimeRoot = requiredString(admission.runtime_root, "native journal runtime_root"); + const identity = settlementIdentityFromPlan(asObject(asObject(journal.plan).transaction)); + // Match journal_store.turn_journal_path: a validated Turn key becomes + // its bare digest filename, including on Windows. + const digest = typeof journal.turn_key === "string" ? /^sha256:([0-9a-f]{64})$/.exec(journal.turn_key)?.[1] : undefined; + const journalName = `${digest}.json`; + if (digest === undefined || identity.failure !== null || !isAbsolute(runtimeRoot) || resolve(runtimeRoot) !== runtimeRoot + || basename(path) !== journalName + || path !== join(runtimeRoot, "goals", identity.value.goal_id, "turns", journalName)) { + throw new EffectRuntimeRequestError("Native journal path does not match its verified Goal and Turn identity.", + "journal_source_admission_invalid"); + } + const owner = parseQuotaAccountingOwner({goalRefValue: goalBinding.goal_ref, + sourceAdmissionValue: admission, runtimeRoot, goalId: identity.value.goal_id}); + return await withBorrowedQuotaAccountingOwner(owner, async () => { + requireNativeQuotaWorkAgent(owner, identity.value.agent_id, false); + return await commit(); + }); + } if (goalBinding.kind === "legacy") { if (params.source_admission !== undefined) { throw new EffectRuntimeRequestError( diff --git a/loopx/control_plane/work_items/recommendation_source_io.py b/loopx/control_plane/work_items/recommendation_source_io.py index 339a1b2432..74f5f8fed4 100644 --- a/loopx/control_plane/work_items/recommendation_source_io.py +++ b/loopx/control_plane/work_items/recommendation_source_io.py @@ -137,6 +137,10 @@ def project_agent_next_actions(goal: dict[str, Any], todo_fields: dict[str, Any] def recommendation_source_guard( registry_path: Path, state_path: Path, goal_id: str, *, source_registry: Path, + runtime_root: Path | None = None, + goal_ref: Mapping[str, Any] | None = None, + source_admission: Mapping[str, Any] | None = None, + agent_id: str | None = None, ) -> Iterator[tuple[dict[str, Any] | None, str]]: # Registration/configuration use the same registry lock. Quota's outer # index/source guard precedes this short registry -> state critical section. @@ -144,9 +148,25 @@ def recommendation_source_guard( # Configuration locks source before its shared projection. Keep that order, # and revalidate the projection's route under both locks before using it. with ExitStack() as stack: - stack.enter_context(exclusive_cross_runtime_file_lock(source_registry, operation="recommendation-writeback")) - if registry_path.resolve() != source_registry.resolve(): - stack.enter_context(exclusive_cross_runtime_file_lock(registry_path, operation="recommendation-writeback")) + if source_admission is not None and source_admission.get("schema_version") == "loopx_quota_native_admission_v0": + from ..quota.accounting_admission import _require_held_native_registry_resources + + if runtime_root is None: + raise ValueError("Next Action native resources require their explicit runtime root") + # Reuse the enclosing accounting resource lifetime, without + # granting Next Action write authority or deriving its GoalRef. + try: + _require_held_native_registry_resources( + runtime_root=runtime_root, registry_path=registry_path, goal_id=goal_id, + goal_ref=goal_ref, source_admission=source_admission, + source_registry=source_registry, agent_id=agent_id, + ) + except RuntimeError as exc: + raise ValueError("Next Action native resources no longer match their enclosing scope") from exc + else: + stack.enter_context(exclusive_cross_runtime_file_lock(source_registry, operation="recommendation-writeback")) + if registry_path.resolve() != source_registry.resolve(): + stack.enter_context(exclusive_cross_runtime_file_lock(registry_path, operation="recommendation-writeback")) current_source, goal = load_recommendation_source_goal(registry_path, goal_id) if current_source != source_registry.resolve(): raise ValueError("Next Action source registry changed; read current status and rejudge before retrying") diff --git a/loopx/control_plane/work_items/replan_history_codec.py b/loopx/control_plane/work_items/replan_history_codec.py index 0b41d5ba2b..bc43d3a6f8 100644 --- a/loopx/control_plane/work_items/replan_history_codec.py +++ b/loopx/control_plane/work_items/replan_history_codec.py @@ -228,7 +228,7 @@ def project_replan_history( def _project_admitted_replan_request(params: dict[str, Any], context: dict[str, Any]) -> Any: # One live admission and one snapshot cover all Turns. Borrow an enclosing # refresh transaction instead of releasing its source/index locks early. - from ..quota.accounting_admission import quota_accounting_admission + from ..quota.accounting_admission import _borrow_native_read_admission, quota_accounting_admission source = dict(params["settlement_source"]) goal_ref = context.get("goal_ref") @@ -244,6 +244,15 @@ def invoke(admission: dict[str, Any] | None, *, borrow: bool) -> Any: if context.get("source_admission") is not None: return invoke(context["source_admission"], borrow=True) + native_read = _borrow_native_read_admission( + runtime_root=Path(source["runtime_root"]), registry_path=context.get("registry_path"), + goal_id=source["goal_id"], goal_ref=goal_ref, agent_id=params.get("agent_id")) + if native_read is not None: + # This is the paired resource identity of one read-only TS projection, + # never a replacement for a receipt, journal or business effect. + if goal_ref is None: + goal_ref = native_read["planned_goal_ref"] + return invoke(native_read, borrow=True) with quota_accounting_admission( runtime_root=Path(source["runtime_root"]), registry_path=context.get("registry_path"), diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index e6f4d84d95..03d42e8b69 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -542,6 +542,11 @@ def render_state_refresh_markdown(payload: dict[str, Any]) -> str: f"- state_updated_at: `{frontmatter.get('updated_at')}`", f"- health_check: `{payload.get('health_check')}`", ] + lines.extend( + f"- {field}: `{payload[field]}`" + for field in ("todo_id", "turn_instance_id", "replan_obligation_id") + if payload.get(field) + ) lines.extend(delivery_lines) lines.extend(render_settlement_progress_markdown(payload)) if "external_sink_delivery_authorized" in payload: @@ -1523,9 +1528,13 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): # two deltas from one stale basis; the appended row advances the basis. with ExitStack() as usage_booking_guard: if normalized_next_action: + native_resources = ({"runtime_root": runtime_root, "goal_ref": goal_ref, + "source_admission": source_admission, "agent_id": normalized_agent_id or None} + if source_admission is not None + and source_admission.get("schema_version") == "loopx_quota_native_admission_v0" else {}) current_goal, current_text = usage_booking_guard.enter_context( recommendation_source_guard(registry_path, resolved_state_file, safe_goal_id, - source_registry=next_action_source_registry)) + source_registry=next_action_source_registry, **native_resources)) if current_goal is None: raise ValueError("Next Action source Goal disappeared; reread status") current_planning = load_refresh_planning_source( diff --git a/tests/control_plane/test_quota_execution_preparation.py b/tests/control_plane/test_quota_execution_preparation.py index c31b13d319..44265a2c7c 100644 --- a/tests/control_plane/test_quota_execution_preparation.py +++ b/tests/control_plane/test_quota_execution_preparation.py @@ -1536,8 +1536,12 @@ def test_writeback_requires_current_same_generation_cas( def test_validation_recovery_cannot_execute_replaced_epoch( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, provider: str, ) -> None: + from loopx.control_plane.quota.accounting_admission import quota_accounting_admission + from loopx.control_plane.quota.execution_preparation import ( + read_turn_execution_lease_qualification, + ) from tests.control_plane.test_execution_loss_closeout_cli import ( - OWNER, TURN, _fixture as execution_loss_fixture, + OWNER, TURN, _fixture as execution_loss_fixture, _readback, ) context = execution_loss_fixture( @@ -1548,17 +1552,30 @@ def test_validation_recovery_cannot_execute_replaced_epoch( current = context["current_lease"] assert current["lease_epoch"] > original["lease_epoch"] assert current["idempotency_key"] != original["idempotency_key"] + settlement = _readback(context) + assert settlement is not None and settlement.identity.value is not None writeback = { **context, "turn": TURN, "agent_id": OWNER, "proof": {"idempotency_key": original["idempotency_key"], "expected_version": original["version"]}, } before = _writeback_side_effect_snapshot(writeback) - with pytest.raises(RefreshTaskLeaseUnavailable): - _run_real_semantic_writeback( - writeback, proof=writeback["proof"], - validation_recovery_turn_instance_id=TURN, - ) + with quota_accounting_admission( + runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal_id"], goal_ref=context["goal_ref"], + operation="validation-recovery-replaced-epoch", + ) as source_admission: + assert source_admission is not None + assert source_admission["schema_version"] == "loopx_quota_source_admission_v0" + assert source_admission["profile_id"] == "source_session_v1" + assert source_admission["planned_goal_ref"] == context["goal_ref"] + with pytest.raises(RefreshTaskLeaseUnavailable): + read_turn_execution_lease_qualification( + runtime_root=context["runtime"], registry_path=context["registry"], + identity=settlement.identity.value, requested_proof=writeback["proof"], + goal_ref=context["goal_ref"], source_admission=source_admission, + validation_recovery_turn_instance_id=TURN, + ) assert _writeback_side_effect_snapshot(writeback) == before @@ -1607,23 +1624,26 @@ def test_legacy_prepared_receipt_is_exact_version_only( @pytest.mark.parametrize("provider", ["file", "sqlite"]) -def test_exact_source_supersede_admits_only_its_original_turn_writeback( +def test_supersede_admits_only_its_original_turn_writeback( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, provider: str, ) -> None: + from loopx.state_refresh import refresh_state_run from tests.control_plane.test_execution_loss_closeout_cli import ( OWNER, TURN, _binding, _fixture as execution_loss_fixture, _proof, _readback, ) context = execution_loss_fixture( tmp_path, monkeypatch, capsys, provider, - source_profile=True, replace=False, + source_profile=False, replace=False, ) expected_todo = "todo_" + hashlib.sha256(b"execution-loss-task").hexdigest()[:24] expected_effect = ( - f"execution-loss-closeout:original-worker:{expected_todo}:original-prepared-turn" + f"{context['goal_id']}:original-worker:{expected_todo}:original-prepared-turn" ) - admission_identity = _readback(context).identity.value.as_dict() - assert admission_identity["goal_id"] == "execution-loss-closeout" + original_settlement = _readback(context) + assert original_settlement is not None and original_settlement.identity.value is not None + admission_identity = original_settlement.identity.value.as_dict() + assert admission_identity["goal_id"] == context["goal_id"] assert admission_identity["agent_id"] == "original-worker" assert admission_identity["todo_id"] == expected_todo assert admission_identity["turn_instance_id"] == "original-prepared-turn" @@ -1642,19 +1662,79 @@ def test_exact_source_supersede_admits_only_its_original_turn_writeback( } assert retired["settlement_identity"] == admission_identity before_wrong_turn = _writeback_side_effect_snapshot(writeback) - with pytest.raises(RuntimeError, match="exact settlement readback unexpectedly returned not-found"): + with pytest.raises(ValueError, match="matching Turn guard is missing"): _run_real_semantic_writeback( {**writeback, "turn": "foreign-prepared-turn"}, proof=writeback["proof"], ) assert _writeback_side_effect_snapshot(writeback) == before_wrong_turn - result = _run_real_semantic_writeback(writeback, proof=writeback["proof"]) + original_prepared = original_settlement.original_prepared_receipt + original_event_id = original_settlement.original_prepared_event_id + original_receipt_sha256 = original_settlement.original_prepared_receipt_sha256 + expected_prepared_lease = { + "prepared_lease_key_sha256": hashlib.sha256( + lease["idempotency_key"].encode("utf-8") + ).hexdigest(), + "prepared_lease_version": lease["version"], + "prepared_lease_epoch": lease["lease_epoch"], + } + assert original_settlement.prepared_lease == expected_prepared_lease + result = refresh_state_run( + registry_path=context["registry"], + runtime_root_override=str(context["runtime"]), + goal_id=context["goal_id"], project=context["project"], state_file=None, + classification="receipt_bound_supersede_writeback", + recommended_action="Review the remaining Goal after recording the retired task Turn.", + delivery_batch_scale="single_surface", delivery_outcome="outcome_progress", + delivery_boundary="semantic_closeout", todo_id=context["todo"], + turn_instance_id=TURN, agent_id=OWNER, + progress_observation={ + "schema_version": "typed_progress_observation_v0", + "result_class": "advanced", "work_item_id": context["todo"], + "evidence_ids": ["evidence:original-task-superseded"], + }, + agent_vision_packet={ + "schema_version": "goal_vision_replan_contract_v0", "state": "active", + "vision_patch": { + "vision_summary": "Retain the Goal after retiring the original bounded task.", + "acceptance_summary": "The retired task records only its admitted original Turn.", + "advancement_policy": "as_needed", + }, + "path_delta": { + "schema_version": "goal_path_delta_v0", "outcome": "continue", + "prior_assumption": "The original task remained the active execution route.", + "observed_reality": "The task was superseded after its original Turn was admitted.", + "retained": ["The original Turn identity and prepared lease proof."], + "changed": ["Retire the original task and continue from the remaining Goal."], + "evidence_refs": ["evidence:original-task-superseded"], + }, + }, + autonomous_replan_recorded=True, + task_lease_proof=writeback["proof"], dry_run=False, sync_global=False, + ) assert result["ok"] is True run = json.loads(Path(result["json_path"]).read_text(encoding="utf-8")) assert run["settlement_identity"] == admission_identity assert run["todo_id"] == expected_todo - index = context["runtime"] / "goals" / "execution-loss-closeout" / "runs" / "index.jsonl" + assert run["vision_checkpoint"]["decision"] == "patched" + assert run["vision_checkpoint"]["satisfied"] is True + assert run["agent_vision"]["state"] == "active" + assert run["agent_vision"]["path_delta"]["changed"] == [ + "Retire the original task and continue from the remaining Goal." + ] + markdown = Path(result["markdown_path"]) + assert markdown.exists() and TURN in markdown.read_text(encoding="utf-8") + index = context["runtime"] / "goals" / context["goal_id"] / "runs" / "index.jsonl" [index_row] = [json.loads(line) for line in index.read_text(encoding="utf-8").splitlines()] assert index_row["settlement_identity"] == admission_identity + assert Path(index_row["json_path"]) == Path(result["json_path"]) + assert Path(index_row["markdown_path"]) == markdown + final = _readback(context) + assert final is not None + assert final.original_prepared_receipt == original_prepared + assert final.original_prepared_event_id == original_event_id + assert final.original_prepared_receipt_sha256 == original_receipt_sha256 + assert final.prepared_lease == expected_prepared_lease + assert final.writeback_run == index_row @pytest.mark.parametrize("provider", ["file", "sqlite"]) @@ -1793,7 +1873,11 @@ def probe_marker() -> subprocess.CompletedProcess[str]: "turn-same-generation-writeback#durable_writeback" ), "delivery_boundary": "semantic_closeout", - "original_prepared": original_receipt, + "original_prepared": { + "prepared_lease_key_sha256": hashlib.sha256(b"same-generation-writeback").hexdigest(), + "prepared_lease_version": 1, + "prepared_lease_epoch": 1, + }, "selected_obligation_id": None, }] released = probe_marker() @@ -1823,14 +1907,41 @@ def probe_marker() -> subprocess.CompletedProcess[str]: runs = list((context["runtime"] / "goals" / GOAL / "runs").glob("*.json")) assert len(runs) == 1 run = json.loads(runs[0].read_text(encoding="utf-8")) - assert final.writeback_run == run - assert run["settlement_identity"]["turn_instance_id"] == context["turn"] markdown = runs[0].with_suffix(".md") + state_bytes = (context["project"] / "ACTIVE_GOAL_STATE.md").read_bytes() + state_digest = hashlib.sha256(state_bytes).hexdigest()[:16] + assert run["state"]["sha256_16"] == state_digest + assert run["settlement_identity"]["turn_instance_id"] == context["turn"] assert markdown.exists() and context["turn"] in markdown.read_text(encoding="utf-8") index = context["runtime"] / "goals" / GOAL / "runs" / "index.jsonl" index_rows = [json.loads(line) for line in index.read_text(encoding="utf-8").splitlines()] assert len(index_rows) == 1 - assert index_rows[0]["settlement_identity"] == run["settlement_identity"] + # Settlement readback owns the exact canonical index row; the physical + # JSON also retains adapter presentation that is outside that projection. + assert final.writeback_run == index_rows[0] + assert index_rows[0]["state"] == { + "sha256_16": state_digest, "frontmatter": {"updated_at": None}, + } + expected_identity = { + "schema_version": "quota_settlement_identity_v0", + "effect_id": ( + "goal-execution-preparation:agent-a:todo_execution_preparation:" + "turn-same-generation-writeback" + ), + "goal_id": GOAL, "agent_id": AGENT, "todo_id": TODO, + "turn_instance_id": "turn-same-generation-writeback", + } + for record in (run, index_rows[0]): + assert record["settlement_identity"] == expected_identity + assert record["goal_id"] == GOAL and record["agent_id"] == AGENT + assert record["todo_id"] == TODO and record["turn_instance_id"] == context["turn"] + assert record["delivery_outcome"] == "outcome_gap" + progress = record["progress_observation"] + assert progress["schema_version"] == "typed_progress_observation_v0" + assert progress["result_class"] == "blocked" and progress["work_item_id"] == TODO + assert progress["blocker_id"] == "synthetic-writeback-fence" + assert progress["evidence_ids"] == ["evidence:synthetic-writeback-fence"] + assert index_rows[0]["progress_observation"] == run["progress_observation"] assert Path(index_rows[0]["json_path"]) == runs[0] assert Path(index_rows[0]["markdown_path"]) == markdown diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index 94d102be27..6ea91c53cc 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -166,6 +166,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/quota/heartbeat_receipt_identity.ts", "control_plane/quota/refresh_external_delivery.ts", "control_plane/quota/semantic_replan_guard.ts", + "control_plane/quota/source_admission.ts", "control_plane/quota/turn_envelope.ts", "control_plane/runtime/checkpoint_artifacts.ts", "control_plane/runtime/checkpoint_retention.ts", diff --git a/tests/control_plane_ts/heartbeat_resume_lease.test.ts b/tests/control_plane_ts/heartbeat_resume_lease.test.ts index 67b5d3ffc7..e3d3955376 100644 --- a/tests/control_plane_ts/heartbeat_resume_lease.test.ts +++ b/tests/control_plane_ts/heartbeat_resume_lease.test.ts @@ -13,6 +13,7 @@ import { HEARTBEAT_PREPARED_PUBLICATION_REQUEST_SCHEMA, qualifyHeartbeatPreparedPublication, qualifyHeartbeatResumeLease, + qualifyHeartbeatNativeWorkAdmission, heartbeatPreparedWorkAdmission, heartbeatValidationEvent, } from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; @@ -60,6 +61,68 @@ function resume(overrides: Partial[0]> = {}) { + return qualifyHeartbeatNativeWorkAdmission({identity, currentDetails: {...preparedReceipt.details as JsonObject, + turn_instance_id: identity.turn_instance_id, prepared_lease_version: currentLease.version}, + existingReceipt: null, receiptHistory: [], todo: {...todo(identity.todo_id!), claimed_by: identity.agent_id}, + lease: currentLease, handoffMode: "hard_lease", replayPhase: "open", closeoutStarted: false, + observedAt: new Date("2026-01-01T00:01:00Z"), ...overrides}); +} + +test("fresh native admission has its own primary proof and preserves the original proof on replay", () => { + const before = structuredClone(preparedReceipt); + const initial = nativeAdmission(); + assert.equal(initial.state, "ready"); + assert.equal(initial.append, true); + assert.equal(initial.source_event_id, null); + assert.equal(Object.hasOwn(initial.details as JsonObject, "settlement_receipt_revision"), false); + assert.deepEqual(initial.original_prepared, {...originalPrepared, prepared_lease_version: 120}); + const replay = nativeAdmission({existingReceipt: preparedReceipt, receiptHistory: [preparedReceipt]}); + assert.deepEqual(replay, {state: "ready", append: false, details: null, source_event_id: null, + original_prepared: originalPrepared, current_task_lease_proof: {idempotency_key: originalKey, expected_version: 120}}); + assert.deepEqual(preparedReceipt, before); +}); + +test("fresh native admission rejects stale CAS, foreign preparation, closed work and partial or legacy precursors", () => { + const realDetails = nativeAdmission().details as JsonObject; + for (const patch of [{prepared_lease_version: 119}, {prepared_lease_epoch: 51}, + {prepared_lease_epoch: undefined}, {prepared_lease_key_sha256: "b".repeat(64)}, + {selected_todo_id: "todo_other"}, {turn_instance_id: "other-turn"}, + {settlement_effect_id: "other-effect"}, {closeout_required: false}, + {prepared_lease_required: false}, {delivery_allowed: false}, {quiet_noop_allowed: true}, + {must_attempt_work: undefined}, {settlement_receipt_revision: "work_admission"}]) { + assert.throws(() => nativeAdmission({currentDetails: {...realDetails, ...patch}}), + {code: "refresh_task_lease_unavailable"}); + } + for (const patch of [{status: "released"}, {expires_at: "2025-01-01T00:00:00Z"}, + {owner: "other-agent"}, {version: 121}]) { + assert.throws(() => nativeAdmission({lease: {...currentLease, ...patch}}), {code: "refresh_task_lease_unavailable"}); + } + assert.throws(() => nativeAdmission({todo: {...todo(identity.todo_id!, "done"), claimed_by: identity.agent_id}}), + {code: "refresh_task_lease_unavailable"}); + assert.throws(() => nativeAdmission({closeoutStarted: true}), {code: "refresh_task_lease_unavailable"}); + for (const previous of [{todo_id: identity.todo_id, settlement_effect_id: identity.effect_id, + prepared_lease_required: true, ...originalPrepared}, + {...realDetails, closeout_required: false, must_attempt_work: undefined}]) { + const existing: JsonObject = {...preparedReceipt, status: "turn_run_once", details: previous}; + assert.throws(() => nativeAdmission({existingReceipt: existing, receiptHistory: [existing]}), + {code: "refresh_task_lease_unavailable"}); + } +}); + +test("fresh native admission reuses causal qualification only for a genuine complete negative precursor", () => { + const existing: JsonObject = {...preparedReceipt, event_id: "preparation-wait", status: "repair_required", + details: {...preparedReceipt.details as JsonObject, closeout_required: false, delivery_allowed: false, + prepared_lease_key_sha256: undefined, prepared_lease_version: undefined, prepared_lease_epoch: undefined}}; + const qualified = nativeAdmission({existingReceipt: existing, receiptHistory: [existing]}); + assert.equal(qualified.append, true); + assert.equal(qualified.source_event_id, existing.event_id); + assert.equal((qualified.details as JsonObject).settlement_receipt_revision, "work_admission"); + const carrier: JsonObject = {...preparedReceipt, event_id: "fresh-prepared", details: qualified.details, + causality: {source_event_id: existing.event_id, caused_by: existing.event_id}}; + assert.deepEqual(heartbeatValidationEvent([existing, carrier], carrier, identity, false), carrier); +}); + test("resume distinguishes current CAS from frozen prepared generation", () => { const before = structuredClone({preparedReceipt, currentLease}); const renewed = resume(); diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index c63359d7b8..f1da635b1e 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -38,6 +38,8 @@ import { preflightPriorHostTurnCloseout, } from "../../loopx/control_plane/quota/unsettled_host_turn_recovery.ts"; import {heartbeatSemanticAdmissionEvent} from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; +import {QUOTA_NATIVE_ADMISSION_SCHEMA} from "../../loopx/control_plane/quota/source_admission.ts"; +import {shadowMaintenanceLockPath} from "../../loopx/control_plane/coordination/shadow_management.ts"; import {buildVisionCheckpoint, VISION_REFRESH_REQUEST_SCHEMA, visionAuthoringContract} from "../../loopx/control_plane/goals/vision_checkpoint.ts"; import {decodeRefreshRetry, refreshRecovery} from "../../loopx/control_plane/quota/refresh_recovery.ts"; @@ -2950,6 +2952,233 @@ async function todoSemanticFixture(provider: "file" | "sqlite", seedTodo?: JsonO return {root, log, store, admission}; } +async function withNativeAdmission(root: string, run: (binding: JsonObject, registry: string) => Promise, + agent: string | null = agentId, options: {exact?: boolean; publication?: boolean; + registryPath?: string; declaredRuntimeRoot?: string; registryRole?: string} = {}): Promise { + const registry = options.registryPath ?? join(root, "native-project", ".loopx", "registry.json"); + await mkdir(dirname(registry), {recursive: true}); + await writeFile(registry, JSON.stringify({schema_version: "0.1", registry_role: options.registryRole ?? "project-local", + common_runtime_root: options.declaredRuntimeRoot ?? root, goals: [{id: goalId, ...(options.exact === false ? {} : {goal_instance_id: instanceA}), status: "active", + coordination: {registered_agents: [agentId]}}]})); + const targets = [join(root, "goals", goalId, "runs", "index.jsonl"), registry, + ...(options.publication === false ? [] : [shadowMaintenanceLockPath(root, goalId), + join(root, "goals", goalId, "rollout-event-log.jsonl")])]; + const roles = ["run_index", "registry_source", "authority_maintenance", "heartbeat_receipt"]; + const locks: Awaited>[] = []; + for (const target of targets) locks.push(await acquireFileMutationLock(target)); + try { + const ref = options.exact === false ? null : {goal_id: goalId, goal_instance_id: instanceA}; + return await run({...(ref === null ? {} : {goal_ref: ref}), borrow_source_admission: true, source_admission: { + schema_version: QUOTA_NATIVE_ADMISSION_SCHEMA, registry_path: registry, planned_goal_ref: ref, + runtime_root: root, ...(agent === null ? {} : {agent_id: agent}), + registry_sources: [{path: registry, sha256: createHash("sha256").update(await readFile(registry)).digest("hex")}], + locks: targets.map((target, index) => ({role: roles[index], target, pid: process.pid, token: locks[index]!.token})), + }}, registry); + } finally { + for (const index of [...targets.keys()].reverse()) { + await releaseFileMutationLock(targets[index]!, locks[index]!.token, null, true); + } + } +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: native relative runtime registration follows project, plain and global registry locations`, async () => { + const f = await todoSemanticFixture(provider); + await writeFile(f.log, ""); + try { + // Native join exercises the host's real path flavor, including Win32 in + // Windows CI. All locations declare the parent of native-project as the + // runtime; a project .loopx registry must first leave its .loopx folder. + const project = join(f.root, "native-project"); + const locations = [ + {path: join(project, ".loopx", "registry.json"), role: "project-local"}, + {path: join(project, "registry.json"), role: "project-local"}, + {path: join(project, "registry.global.json"), role: "global-local"}, + ]; + const logBefore = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + for (const location of locations) { + await withNativeAdmission(f.root, async binding => { + const result = await readQuotaSettlement(request(f.root, {...binding, + native_work_admission: {check_prior_closeout_only: true}})); + assert.equal((result.native_work_admission as JsonObject).prior_closeout_checked, true, location.path); + assert.equal((result.native_work_admission as JsonObject).current_task_lease_proof, null); + assert.deepEqual(await readFile(f.log), logBefore); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + }, agentId, {exact: false, publication: false, registryPath: location.path, + declaredRuntimeRoot: "..", registryRole: location.role}); + } + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} + +function nativeWorkDetails(work = identity, version = 120): JsonObject { + return {todo_id: work.todo_id, selected_todo_id: work.todo_id, turn_instance_id: work.turn_instance_id, + settlement_effect_id: work.effect_id, ok: true, should_run: true, must_attempt_work: true, + delivery_allowed: true, quiet_noop_allowed: false, closeout_required: true, prepared_lease_required: true, + prepared_lease_key_sha256: createHash("sha256").update("native-execution").digest("hex"), + prepared_lease_version: version, prepared_lease_epoch: 50}; +} + +for (const provider of ["file", "sqlite"] as const) { + test(`${provider}: native primary positive admission uses current CAS and proves its own closeout debt`, async () => { + const lease: JsonObject = {schema_version: "task_lease_v0", goal_id: goalId, todo_id: todoId, + owner: agentId, status: "active", idempotency_key: "native-execution", version: 120, lease_epoch: 50, + acquired_at: "2026-01-01T00:00:00Z", updated_at: "2026-01-01T00:00:01Z", expires_at: "2099-01-01T00:00:00Z"}; + const f = await todoSemanticFixture(provider, undefined, [lease]); + await writeFile(f.log, ""); + try { + await withNativeAdmission(f.root, async binding => { + const readRequest = {...request(f.root), ...binding}; + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const checked = await readQuotaSettlement({...readRequest, + native_work_admission: {check_prior_closeout_only: true}}); + assert.deepEqual(checked.native_work_admission, {prior_closeout_checked: true, append: false, details: null, + source_event_id: null, original_prepared: null, current_task_lease_proof: null}); + const first = await readQuotaSettlement({...readRequest, native_work_admission: {details: nativeWorkDetails()}}); + const admitted = requireJsonObject(first.native_work_admission, "native admission"); + assert.equal(admitted.append, true); + assert.equal(admitted.source_event_id, null); + assert.equal(Object.hasOwn(admitted.details as JsonObject, "settlement_receipt_revision"), false); + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + const event: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", + event_id: "native-primary", goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", + recorded_at: "2026-01-01T00:00:00Z", goal_ref: binding.goal_ref, details: admitted.details}; + await appendFile(f.log, JSON.stringify(event) + "\n"); + const replay = await readQuotaSettlement({...readRequest, native_work_admission: {details: nativeWorkDetails()}}); + assert.equal((replay.native_work_admission as JsonObject).append, false); + assert.deepEqual(replay.original_prepared_receipt, event); + const cleanup = await readQuotaSettlement({...readRequest, native_task_lease_cleanup: cleanupRequest}); + assert.deepEqual(cleanup.native_task_lease_cleanup, {schema_version: NATIVE_TASK_LEASE_CLEANUP_SCHEMA, + disposition: "retain", reason: "original_turn_owes_closeout"}); + const saved = settlementIdentity({goal_id: goalId, agent_id: agentId, todo_id: todoId, + turn_instance_id: turnId + ".saved-candidate"}); + for (const candidate of [{check_prior_closeout_only: true}, {details: nativeWorkDetails(saved)}]) { + await assert.rejects(readQuotaSettlement({...readRequest, turn_instance_id: saved.turn_instance_id, + native_work_admission: candidate}), {code: "refresh_task_lease_unavailable"}); + } + const after = await readFile(f.log); + for (const changed of [nativeWorkDetails(identity, 119), {...nativeWorkDetails(), delivery_allowed: false}]) { + await assert.rejects(readQuotaSettlement({...readRequest, native_work_admission: {details: changed}}), + {code: "refresh_task_lease_unavailable"}); + } + assert.deepEqual(await readFile(f.log), after); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + }); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + test(`${provider}: native admission preserves exact registry role, Agent, store root and retained resource fences`, async () => { + const f = await todoSemanticFixture(provider); + await writeFile(f.log, ""); + try { + await withNativeAdmission(f.root, async (binding, registry) => { + const native = requireJsonObject(binding.source_admission, "native owner"); + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + const originalRegistry = await readFile(registry); + for (const patch of [{goal_instance_id: instanceB}, {status: "complete"}, + {coordination: {registered_agents: ["other-agent"]}}]) { + const document = JSON.parse(originalRegistry.toString("utf8")); + document.goals[0] = {...document.goals[0], ...patch}; + const bytes = JSON.stringify(document); + await writeFile(registry, bytes); + const changed = {...native, registry_sources: [{path: registry, + sha256: createHash("sha256").update(bytes).digest("hex")}]}; + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, source_admission: changed, + native_work_admission: {check_prior_closeout_only: true}})), {code: "goal_instance_conflict"}); + } + for (const patch of [{registry_role: "public-safe-projection"}, {profile_id: "source_session_v1"}, + {common_runtime_root: join(f.root, "different-store")}, {common_runtime_root: undefined}]) { + const bytes = JSON.stringify({...JSON.parse(originalRegistry.toString("utf8")), ...patch}); + await writeFile(registry, bytes); + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, source_admission: {...native, + registry_sources: [{path: registry, sha256: createHash("sha256").update(bytes).digest("hex")}]}, + native_work_admission: {check_prior_closeout_only: true}})), {code: "goal_instance_conflict"}); + } + await writeFile(registry, originalRegistry); + for (const patch of [{locks: (native.locks as JsonObject[]).slice(0, -1)}, + {locks: (native.locks as JsonObject[]).map((lock, index) => index === 2 ? {...lock, role: "source_guard"} : lock)}, + {planned_goal_ref: {goal_id: goalId, goal_instance_id: instanceB}}, {runtime_root: join(f.root, "other-root")}]) { + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, source_admission: {...native, ...patch}, + native_work_admission: {check_prior_closeout_only: true}})), {code: "quota_source_admission_invalid"}); + } + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, agent_id: "other-agent", + native_work_admission: {check_prior_closeout_only: true}})), {code: "quota_source_admission_invalid"}); + for (const candidate of [{check_prior_closeout_only: false}, {check_prior_closeout_only: true, details: {}}, + {details: {}, resume_lease_candidate: {requested_proof: null}}]) { + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, native_work_admission: candidate}))); + } + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + }); + await withNativeAdmission(f.root, async binding => { + const pure = await readQuotaSettlement(request(f.root, binding)); + assert.equal(Object.hasOwn(pure, "native_work_admission"), false); + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, + native_work_admission: {check_prior_closeout_only: true}})), {code: "quota_source_admission_invalid"}); + }, null); + await assert.rejects(readQuotaSettlement(request(f.root, {native_work_admission: {check_prior_closeout_only: true}})), + {code: "quota_source_admission_invalid"}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + test(`${provider}: alias native primary keeps absent GoalRef and general reads need no publication locks`, async () => { + const lease: JsonObject = {schema_version: "task_lease_v0", goal_id: goalId, todo_id: todoId, + owner: agentId, status: "active", idempotency_key: "native-execution", version: 120, lease_epoch: 50, + acquired_at: "2026-01-01T00:00:00Z", updated_at: "2026-01-01T00:00:01Z", expires_at: "2099-01-01T00:00:00Z"}; + const f = await todoSemanticFixture(provider, undefined, [lease]); + await writeFile(f.log, ""); + try { + await withNativeAdmission(f.root, async binding => { + const admission = await readQuotaSettlement(request(f.root, {...binding, + native_work_admission: {details: nativeWorkDetails()}})); + assert.equal((admission.native_work_admission as JsonObject).append, true); + assert.equal(Object.hasOwn(binding, "goal_ref"), false); + assert.equal(Object.hasOwn(admission, "goal_ref"), false); + assert.equal(Object.hasOwn((admission.native_work_admission as JsonObject).details as JsonObject, "goal_ref"), false); + }, agentId, {exact: false}); + await withNativeAdmission(f.root, async binding => { + const pure = await readQuotaSettlement(request(f.root, binding)); + assert.equal(Object.hasOwn(pure, "native_work_admission"), false); + const checked = await readQuotaSettlement(request(f.root, {...binding, + native_work_admission: {check_prior_closeout_only: true}})); + assert.equal((checked.native_work_admission as JsonObject).current_task_lease_proof, null); + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, + native_work_admission: {details: nativeWorkDetails()}})), {code: "quota_source_admission_invalid"}); + }, agentId, {publication: false}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); + test(`${provider}: prior native debt blocks its Todo while independent work keeps its own admission`, async () => { + const {todo} = await import("./shadow_file_fixture.ts"); + const otherId = "todo_independent"; + const f = await todoSemanticFixture(provider, undefined, [], [{...todo(otherId), claimed_by: agentId}]); + const original: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", + event_id: "original-debt", goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", + recorded_at: "2026-01-01T00:00:00Z", goal_ref: {goal_id: goalId, goal_instance_id: instanceA}, details: nativeWorkDetails()}; + await writeFile(f.log, JSON.stringify(original) + "\n"); + try { + await withNativeAdmission(f.root, async binding => { + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, turn_instance_id: turnId + ".saved-candidate", + native_work_admission: {check_prior_closeout_only: true}})), {code: "refresh_task_lease_unavailable"}); + const independent = await readQuotaSettlement(request(f.root, {...binding, todo_id: otherId, + turn_instance_id: "independent-new-turn", native_work_admission: {check_prior_closeout_only: true}})); + assert.equal((independent.native_work_admission as JsonObject).prior_closeout_checked, true); + assert.equal((independent.native_work_admission as JsonObject).current_task_lease_proof, null); + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + for (const patch of [{agent_id: null}, {run_id: null}, {goal_ref: undefined}, + {goal_ref: {goal_id: goalId, goal_instance_id: "malformed-instance"}}]) { + const malformed = JSON.stringify({...original, ...patch}) + "\n"; + await writeFile(f.log, malformed); + await assert.rejects(readQuotaSettlement(request(f.root, {...binding, + turn_instance_id: turnId + ".saved-candidate", + native_work_admission: {check_prior_closeout_only: true}})), {code: "heartbeat_receipt_identity_conflict"}); + assert.equal(await readFile(f.log, "utf8"), malformed); + } + }, agentId, {publication: false}); + } finally {await rm(f.root, {recursive: true, force: true});} + }); +} + for (const provider of ["file", "sqlite"] as const) { test(`${provider}: Todo-bound semantic writeback without business lease remains operation scoped`, async () => { const f = await todoSemanticFixture(provider); diff --git a/tests/control_plane_ts/turn_journal_effects.test.ts b/tests/control_plane_ts/turn_journal_effects.test.ts index 9830852805..06fd619b8a 100644 --- a/tests/control_plane_ts/turn_journal_effects.test.ts +++ b/tests/control_plane_ts/turn_journal_effects.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; @@ -8,6 +8,7 @@ import test from "node:test"; import { EffectRuntimeConflictError } from "../../loopx/control_plane/effect_runtime_errors.ts"; import { acquireFileMutationLock, + mutationLockOwner, releaseFileMutationLock, } from "../../loopx/control_plane/effect_runtime_io.ts"; import { commitTurnJournal } from "../../loopx/control_plane/turn_driver/turn_journal_effects.ts"; @@ -71,8 +72,9 @@ function sourceJournal( goalInstanceId: string, key: string, completedPhases: readonly string[] = [], + agentId = "fixture-agent", ): Record { - const snapshot = journal("in_progress", completedPhases, "fixture-agent", key); + const snapshot = journal("in_progress", completedPhases, agentId, key); const plan = snapshot.plan as Record; const goalRef = { goal_id: "fixture-goal", @@ -461,6 +463,191 @@ test("legacy journals reject source admission without changing their wire", asyn }); }); +interface NativeJournalFixture { + runtimeRoot: string; + path: string; + registries: string[]; + admission: Record; + locks: Record[]; +} + +async function withNativeJournal( + run: (fixture: NativeJournalFixture) => Promise, + sourceRoute = false, +): Promise { + const directory = await mkdtemp(join(tmpdir(), "loopx-ts-native-journal-")); + const runtimeRoot = join(directory, "runtime"); + const invoked = join(directory, "project", ".loopx", "registry.json"); + const registries = sourceRoute ? [join(directory, "source", ".loopx", "registry.json"), invoked] : [invoked]; + const targets = [join(runtimeRoot, "goals", "fixture-goal", "runs", "index.jsonl"), ...registries]; + const locks: Record[] = []; + try { + for (const [index, path] of registries.entries()) { + await mkdir(dirname(path), {recursive: true}); + await writeFile(path, JSON.stringify({schema_version: "0.1", registry_role: "project-local", + common_runtime_root: runtimeRoot, goals: [{id: "fixture-goal", goal_instance_id: instanceA, + status: "active", coordination: {registered_agents: ["fixture-agent", "other-registered-agent"]}, + ...(sourceRoute && index === 1 ? {source_registry: registries[0]} : {})}]})); + } + for (const [index, target] of targets.entries()) { + const lock = await acquireFileMutationLock(target); + locks.push({role: index === 0 ? "run_index" : "registry_source", target, pid: process.pid, token: lock.token}); + } + await run({runtimeRoot, path: join(runtimeRoot, "goals", "fixture-goal", "turns", `${"a".repeat(64)}.json`), + registries, locks, admission: {schema_version: "loopx_quota_native_admission_v0", registry_path: invoked, + runtime_root: runtimeRoot, planned_goal_ref: {goal_id: "fixture-goal", goal_instance_id: instanceA}, + agent_id: "fixture-agent", registry_sources: await Promise.all(registries.map(async path => ({path, + sha256: createHash("sha256").update(await readFile(path)).digest("hex")}))), locks}}); + } finally { + for (const lock of [...locks].reverse()) { + await releaseFileMutationLock(String(lock.target), String(lock.token), null, true); + } + await rm(directory, {recursive: true, force: true}); + } +} + +function nativeJournalRequest(fixture: NativeJournalFixture, snapshot = sourceJournal(instanceA, turnKey)) { + return {path: fixture.path, journal: snapshot, expected_effect_id: effectId(), source_admission: fixture.admission}; +} + +for (const sourceRoute of [false, true]) { + test(`native exact journal ${sourceRoute ? "source and invoked registries" : "project registry"} preserves numbers and borrowed locks`, async () => { + await withNativeJournal(async fixture => { + // Only metadata resources are held: no Todo lease, prepared receipt, + // maintenance fence or heartbeat publication grants this journal write. + const snapshot: Record = {...sourceJournal(instanceA, turnKey), values: [1, -0, 2 ** 60]}; + (snapshot.plan as Record).measurement = 1; + const request = {...nativeJournalRequest(fixture, snapshot), source_number_tokens: [ + [["values", 0], "1.0"], [["values", 1], "-0.0"], [["values", 2], "1152921504606846977"], + [["plan", "measurement"], "1.0"], + ]}; + const written = await commitTurnJournal(request); + const encoded = await readFile(fixture.path, "utf8"); + assert.equal(written.appended, true); + assert.match(encoded, /1\.0,\n\s+-0\.0,\n\s+1152921504606846977/); + assert.match(encoded, /"measurement": 1\.0/); + assert.ok(!encoded.includes("source_number_tokens")); + const replay = await commitTurnJournal(request); + assert.equal(replay.replayed, true); + assert.equal(replay.operation_id, written.operation_id); + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture, snapshot)), /plan is immutable/); + assert.equal(await readFile(fixture.path, "utf8"), encoded); + for (const witness of fixture.locks) { + const owner = await mutationLockOwner(String(witness.target)); + assert.equal(owner?.pid, witness.pid); + assert.equal(owner?.token, witness.token); + } + }, sourceRoute); + }); +} + +test("native exact journals retain plan, checkpoint and terminal guards", async () => { + await withNativeJournal(async fixture => { + const initial = nativeJournalRequest(fixture); + await commitTurnJournal(initial); + const encoded = await readFile(fixture.path, "utf8"); + const changedPlan = sourceJournal(instanceA, turnKey); + (changedPlan.plan as Record).host = {kind: "changed-after-start"}; + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture, changedPlan)), /transaction plan is immutable/); + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture, + {...sourceJournal(instanceA, turnKey, phases), status: "committed"})), /skip transaction checkpoints/); + assert.equal(await readFile(fixture.path, "utf8"), encoded); + for (const count of [2, 3, 4, 5]) { + await commitTurnJournal(nativeJournalRequest(fixture, sourceJournal(instanceA, turnKey, phases.slice(0, count)))); + if (count === 3) { + const checkpoint = await readFile(fixture.path, "utf8"); + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture, + sourceJournal(instanceA, turnKey, phases.slice(0, 2)))), /completed phases cannot regress or fork/); + assert.equal(await readFile(fixture.path, "utf8"), checkpoint); + } + } + const committed = {...sourceJournal(instanceA, turnKey, phases), status: "committed"}; + await commitTurnJournal(nativeJournalRequest(fixture, committed)); + const terminal = await readFile(fixture.path, "utf8"); + assert.equal((await commitTurnJournal(nativeJournalRequest(fixture, committed))).replayed, true); + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture, {...committed, reason: "late mutation"})), + /tombstones are immutable/); + assert.equal(await readFile(fixture.path, "utf8"), terminal); + }); +}); + +const nativeJournalRejections: { + name: string; + request: (fixture: NativeJournalFixture) => Record; + code: string; +}[] = [ + {name: "foreign planned GoalRef", request: f => ({source_admission: {...f.admission, + planned_goal_ref: {goal_id: "fixture-goal", goal_instance_id: instanceB}}}), code: "quota_source_admission_invalid"}, + {name: "foreign journal GoalRef", request: () => ({journal: sourceJournal(instanceB, turnKey)}), code: "quota_source_admission_invalid"}, + {name: "foreign registered admission Agent", request: f => ({source_admission: {...f.admission, + agent_id: "other-registered-agent"}}), code: "quota_source_admission_invalid"}, + {name: "foreign registered journal Agent", request: () => ({expected_effect_id: effectId("other-registered-agent"), + journal: sourceJournal(instanceA, turnKey, [], "other-registered-agent")}), code: "quota_source_admission_invalid"}, + {name: "missing admission Agent", request: f => ({source_admission: {...f.admission, agent_id: undefined}}), code: "quota_source_admission_invalid"}, + {name: "unregistered admission Agent", request: f => ({source_admission: {...f.admission, agent_id: "unregistered-agent"}}), code: "goal_instance_conflict"}, + {name: "foreign runtime", request: f => ({source_admission: {...f.admission, runtime_root: join(f.runtimeRoot, "foreign")}}), code: "journal_source_admission_invalid"}, + {name: "relative runtime", request: f => ({source_admission: {...f.admission, runtime_root: "runtime"}}), code: "journal_source_admission_invalid"}, + {name: "foreign physical Goal path", request: f => ({path: join(f.runtimeRoot, "goals", "other-goal", "turns", `${"a".repeat(64)}.json`)}), code: "journal_source_admission_invalid"}, + {name: "foreign Turn path", request: f => ({path: join(dirname(f.path), `${"b".repeat(64)}.json`)}), code: "journal_source_admission_invalid"}, + {name: "unstripped Turn filename", request: f => ({path: join(dirname(f.path), `${turnKey}.json`)}), code: "journal_source_admission_invalid"}, + {name: "malformed Turn key", request: () => ({expected_effect_id: effectId("fixture-agent", "ordinary-turn"), + journal: sourceJournal(instanceA, "ordinary-turn")}), code: "journal_source_admission_invalid"}, + {name: "alias journal admission upgrade", request: () => ({journal: journal()}), code: "journal_source_admission_invalid"}, + {name: "foreign run-index lock", request: f => ({source_admission: {...f.admission, + locks: f.locks.map((lock, index) => index === 0 ? {...lock, target: join(f.runtimeRoot, "other-index.jsonl")} : lock)}}), code: "quota_source_admission_invalid"}, + {name: "foreign registry lock token", request: f => ({source_admission: {...f.admission, + locks: f.locks.map((lock, index) => index === 1 ? {...lock, token: "foreign-lock-token"} : lock)}}), code: "quota_source_admission_expired"}, + {name: "foreign lock host", request: f => ({source_admission: {...f.admission, + locks: f.locks.map(lock => ({...lock, pid: process.pid + 1}))}}), code: "quota_source_admission_expired"}, + {name: "missing registry lock", request: f => ({source_admission: {...f.admission, locks: f.locks.slice(0, 1)}}), code: "quota_source_admission_invalid"}, + {name: "changed registry byte witness", request: f => ({source_admission: {...f.admission, + registry_sources: [{path: f.registries[0], sha256: "0".repeat(64)}]}}), code: "goal_instance_conflict"}, +]; + +for (const rejection of nativeJournalRejections) { + test(`native exact journal rejects ${rejection.name} before creation and replacement`, async () => { + for (const existing of [false, true]) { + await withNativeJournal(async fixture => { + if (existing) await commitTurnJournal(nativeJournalRequest(fixture)); + const before = existing ? await readFile(fixture.path, "utf8") : null; + const request = {...nativeJournalRequest(fixture), ...rejection.request(fixture)}; + await assert.rejects(commitTurnJournal(request), {code: rejection.code}); + if (before === null) await assert.rejects(readFile(fixture.path), {code: "ENOENT"}); + else assert.equal(await readFile(fixture.path, "utf8"), before); + if (request.path !== fixture.path && rejection.name !== "unstripped Turn filename") { + await assert.rejects(readFile(String(request.path)), {code: "ENOENT"}); + } + }); + } + }); +} + +test("native exact journal rejects expired resources and changed persisted identity without replacing bytes", async () => { + for (const changed of ["run_index", "registry_source", "registry_bytes", "GoalRef", "Source profile"]) { + await withNativeJournal(async fixture => { + await commitTurnJournal(nativeJournalRequest(fixture)); + const encoded = await readFile(fixture.path, "utf8"); + if (changed === "run_index" || changed === "registry_source") { + const lock = fixture.locks.find(witness => witness.role === changed)!; + await releaseFileMutationLock(String(lock.target), String(lock.token)); + } else { + const registry = fixture.registries[0]!; + const document = JSON.parse(await readFile(registry, "utf8")); + if (changed === "GoalRef") document.goals[0].goal_instance_id = instanceB; + else if (changed === "Source profile") document.profile_id = "source_session_v1"; + else document.description = "changed bytes"; + await writeFile(registry, JSON.stringify(document)); + if (changed !== "registry_bytes") fixture.admission.registry_sources = [{path: registry, + sha256: createHash("sha256").update(await readFile(registry)).digest("hex")}]; + } + await assert.rejects(commitTurnJournal(nativeJournalRequest(fixture)), { + code: changed === "run_index" || changed === "registry_source" ? "quota_source_admission_expired" : "goal_instance_conflict", + }); + assert.equal(await readFile(fixture.path, "utf8"), encoded); + }); + } +}); + test("TS journal owner accepts the complete monotonic transaction", async () => { await withJournalPath(async (path) => { await commit(path, journal()); diff --git a/tests/test_codex_turn_completion.py b/tests/test_codex_turn_completion.py index 3cc73cd989..4f2574eeee 100644 --- a/tests/test_codex_turn_completion.py +++ b/tests/test_codex_turn_completion.py @@ -6,6 +6,8 @@ import io import json import sys +import traceback +from copy import deepcopy from hashlib import sha256 from pathlib import Path from types import SimpleNamespace @@ -344,6 +346,7 @@ def _effect_kinds(runtime: Path, journal: dict) -> list[str]: @pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("exact_ref", [False, True]) @pytest.mark.parametrize( "interruption", [None, "after_debit", "lost_completion_receipt"] ) @@ -351,6 +354,7 @@ def test_native_terminal_closeout_keeps_the_original_lease_and_turn( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, + exact_ref: bool, interruption: str | None, ) -> None: isolate_sqlite_runtime(tmp_path, monkeypatch) @@ -365,6 +369,12 @@ def test_native_terminal_closeout_keeps_the_original_lease_and_turn( "codex-fixture", "todo_fixture0001", ) + goal_ref = None + if exact_ref: + goal_ref = {"goal_id": goal_id, "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + registration = json.loads(registry.read_text()) + registration["goals"][0]["goal_instance_id"] = goal_ref["goal_instance_id"] + registry.write_text(json.dumps(registration) + "\n") code, claim = _cli( registry, runtime, @@ -389,7 +399,7 @@ def test_native_terminal_closeout_keeps_the_original_lease_and_turn( registry_path=registry, coordination_runtime_root=runtime ) ) - host_calls, lease_reads = [], [] + host_calls, lease_reads, host_exceptions = [], [], [] def resolve_lease(): lease_reads.append(lease["version"]) @@ -398,11 +408,22 @@ def resolve_lease(): ) def host(request, launch_lifetime): - with launch_lifetime(): - host_calls.append(request["turn_key"]) - (project / "completion-artifact.txt").write_text( - "completed", encoding="utf-8" - ) + try: + with launch_lifetime(): + host_calls.append(request["turn_key"]) + (project / "completion-artifact.txt").write_text( + "completed", encoding="utf-8" + ) + except Exception as exc: + host_exceptions.append({ + "type": type(exc).__name__, + "message": str(exc), + "code": getattr(exc, "diagnostic_code", None) or getattr(exc, "code", None), + "payload": getattr(exc, "payload", None), + "frames": [{"file": frame.filename, "line": frame.lineno, "function": frame.name} + for frame in traceback.extract_tb(exc.__traceback__)], + }) + raise return { "schema_version": "loopx_turn_result_v0", "turn_key": request["turn_key"], @@ -436,7 +457,7 @@ def host(request, launch_lifetime): "--execute", ] - def invoke(*extra: str): + def invoke(*extra: str, current_ref=None): args = build_parser().parse_args([*base, *extra]) args.host, args.session_scope = "codex-app-server", "agent" result: dict = {} @@ -453,6 +474,7 @@ def invoke(*extra: str): "goal_id": goal_id, "agent_id": agent_id, }, + native_goal_ref=current_ref if current_ref is not None else goal_ref, ) return code, result @@ -474,6 +496,17 @@ def lose_terminal_attempt(**kwargs): ) code, result = invoke("--todo-id", todo_id) journal = _turn_journal(runtime) + assert journal["plan"].get("goal_ref") == goal_ref, result + event_path = runtime / "goals" / goal_id / "rollout-event-log.jsonl" + run_id = journal["plan"]["transaction"]["turn_instance_id"] + + def original_receipt_lines(): + return [line for line in event_path.read_bytes().splitlines(keepends=True) + if (row := json.loads(line)).get("event_kind") == "quota_should_run" + and row.get("run_id") == run_id and row.get("agent_id") == agent_id] + + receipt_bytes = original_receipt_lines() + assert receipt_bytes if interruption is not None: assert code == 1 and journal["status"] == "in_progress", result assert journal["completed_phases"][-1] == "quota_spend" @@ -483,12 +516,17 @@ def lose_terminal_attempt(**kwargs): ) monkeypatch.setattr(turn_run_once, "write_turn_validated_completion", complete) code, result = invoke( - "--resume-turn-key", journal["turn_key"], "--retry-failed-turn" + "--resume-turn-key", journal["turn_key"], "--retry-failed-turn", + current_ref={"goal_id": goal_id, "goal_instance_id": "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} + if exact_ref else None, ) - assert code == 0 and result["status"] == "committed", result.get( - "error" - ) or result.get("reason") - assert _turn_journal(runtime)["terminal_closeout"]["completion"] == { + assert code == 0 and result["status"] == "committed", json.dumps( + {"execution": result, "host_exceptions": host_exceptions}, + ensure_ascii=False, indent=2, default=str) + final_journal = _turn_journal(runtime) + assert final_journal["plan"].get("goal_ref") == goal_ref + assert original_receipt_lines() == receipt_bytes + assert final_journal["terminal_closeout"]["completion"] == { "todo_id": todo_id, "continuation": "no_followup", } @@ -510,6 +548,8 @@ def lose_terminal_attempt(**kwargs): assert code == 0 and replay["replayed"] is True, replay assert not any(replay["effects"].values()) assert host_calls == [journal["turn_key"]] + assert _turn_journal(runtime)["plan"].get("goal_ref") == goal_ref + assert original_receipt_lines() == receipt_bytes assert _effect_kinds(runtime, journal) == [ "refresh_state", "quota_spend", @@ -848,3 +888,757 @@ def test_builtin_codex_advertises_explicit_completion_without_promoting_progress ): kinds = codex_cli_result_schema()["properties"]["result_kind"]["enum"] assert "validated_completion" in kinds and "validated_progress" in kinds + + +def _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, *, exact_ref=False, mirror=False): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider, handoff_mode="hard_lease") + goal, agent, todo = "loopx-turn-fixture", "codex-fixture", "todo_fixture0001" + registration = json.loads(registry.read_text()) + registration["registry_role"] = "project-local" + goal_ref = None + if exact_ref: + goal_ref = {"goal_id": goal, "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + registration["goals"][0]["goal_instance_id"] = goal_ref["goal_instance_id"] + registry.write_text(json.dumps(registration) + "\n") + source_registry = registry + if mirror: + registry = registry.with_name("mirror-registry.json") + registration["registry_role"] = "global-local" + registration["goals"][0]["source_registry"] = str(source_registry.resolve()) + registry.write_text(json.dumps(registration) + "\n") + claim_args = ("todo", "claim", "--goal-id", goal, "--todo-id", todo, + "--agent-id", agent, "--claimed-by", agent) + code, claimed = _cli(registry, runtime, *claim_args) + assert code == 0, claimed + code, inspected = _cli(registry, runtime, "task-lease", "inspect", "--goal-id", goal, "--todo-id", todo) + assert code == 0, inspected + lease = inspected["lease"] + log = runtime / "goals" / goal / "rollout-event-log.jsonl" + host_receipts = [] + + def host(request, launch_lifetime): + with launch_lifetime(): + receipts = [json.loads(line) for line in log.read_bytes().splitlines() + if json.loads(line).get("event_kind") == "quota_should_run"] + host_receipts.append(receipts[-1]) + raise OSError("fixture stopped after observing native admission") + + def invoke(turn, *, override_ref=None, resume_turn_key=None): + args = build_parser().parse_args([ + "turn", "run-once", "--host", "generic-cli", "--goal-id", goal, + "--agent-id", agent, "--project", str(project), + *(["--resume-turn-key", resume_turn_key, "--retry-failed-turn"] if resume_turn_key + else ["--todo-id", todo, "--turn-instance-id", turn]), + "--no-global-sync", "--execute", + ]) + args.host, args.session_scope = "codex-app-server", "agent" + result = {} + code = handle_turn_command(args, registry_path=registry, runtime_root_arg=str(runtime), + output_format=lambda *_a, **_kw: "json", print_payload=lambda value, *_a: result.update(value), + native_host_runner=host, native_session_binding_resolver=lambda _: { + "schema_version": "loopx_turn_session_binding_v0", "goal_id": goal, "agent_id": agent, + }, native_goal_ref=override_ref if override_ref is not None else goal_ref) + return code, result + + return {"project": project, "runtime": runtime, "registry": registry, "goal": goal, "agent": agent, + "todo": todo, "lease": lease, "log": log, "host_receipts": host_receipts, + "claim_args": claim_args, "invoke": invoke, "goal_ref": goal_ref, + "source_registry": source_registry} + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_native_replan_metadata_read_borrows_real_scope_without_promoting_aliases( + tmp_path, monkeypatch, provider, +): + from loopx.control_plane.quota.accounting_admission import ( + _borrow_native_read_admission, quota_accounting_admission, + ) + from loopx.control_plane.work_items import replan_history_codec as codec + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=True, mirror=True) + read_context = codec.effective_turn_cadence_context({"id": "loopx-turn-fixture", + "execution_profile": {"replan_after_effective_turns": 3}}, context["runtime"]) + assert read_context["goal_ref"] is None and read_context["registry_path"] is None + rows = [] + # These are read-only history facts. They are never written as Runs or receipts. + for label, ref, numbers in [ + ("foreign", {"goal_id": "loopx-turn-fixture", + "goal_instance_id": "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, [6, 5]), + ("alias", None, [4, 3]), + ("current", {"goal_id": "loopx-turn-fixture", + "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, [2, 1]), + ]: + for number in numbers: + rows.append({"agent_id": "codex-fixture", "classification": "bounded_delivery", + "generated_at": f"2026-09-22T00:00:{number:02d}Z", "turn_instance_id": f"history-{label}-{number}", + "progress_observation": {"schema_version": "typed_progress_observation_v0", + "result_class": "unchanged", "surface_id": f"{label}-fixture-surface", + "evidence_ids": [f"evidence-{label}"]}, + **({"goal_ref": ref} if ref is not None else {})}) + actual_effect = codec.effect_runtime_result + requests = [] + + def observe_projection(method, params, **kwargs): + result = actual_effect(method, params, **kwargs) + if method == "work_item.replan_history.project": + requests.append(json.loads(json.dumps(params))) + return result + + monkeypatch.setattr(codec, "effect_runtime_result", observe_projection) + receipt_bytes, registry_bytes = context["log"].read_bytes(), context["registry"].read_bytes() + + def project(cadence=read_context): + return codec.project_replan_history(rows, operation="progress", agent_id="codex-fixture", + effective_turn_cadence=cadence) + + unheld = project() + assert unheld["kind"] == "typed_progress_repeat" and unheld["run_count"] == 2 + assert unheld["progress_baseline"]["surface_id"] == "alias-fixture-surface" + assert "goal_ref" not in requests[-1]["settlement_source"] + assert "source_admission" not in requests[-1]["settlement_source"] + with quota_accounting_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=None, native_agent_id=context["agent"], + operation="fixture-alias-metadata-read"): + assert _borrow_native_read_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=None, agent_id=context["agent"]) is None + # Alias reads keep their original explicit registry context and use + # the existing accounting fallback rather than the new exact helper. + assert project({**read_context, "registry_path": context["registry"]}) == unheld + assert "goal_ref" not in requests[-1]["settlement_source"] + assert requests[-1]["settlement_source"]["source_admission"]["planned_goal_ref"] is None + assert requests[-1]["settlement_source"]["borrow_source_admission"] is False + with quota_accounting_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=context["goal_ref"], native_agent_id=context["agent"], + native_publication=True, operation="fixture-exact-metadata-read") as held: + original_held = json.loads(json.dumps(held)) + current = project() + assert current["kind"] == "typed_progress_repeat" and current["run_count"] == 2 + assert current["progress_baseline"]["surface_id"] == "current-fixture-surface" + source = requests[-1]["settlement_source"] + admission = source["source_admission"] + assert source["borrow_source_admission"] is True + assert source["goal_ref"] == admission["planned_goal_ref"] == { + "goal_id": "loopx-turn-fixture", "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + assert admission["agent_id"] == "codex-fixture" and "profile_id" not in admission + assert admission["runtime_root"] == str(context["runtime"].resolve()) + assert admission["registry_sources"] == [ + {"path": str(path.resolve()), "sha256": sha256(path.read_bytes()).hexdigest()} + for path in [context["source_registry"], context["registry"]]] + assert [lock["role"] for lock in admission["locks"]] == [ + "run_index", "registry_source", "registry_source"] + assert held == original_held + assert any(lock["role"] == "authority_maintenance" for lock in held["locks"]) + assert read_context["goal_ref"] is None and read_context["source_admission"] is None + assert context["log"].read_bytes() == receipt_bytes + assert context["registry"].read_bytes() == registry_bytes + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_native_replan_read_scope_rejects_foreign_context_and_generic_writer(tmp_path, monkeypatch, provider): + from loopx.control_plane.quota.accounting_admission import ( + _borrow_native_read_admission, quota_accounting_admission, + ) + from loopx.control_plane.work_items import replan_history_codec as codec + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=True) + read_context = codec.effective_turn_cadence_context({"id": "loopx-turn-fixture", + "execution_profile": {"replan_after_effective_turns": 3}}, context["runtime"], + registry_path=context["registry"], goal_ref={"goal_id": "loopx-turn-fixture", + "goal_instance_id": "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}) + before = context["log"].read_bytes() + with quota_accounting_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=context["goal_ref"], native_agent_id=context["agent"], + operation="fixture-exact-read-scope"): + with pytest.raises(RuntimeError, match="does not match its held resource scope"): + codec.project_replan_history(operation="progress", agent_id=context["agent"], + effective_turn_cadence=read_context) + common = {"runtime_root": context["runtime"], "registry_path": context["registry"], + "goal_id": context["goal"], "goal_ref": context["goal_ref"], "agent_id": context["agent"]} + for changed in [{"runtime_root": tmp_path / "foreign-runtime"}, + {"registry_path": tmp_path / "foreign-registry.json"}, + {"goal_id": "foreign-goal"}, {"agent_id": "foreign-agent"}]: + with pytest.raises((RuntimeError, ValueError)): + _borrow_native_read_admission(**{**common, **changed}) + # The new read seam must not change ordinary writer borrowing, even + # when that writer does not acquire an additional legacy index lock. + with pytest.raises(RuntimeError, match="does not match its held resource scope"): + with quota_accounting_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=None, operation="fixture-native-writer", + lock_legacy_index=False): + pytest.fail("a generic writer cannot replace the held exact scope with an alias") + assert context["log"].read_bytes() == before + assert context["host_receipts"] == [] + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("fault", ["missing_scope", "altered_witness"]) +def test_native_writeback_guard_rejects_resource_loss_after_real_qualification( + tmp_path, monkeypatch, provider, fault, +): + from loopx.control_plane.quota import accounting_admission, execution_preparation + from loopx.file_lock import cross_runtime_lock_witness + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=True) + turn = "native-writeback-resource-loss" + code, failed = context["invoke"](turn) + assert code == 1 and len(context["host_receipts"]) == 1, failed + receipt_bytes, registry_bytes = context["log"].read_bytes(), context["registry"].read_bytes() + original_lease = deepcopy(context["lease"]) + proof = {"idempotency_key": original_lease["idempotency_key"], "expected_version": original_lease["version"]} + actual_qualifier = execution_preparation.read_turn_execution_lease_qualification + ready_qualifications, writer_calls = [], [] + missing_context_token = None + + def lose_resources_after_qualification(**kwargs): + nonlocal missing_context_token + qualified = actual_qualifier(**kwargs) + assert qualified.resume_lease_qualification["state"] == "ready" + ready_qualifications.append(qualified) + if fault == "missing_scope": + missing_context_token = accounting_admission._held_native_admission.set(None) + else: + # Alter only the offered copy, after TS has read the genuine scope. + kwargs["source_admission"]["registry_sources"][0]["sha256"] = "0" * 64 + kwargs["source_admission"]["locks"][0]["token"] = "post-qualification-foreign-index-token" + return qualified + + with accounting_admission.quota_accounting_admission( + runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=context["goal_ref"], native_agent_id=context["agent"], + operation="fixture-post-qualification-resource-loss") as held: + original_held = deepcopy(held) + offered = deepcopy(held) + settlement = turn_run_once.read_heartbeat_settlement( + context["runtime"], goal_id=context["goal"], agent_id=context["agent"], todo_id=context["todo"], + turn_instance_id=turn, registry_path=context["registry"], goal_ref=context["goal_ref"], + source_admission=held, borrow_source_admission=True) + assert settlement is not None and settlement.prepared_lease is not None + assert settlement.identity.value is not None + assert settlement.prepared_lease == { + "prepared_lease_key_sha256": sha256(original_lease["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": original_lease["version"], + "prepared_lease_epoch": original_lease["lease_epoch"]} + assert settlement.heartbeat_receipt["goal_ref"] == context["goal_ref"] + monkeypatch.setattr(execution_preparation, "read_turn_execution_lease_qualification", + lose_resources_after_qualification) + try: + with pytest.raises(execution_preparation.RefreshTaskLeaseUnavailable, + match="resources are no longer held") as rejected: + with execution_preparation.guard_first_writeback_execution( + runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], todo_id=context["todo"], agent_id=context["agent"], + requested_proof=proof, settlement=settlement, locked_goal=None, + goal_ref=context["goal_ref"], source_admission=offered): + writer_calls.append("entered") + finally: + if missing_context_token is not None: + accounting_admission._held_native_admission.reset(missing_context_token) + assert len(ready_qualifications) == 1 and writer_calls == [] + assert rejected.value.code == "refresh_task_lease_unavailable" + assert "lost its enclosing resource scope" in str(rejected.value.__cause__) + assert ready_qualifications[0].prepared_lease == settlement.prepared_lease + assert held == original_held + assert accounting_admission._held_native_admission.get()[1] == original_held + for witness in held["locks"]: + live = cross_runtime_lock_witness(Path(witness["target"])) + assert {key: live[key] for key in ("target", "pid", "token")} == { + key: witness[key] for key in ("target", "pid", "token")} + assert context["log"].read_bytes() == receipt_bytes + assert context["registry"].read_bytes() == registry_bytes + code, inspected = _cli(context["registry"], context["runtime"], "task-lease", "inspect", + "--goal-id", context["goal"], "--todo-id", context["todo"]) + assert code == 0 and inspected["lease"] == original_lease, inspected + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("mirror", [False, True]) +def test_native_next_action_uses_original_exact_resources(tmp_path, monkeypatch, provider, mirror): + from loopx.state_refresh import refresh_state_run + from loopx.control_plane.quota.accounting_admission import quota_accounting_admission + from loopx.control_plane.quota.execution_preparation import read_turn_execution_lease_qualification + from loopx.control_plane.work_items.refresh_recommendation import load_refresh_planning_source + from loopx.control_plane.work_items.recommendation_source_io import ( + load_recommendation_source_goal, project_agent_next_actions, + recommendation_runs, recommendation_source_context, + ) + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=True, mirror=mirror) + turn = "native-next-action-original-turn" + code, failed = context["invoke"](turn) + assert code == 1 and len(context["host_receipts"]) == 1, failed + original_journal = _turn_journal(context["runtime"]) + source_registry, goal = load_recommendation_source_goal(context["registry"], context["goal"]) + assert source_registry == context["source_registry"].resolve() and goal is not None + state = Path(goal["repo"]) / goal["state_file"] + state_bytes, registry_bytes = state.read_bytes(), context["registry"].read_bytes() + lease = deepcopy(context["lease"]) + proof = {"idempotency_key": lease["idempotency_key"], "expected_version": lease["version"]} + + def routes(): + planning = load_refresh_planning_source(context["runtime"], context["goal"], state, + require_display=True, registry_path=source_registry) + source = recommendation_source_context(goal, planning.state_text, + source_registry=source_registry, todo_fields=planning.todo_fields) + return project_agent_next_actions(goal, planning.todo_fields, source=source, + runs=recommendation_runs(context["runtime"], context["goal"])) + + action = next(row for row in routes() if row["agent_id"] == "codex-fixture") + assert action["todo_id"] == "todo_fixture0001" + basis = action["next_action_basis"] + before_receipt = [line for line in context["log"].read_bytes().splitlines(keepends=True) + if json.loads(line).get("event_kind") == "quota_should_run"] + step = "Inspect the recorded Host interruption before retrying the original work." + with quota_accounting_admission(runtime_root=context["runtime"], registry_path=context["registry"], + goal_id=context["goal"], goal_ref=context["goal_ref"], native_agent_id=context["agent"], + operation="fixture-native-next-action") as held: + settlement = turn_run_once.read_heartbeat_settlement(context["runtime"], goal_id=context["goal"], + agent_id=context["agent"], todo_id=context["todo"], turn_instance_id=turn, + registry_path=context["registry"], goal_ref=context["goal_ref"], + source_admission=held, borrow_source_admission=True) + assert settlement is not None and settlement.identity.value is not None + qualified = read_turn_execution_lease_qualification(runtime_root=context["runtime"], + registry_path=context["registry"], identity=settlement.identity.value, + requested_proof=proof, goal_ref=context["goal_ref"], source_admission=held) + assert qualified.resume_lease_qualification["state"] == "ready" + written = refresh_state_run(registry_path=context["registry"], runtime_root_override=str(context["runtime"]), + goal_id=context["goal"], project=None, state_file=None, classification="fixture_host_interruption", + recommended_action=None, agent_id=context["agent"], todo_id=context["todo"], turn_instance_id=turn, + next_action=step, next_action_basis=basis, delivery_batch_scale="single_surface", + delivery_outcome="outcome_gap", delivery_boundary="semantic_closeout", + progress_observation={"schema_version": "typed_progress_observation_v0", "result_class": "blocked", + "work_item_id": context["todo"], "blocker_id": "fixture-host-interruption", + "evidence_ids": ["turn:" + original_journal["turn_key"].removeprefix("sha256:")]}, + vision_unchanged_reason="The original work remains open; only its interrupted Turn is closing.", + task_lease_proof=proof, goal_ref=context["goal_ref"], dry_run=False, sync_global=False) + assert written["ok"] is True and written["appended"] is True, written + assert written["recommended_action"] == step + assert written["recommended_action_resolution"]["recommended_action_source"] == "agent_lane_step" + assert written["recommended_action_resolution"]["todo_id"] == context["todo"] + record = json.loads(Path(written["json_path"]).read_text()) + assert record["goal_ref"] == context["goal_ref"] and record["agent_id"] == "codex-fixture" + assert record["progress_observation"]["result_class"] == "blocked" + assert record["blocked_retry"]["schema_version"] == "quota_blocked_retry_v0" + current = next(row for row in routes() if row["agent_id"] == "codex-fixture") + assert current["todo_id"] == "todo_fixture0001" and current["next_step"] == step + assert current["next_action_basis"] != basis + later = turn_run_once.read_heartbeat_settlement(context["runtime"], goal_id=context["goal"], + agent_id=context["agent"], todo_id=context["todo"], turn_instance_id=turn, + registry_path=context["registry"], goal_ref=context["goal_ref"]) + assert later is not None and later.prepared_lease == settlement.prepared_lease + assert later.writeback_run is not None and later.writeback_run["goal_ref"] == context["goal_ref"] + assert later.spend_event is None and later.completion_event is None + assert [line for line in context["log"].read_bytes().splitlines(keepends=True) + if json.loads(line).get("event_kind") == "quota_should_run"] == before_receipt + assert state.read_bytes() == state_bytes and context["registry"].read_bytes() == registry_bytes + code, inspected = _cli(context["registry"], context["runtime"], "task-lease", "inspect", + "--goal-id", context["goal"], "--todo-id", context["todo"]) + assert code == 0 and inspected["lease"] == lease, inspected + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("exact_ref", [False, True]) +def test_fresh_native_records_modern_prepared_admission_before_host(tmp_path, monkeypatch, provider, exact_ref): + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=exact_ref) + turn = "fresh-native-admission" + code, result = context["invoke"](turn) + assert code == 1 and result["ok"] is False, result # Host deliberately stops after admission. + assert len(context["host_receipts"]) == 1, (code, result) + receipt = context["host_receipts"][0] + details, lease = receipt["details"], context["lease"] + assert (receipt["goal_id"], receipt["agent_id"], receipt["run_id"]) == ( + context["goal"], context["agent"], turn) + assert details["todo_id"] == details["selected_todo_id"] == context["todo"] + assert {field: details[field] for field in ( + "ok", "should_run", "must_attempt_work", "delivery_allowed", "quiet_noop_allowed", "closeout_required", + )} == {"ok": True, "should_run": True, "must_attempt_work": True, + "delivery_allowed": True, "quiet_noop_allowed": False, "closeout_required": True} + assert details["prepared_lease_key_sha256"] == sha256(lease["idempotency_key"].encode()).hexdigest() + assert details["prepared_lease_version"] == lease["version"] + assert details["prepared_lease_epoch"] == lease["lease_epoch"] + assert "causality" not in receipt + assert receipt.get("goal_ref") == context["goal_ref"] + if exact_ref: + assert receipt["goal_ref"] == { + "goal_id": "loopx-turn-fixture", "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + original = context["log"].read_bytes() + renewed = renew_task_lease(registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=context["goal"], todo_id=context["todo"], owner=context["agent"], + idempotency_key=lease["idempotency_key"], expected_version=lease["version"], ttl_seconds=600) + assert renewed["ok"] is True, renewed + assert context["log"].read_bytes() == original + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("mirror", [False, True]) +def test_fresh_native_exact_journal_admission_preserves_original_ref_on_resume( + tmp_path, monkeypatch, provider, mirror, +): + from loopx.control_plane.turn_driver import turn_journal_runtime + + context = _fresh_native_admission_fixture( + tmp_path, monkeypatch, provider, exact_ref=True, mirror=mirror) + actual_effect = turn_journal_runtime.effect_runtime_result + journal_writes = [] + + def observe_journal_write(method, params, **kwargs): + result = actual_effect(method, params, **kwargs) + if method == "turn_journal.write": + journal_writes.append((json.loads(json.dumps(params)), json.loads(json.dumps(result)))) + return result + + monkeypatch.setattr(turn_journal_runtime, "effect_runtime_result", observe_journal_write) + turn = "fresh-native-exact-journal-resume" + code, result = context["invoke"](turn) + assert code == 1 and result["ok"] is False, result + assert len(context["host_receipts"]) == 1, (code, result) + journal = _turn_journal(context["runtime"]) + assert journal["host_attempt_count"] == 1 + capture_path = context["runtime"] / "goals" / "loopx-turn-fixture" / "turns" / "decisions" / ( + journal["turn_key"].removeprefix("sha256:") + ".json") + capture_bytes = capture_path.read_bytes() + original_receipt = context["log"].read_bytes() + fresh_write_count = len(journal_writes) + assert fresh_write_count > 0 + + code, result = context["invoke"](turn, resume_turn_key=journal["turn_key"], override_ref={ + "goal_id": "loopx-turn-fixture", "goal_instance_id": "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) + assert code == 1 and result["ok"] is False, result + assert len(context["host_receipts"]) == 2, (code, result) + resumed = _turn_journal(context["runtime"]) + assert resumed["host_attempt_count"] == 2 + assert resumed["turn_key"] == journal["turn_key"] + assert resumed["plan"] == journal["plan"] + assert capture_path.read_bytes() == capture_bytes + assert len(journal_writes) > fresh_write_count + assert context["log"].read_bytes() == original_receipt + expected_ref = {"goal_id": "loopx-turn-fixture", + "goal_instance_id": "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + assert resumed["plan"]["goal_ref"] == expected_ref + source_paths = ([context["source_registry"], context["registry"]] if mirror + else [context["registry"]]) + for request, committed in journal_writes: + snapshot, admission = request["journal"], request["source_admission"] + assert committed["ok"] is True + assert snapshot["goal_id"] == "loopx-turn-fixture" + assert snapshot["plan"]["goal_ref"] == admission["planned_goal_ref"] == expected_ref + assert snapshot["plan"]["work_contract"]["agent_identity"]["agent_id"] == admission["agent_id"] == "codex-fixture" + assert admission["schema_version"] == "loopx_quota_native_admission_v0" + assert "profile_id" not in admission + assert admission["runtime_root"] == str(context["runtime"].resolve()) + assert admission["registry_path"] == str(context["registry"].resolve()) + assert admission["registry_sources"] == [ + {"path": str(path.resolve()), "sha256": sha256(path.read_bytes()).hexdigest()} + for path in source_paths] + assert [lock["role"] for lock in admission["locks"]] == [ + "run_index", *(["registry_source"] * len(source_paths))] + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("exact_ref", [False, True]) +def test_fresh_native_mirror_qualifies_and_reads_back_real_prepared_receipt( + tmp_path, monkeypatch, provider, exact_ref, +): + from loopx.control_plane import effect_runtime + from loopx.control_plane.quota import settlement_cli + from loopx.control_plane.quota.settlement import SettlementIdentity + from loopx import rollout_event_log + + context = _fresh_native_admission_fixture( + tmp_path, monkeypatch, provider, exact_ref=exact_ref, mirror=True) + actual_details = settlement_cli.quota_rollout_details + actual_effect = effect_runtime.effect_runtime_result + actual_append = rollout_event_log._append_rollout_event_line + producer_facts, publication_reads, appended_lines = [], [], [] + + def observe_producer(*args, **kwargs): + facts = actual_details(*args, **kwargs) + producer_facts.append(json.loads(json.dumps(facts))) + return facts + + def observe_publication(method, params, **kwargs): + result = actual_effect(method, params, **kwargs) + if method == "quota.settlement.read" and "details" in params.get("native_work_admission", {}): + publication_reads.append((json.loads(json.dumps(params)), json.loads(json.dumps(result)))) + return result + + def observe_append(path, event): + result = actual_append(path, event) + appended_lines.append(path.read_bytes().splitlines(keepends=True)[-1]) + return result + + monkeypatch.setattr(settlement_cli, "quota_rollout_details", observe_producer) + monkeypatch.setattr(effect_runtime, "effect_runtime_result", observe_publication) + monkeypatch.setattr(rollout_event_log, "_append_rollout_event_line", observe_append) + turn = "fresh-native-mirror-admission" + code, result = context["invoke"](turn) + + assert code == 1 and result["ok"] is False, result # Host deliberately stops after admission. + assert len(context["host_receipts"]) == 1, (code, result) + receipt = context["host_receipts"][0] + assert len(producer_facts) == 1 + assert len(publication_reads) == 2 + assert [result["native_work_admission"]["append"] for _, result in publication_reads] == [True, False] + source_paths = [context["source_registry"], context["registry"]] + for request, result in publication_reads: + admission = request["source_admission"] + assert request["borrow_source_admission"] is True + assert request.get("goal_ref") == admission["planned_goal_ref"] == context["goal_ref"] + assert admission["registry_path"] == str(context["registry"].resolve()) + assert admission["agent_id"] == context["agent"] + assert admission["runtime_root"] == str(context["runtime"].resolve()) + assert admission["registry_sources"] == [ + {"path": str(path.resolve()), "sha256": sha256(path.read_bytes()).hexdigest()} + for path in source_paths + ] + assert [lock["role"] for lock in admission["locks"]] == [ + "run_index", "registry_source", "registry_source", "authority_maintenance", "heartbeat_receipt"] + assert [lock["target"] for lock in admission["locks"][1:3]] == [ + str(path.resolve()) for path in source_paths] + assert result["native_work_admission"]["state"] == "ready" + qualified = publication_reads[0][1]["native_work_admission"] + readback = publication_reads[1][1] + assert receipt == readback["heartbeat_receipt"] + # The physical rollout carrier represents these absent diagnostics as + # null; the qualified logical facts retain the producer's empty strings. + empty_diagnostics = ("applied_rrule", "replan_obligation_id", "semantic_replan_obligation_id", + "slots", "successor_todo_ids", "target_key") + for field in empty_diagnostics: + assert qualified["details"][field] == "" + assert receipt["details"][field] is None + assert receipt["details"] == {**qualified["details"], + "applied_rrule": None, "replan_obligation_id": None, "semantic_replan_obligation_id": None, + "slots": None, "successor_todo_ids": None, "target_key": None} + assert (receipt["goal_id"], receipt["agent_id"], receipt["run_id"], receipt.get("goal_ref")) == ( + context["goal"], context["agent"], turn, context["goal_ref"]) + expected_prepared = { + "prepared_lease_key_sha256": sha256(context["lease"]["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": context["lease"]["version"], + "prepared_lease_epoch": context["lease"]["lease_epoch"], + } + expected_work = {**expected_prepared, "todo_id": context["todo"], "selected_todo_id": context["todo"], + "ok": True, "should_run": True, "must_attempt_work": True, "delivery_allowed": True, + "quiet_noop_allowed": False, "closeout_required": True, "prepared_lease_required": True, + "admission_resume_when": None, "admission_resume_ready": None} + for details in (producer_facts[0], qualified["details"], receipt["details"]): + for field, expected in expected_work.items(): + assert details[field] == expected and type(details[field]) is type(expected) + expected_identity = SettlementIdentity(goal_id=context["goal"], agent_id=context["agent"], + todo_id=context["todo"], turn_instance_id=turn) + for details in (qualified["details"], receipt["details"]): + assert details["turn_instance_id"] == turn + assert details["settlement_effect_id"] == expected_identity.effect_id + for disposition in (qualified, readback["native_work_admission"]): + assert disposition["original_prepared"] == expected_prepared + assert disposition["current_task_lease_proof"] == { + "idempotency_key": context["lease"]["idempotency_key"], "expected_version": context["lease"]["version"]} + assert len(appended_lines) == 1 and json.loads(appended_lines[0]) == receipt + physical_bytes = context["log"].read_bytes() + assert physical_bytes.endswith(appended_lines[0]) + later = turn_run_once.read_heartbeat_settlement(context["runtime"], goal_id=context["goal"], + agent_id=context["agent"], todo_id=context["todo"], turn_instance_id=turn, + registry_path=context["registry"], goal_ref=context["goal_ref"]) + assert later is not None and later.heartbeat_receipt == receipt + assert later.prepared_lease == expected_prepared + assert context["log"].read_bytes() == physical_bytes + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_native_mirror_source_chain_rejects_before_host_or_publication(tmp_path, monkeypatch, provider): + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, mirror=True) + source = context["source_registry"] + registration = json.loads(source.read_text()) + third_registry = source.with_name("third-registry.json") + third_registry.write_text(json.dumps(registration) + "\n") + registration["goals"][0]["source_registry"] = str(third_registry.resolve()) + source.write_text(json.dumps(registration) + "\n") + before = context["log"].read_bytes() + + code, refused = context["invoke"]("fresh-native-mirror-source-chain") + + assert code == 1 and refused["ok"] is False, refused + assert context["host_receipts"] == [] + assert context["log"].read_bytes() == before + assert refused["error_code"] == "goal_authority_unavailable" + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("foreign_resource", ["registry_source", "authority_maintenance"]) +def test_fresh_native_mirror_foreign_witness_rejects_before_receipt_append( + tmp_path, monkeypatch, provider, foreign_resource, +): + from loopx.control_plane import effect_runtime + from loopx.control_plane.projects.registry_codec import project_registry_transaction + from loopx.file_lock import cross_runtime_lock_witness, exclusive_mutation_file_lock + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, mirror=True) + foreign_registry = tmp_path / "foreign-registry.json" + foreign_registry.write_bytes(context["source_registry"].read_bytes()) + foreign_maintenance = tmp_path / "foreign-maintenance" + actual_effect = effect_runtime.effect_runtime_result + attempted_publications = [] + before = context["log"].read_bytes() + + def offer_foreign_witness(method, params, **kwargs): + if method != "quota.settlement.read" or "details" not in params.get("native_work_admission", {}): + return actual_effect(method, params, **kwargs) + request = json.loads(json.dumps(params)) + admission = request["source_admission"] + # These witnesses come from a current real lock and real registry bytes. + # They identify another resource and cannot replace the owning route/M. + with contextlib.ExitStack() as locks: + if foreign_resource == "registry_source": + transaction = locks.enter_context(project_registry_transaction( + foreign_registry, operation="fixture_foreign_native_registry")) + admission["registry_sources"][0] = transaction.source_read_witness() + admission["locks"][1] = {"role": "registry_source", + **cross_runtime_lock_witness(foreign_registry)} + else: + locks.enter_context(exclusive_mutation_file_lock( + foreign_maintenance, timeout_seconds=0, operation="fixture_foreign_native_maintenance")) + admission["locks"][3] = {"role": "authority_maintenance", + **cross_runtime_lock_witness(foreign_maintenance)} + attempted_publications.append(request) + return actual_effect(method, request, **kwargs) + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", offer_foreign_witness) + code, refused = context["invoke"]("fresh-native-mirror-foreign-witness") + + assert len(attempted_publications) == 1 + assert code == 1 and refused["ok"] is False, refused + assert context["host_receipts"] == [] + assert context["log"].read_bytes() == before + original = refused["original_ts_refusal"] + assert original["code"] == refused["error_code"] == ( + "goal_instance_conflict" if foreign_resource == "registry_source" else "quota_source_admission_invalid") + assert original["kind"] == ( + "conflict" if foreign_resource == "registry_source" else "request_rejected") + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_native_rejects_replacement_before_admission_append(tmp_path, monkeypatch, provider): + from loopx.control_plane.quota import settlement_cli + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider) + actual_details = settlement_cli.quota_rollout_details + replaced = False + + def replace_after_facts(*args, **kwargs): + nonlocal replaced + facts = actual_details(*args, **kwargs) + if not replaced: + replaced = True + lease = context["lease"] + released = release_task_lease(registry_path=context["registry"], runtime_root=context["runtime"], + goal_id=context["goal"], todo_id=context["todo"], owner=context["agent"], + idempotency_key=lease["idempotency_key"], expected_version=lease["version"]) + assert released["ok"] is True, released + code, claimed = _cli(context["registry"], context["runtime"], *context["claim_args"], + "--claim-operation-id", "replacement-before-native-admission") + assert code == 0, claimed + return facts + + monkeypatch.setattr(settlement_cli, "quota_rollout_details", replace_after_facts) + code, refused = context["invoke"]("fresh-native-replacement") + assert replaced and code == 1, refused + assert context["host_receipts"] == [] + assert not any(json.loads(line).get("event_kind") == "quota_should_run" + for line in context["log"].read_bytes().splitlines()) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("failure", ["append", "readback"]) +def test_fresh_native_publication_failure_never_invokes_host(tmp_path, monkeypatch, provider, failure): + from loopx import rollout_event_log + from loopx.control_plane import effect_runtime + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider) + admission_reads = 0 + actual_effect = effect_runtime.effect_runtime_result + + def fail_readback(method, params, **kwargs): + nonlocal admission_reads + if method == "quota.settlement.read" and "details" in params.get("native_work_admission", {}): + admission_reads += 1 + if admission_reads == 2: + raise RuntimeError("fixture native admission readback unavailable") + return actual_effect(method, params, **kwargs) + + def fail_append(*_args, **_kwargs): + raise OSError("fixture native admission append unavailable") + + if failure == "append": + monkeypatch.setattr(rollout_event_log, "_append_rollout_event_line", fail_append) + else: + monkeypatch.setattr(effect_runtime, "effect_runtime_result", fail_readback) + code, refused = context["invoke"]("fresh-native-publication-failure") + assert code == 1 and context["host_receipts"] == [], refused + assert refused["ok"] is False + receipts = [json.loads(line) for line in context["log"].read_bytes().splitlines() + if json.loads(line).get("event_kind") == "quota_should_run"] + assert len(receipts) == (0 if failure == "append" else 1) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_native_cannot_open_another_turn_over_original_todo_debt(tmp_path, monkeypatch, provider): + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider) + context["invoke"]("fresh-native-original-debt") + assert len(context["host_receipts"]) == 1 + original = context["log"].read_bytes() + code, refused = context["invoke"]("fresh-native-second-turn") + assert code == 1, refused + assert len(context["host_receipts"]) == 1 + assert context["log"].read_bytes() == original + code, inspected = _cli(context["registry"], context["runtime"], "task-lease", "inspect", + "--goal-id", context["goal"], "--todo-id", context["todo"]) + assert code == 0 and inspected["lease"] == context["lease"], inspected + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("metadata_present", [False, True]) +def test_fresh_native_exact_ref_requires_registered_incarnation(tmp_path, monkeypatch, provider, metadata_present): + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=metadata_present) + before = context["log"].read_bytes() + code, refused = context["invoke"]("fresh-native-wrong-incarnation", override_ref={ + "goal_id": context["goal"], "goal_instance_id": "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) + assert code == 1 and context["host_receipts"] == [], refused + assert context["log"].read_bytes() == before + assert refused["original_ts_refusal"]["code"] == refused["error_code"] == "goal_instance_conflict" + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_native_malformed_exact_ref_rejects_before_admission(tmp_path, monkeypatch, provider): + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider, exact_ref=True) + before = context["log"].read_bytes() + code, refused = context["invoke"]("fresh-native-malformed-incarnation", override_ref={ + "goal_id": context["goal"], "goal_instance_id": "native-admission-fixture-instance", + }) + + assert code == 1 and refused["ok"] is False, refused + assert refused["error"] == "goal_instance_id must be a Goal instance identifier" + assert context["host_receipts"] == [] + assert context["log"].read_bytes() == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_native_prior_debt_refusal_retains_typed_packet_before_preparer(tmp_path, monkeypatch, provider): + from loopx.cli_commands.quota_should_run import check_fresh_native_heartbeat_prior_closeout + from loopx.control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from loopx.control_plane.quota.settlement import SettlementIdentity + + context = _fresh_native_admission_fixture(tmp_path, monkeypatch, provider) + context["invoke"]("fresh-native-original-debt") + assert len(context["host_receipts"]) == 1 + before = context["log"].read_bytes() + with pytest.raises(RefreshTaskLeaseUnavailable) as refused: + check_fresh_native_heartbeat_prior_closeout( + identity=SettlementIdentity(goal_id=context["goal"], agent_id=context["agent"], + todo_id=context["todo"], turn_instance_id="fresh-native-second-turn"), + runtime_root=context["runtime"], registry_path=context["registry"], goal_ref=None) + original = refused.value.payload["original_ts_refusal"] + assert original["kind"] == "request_rejected" + assert original["code"] == "refresh_task_lease_unavailable" + assert original["message"] == str(refused.value.__cause__) + assert context["log"].read_bytes() == before diff --git a/tests/test_native_steward_reports.py b/tests/test_native_steward_reports.py index 70af53558a..bccf9c20eb 100644 --- a/tests/test_native_steward_reports.py +++ b/tests/test_native_steward_reports.py @@ -2186,6 +2186,413 @@ def test_stopped_no_typed_result_report_recovers_saved_editorial_candidate_witho assert capture.read_bytes() == calls +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("local_session", [False, True], ids=["portfolio", "local-session"]) +@pytest.mark.parametrize("lease_loss", ["active", "replacement", "released"]) +def test_stopped_saved_candidate_lost_execution_preserves_original_and_projects_one_notice( + steward, tmp_path, monkeypatch, provider, local_session, lease_loss): + from loopx.capabilities.native_chat import steward_commands + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + from loopx.capabilities.periodic_report.cadence_journal import cadence_journal_path + from loopx.cli_commands import quota_should_run + from loopx.control_plane.quota import execution_preparation + from loopx.control_plane.quota.settlement import read_heartbeat_settlement + from loopx.control_plane.turn_driver import host_candidate + from loopx.control_plane.turn_driver.journal_store import ( + find_loopx_turn_key_by_settlement_identity, load_turn_journal, turn_journal_path, + ) + + store, runtime, bot, transport, binding, capture, _ = steward + if local_session: + session, reports, _, _ = configured_local_report(steward, tmp_path, monkeypatch, provider) + sid = owner = session["session_id"] + actor = reports.selected(sid)["actor"] + else: + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, provider) + owner = binding["binding_id"] + install_model(steward, tmp_path, sid) + if local_session: + runtime.project_contexts.conversation_bindings = None + # Preserve a complete saved answer while the real Turn owner observes the + # older parser's untyped result. No admission, receipt or lease is invented. + with monkeypatch.context() as old_parser: + old_parser.setattr(host_candidate, "parse_model_json", lambda _text: None) + first = finish(runtime, pump_steward_monitors(runtime)[0]) + native = first["response"]["native_loopx"] + assert native["status"] == "stopped" and native["result_kind"] == "wait" + original_key = native["resume_turn_key"] + journal_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=original_key) + assert load_turn_journal(journal_path)["host_result"]["classification"] == "no_typed_host_result" + request = first["loopx_request"] + _, window = reports.current_window(owner) + folder = attempt_directory(reports.root, window) + assert not (folder / "private-generation.json").exists() + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) + assert task["status"] == "open" + assert StewardTurnHost(runtime).has_recoverable_stopped_report(actor, window, task) + + def readback(): + return read_heartbeat_settlement(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], todo_id=request["todo_id"], + turn_instance_id=request["turn_instance_id"], registry_path=runtime.registry_path, + goal_ref={"goal_id": actor["goal_id"], "goal_instance_id": actor["goal_instance_id"]}) + + original_readback = readback() + assert original_readback is not None and original_readback.identity.value is not None + original_details = original_readback.original_prepared_receipt["details"] + assert original_details["closeout_required"] is True and original_details["must_attempt_work"] is True + assert original_details["delivery_allowed"] is True and original_details["quiet_noop_allowed"] is False + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + actual_command = steward_commands.native_command + original_lease = actual_command(runtime, ["task-lease", "inspect", *route])["lease"] + retained = original_lease + if lease_loss != "active": + actual_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], + "--idempotency-key", original_lease["idempotency_key"], + "--expected-version", str(original_lease["version"])]) + retained = actual_command(runtime, ["task-lease", "inspect", *route])["lease"] + if lease_loss == "replacement": + retained = actual_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", request["turn_instance_id"] + ".replacement", "--ttl-seconds", "3600", + "--expected-version", str(retained["version"])])["lease"] + assert retained["idempotency_key"] != original_lease["idempotency_key"] + assert retained["lease_epoch"] > original_lease["lease_epoch"] + elif lease_loss == "released": + assert retained["status"] == "released" + + mutations, qualifications, preflights = [], [], [] + actual_qualification = execution_preparation.read_turn_execution_lease_qualification + actual_preflight = quota_should_run.check_fresh_native_heartbeat_prior_closeout + + def observe_command(controller, argv): + if argv[:2] in (["task-lease", "renew"], ["task-lease", "acquire"], ["task-lease", "release"]): + mutations.append(list(argv)) + return actual_command(controller, argv) + + def observe_qualification(**kwargs): + qualifications.append((kwargs["identity"].as_dict(), kwargs["requested_proof"])) + return actual_qualification(**kwargs) + + def observe_preflight(**kwargs): + preflights.append((kwargs["identity"].as_dict(), kwargs["goal_ref"])) + return actual_preflight(**kwargs) + + monkeypatch.setattr(steward_commands, "native_command", observe_command) + monkeypatch.setattr(execution_preparation, "read_turn_execution_lease_qualification", observe_qualification) + monkeypatch.setattr(quota_should_run, "check_fresh_native_heartbeat_prior_closeout", observe_preflight) + goal_folder = reports.root / "goals" / actor["goal_id"] + + def goal_bytes(): + return {path.relative_to(goal_folder): path.read_bytes() for path in goal_folder.rglob("*") + if path.is_file() and path.suffix in {".json", ".jsonl", ".md"}} + + protected = {path: path.read_bytes() for path in [runtime.registry_path, + store._turn_path(sid, first["turn_id"]), journal_path, cadence_journal_path(reports.root, actor["goal_id"])]} + original_goal_bytes = goal_bytes() + original_artifacts = {path.name: path.read_bytes() for path in folder.iterdir() if path.is_file()} + calls, writes = capture.read_bytes(), list(bot.writes) + with pytest.raises(execution_preparation.RefreshTaskLeaseUnavailable) as refused: + StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + payload = refused.value.payload + assert payload["error_code"] == execution_preparation.RefreshTaskLeaseUnavailable.code + assert payload["original_ts_refusal"]["code"] == execution_preparation.RefreshTaskLeaseUnavailable.code + assert payload["effects_scope"] == "current_invocation" + assert all(effect is False for effect in payload["effects"].values()) + assert payload["resume_turn_key"] == original_key + assert payload["journal_observation"]["scope"] == "original_turn" + gate = payload["settlement_recovery"] + assert gate["settlement_identity"] == original_readback.identity.value.as_dict() + assert gate["settlement_identity"]["turn_instance_id"] == request["turn_instance_id"] + assert gate["next_cli_actions_argv"] == [ + ["turn", "inspect-journal", "--goal-id", actor["goal_id"], "--agent-id", actor["agent_id"], + "--turn-key", original_key], + ["task-lease", "inspect", *route], + ] + assert not {"ready", "can_execute", "control_writer_proof", "claim", "reactivate"} & gate.keys() + for _ in range(2): + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + health = reports.health(owner) + assert health["recovery_failure"] == payload and health["execution_failed"] + notices = [row for row in store.messages(sid) if row.get("message_id") == health["recovery_message_id"]] + assert len(notices) == 1 and notices[0]["role"] == "error" and notices[0]["turn_id"] == first["turn_id"] + assert all(command in notices[0]["text"] for command in gate["next_cli_actions"]) + failures = [row for row in store.events_after(sid, first["turn_id"], None) + if row["kind"] == "native_loopx.result" and row["payload"] == payload] + assert len(failures) == 1 + assert mutations == [] and qualifications == [] and preflights + assert all(identity["goal_id"] == actor["goal_id"] and identity["agent_id"] == actor["agent_id"] + and identity["todo_id"] == task["todo_id"] + and identity["turn_instance_id"] == request["turn_instance_id"] + ".saved-candidate" + and goal_ref == {"goal_id": actor["goal_id"], "goal_instance_id": actor["goal_instance_id"]} + for identity, goal_ref in preflights) + assert actual_command(runtime, ["task-lease", "inspect", *route])["lease"] == retained + assert next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == task["todo_id"]) == task + assert {path: path.read_bytes() for path in protected} == protected + assert goal_bytes() == original_goal_bytes + assert {path.name: path.read_bytes() for path in folder.iterdir() if path.is_file()} == original_artifacts + assert not (folder / "private-generation.json").exists() and not (folder / "editorial-response.json").exists() + assert find_loopx_turn_key_by_settlement_identity(reports.root, goal_id=actor["goal_id"], + agent_id=actor["agent_id"], todo_id=task["todo_id"], + turn_instance_id=request["turn_instance_id"] + ".saved-candidate") is None + final_readback = readback() + assert final_readback.original_prepared_receipt == original_readback.original_prepared_receipt + assert final_readback.prepared_lease == original_readback.prepared_lease + assert final_readback.original_prepared_event_id == original_readback.original_prepared_event_id + assert final_readback.original_prepared_receipt_sha256 == original_readback.original_prepared_receipt_sha256 + assert final_readback.writeback_run is None and final_readback.spend_run is None and final_readback.completion_event is None + assert read_periodic_report_publication_cursor(runtime_root=reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"]) is None + if not local_session: + assert transport.steward_reports.reconcile() == 0 + assert capture.read_bytes() == calls and bot.writes == writes + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("saved_lease_loss,recovery_attachment", [ + (None, "exact"), ("released", "exact"), ("replacement", "exact"), + ("released", "missing"), ("replacement", "missing"), + ("released", "malformed"), ("replacement", "malformed"), +], ids=["fresh-completes", "saved-released", "saved-replacement", "saved-released-missing", + "saved-replacement-missing", "saved-released-malformed", "saved-replacement-malformed"]) +def test_saved_report_requires_real_original_closeout_then_its_own_fresh_native_admission( + steward, tmp_path, monkeypatch, provider, saved_lease_loss, recovery_attachment): + from hashlib import sha256 + from test_native_steward_turn_host import cli + from loopx.capabilities.native_chat.steward_commands import native_command + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.capabilities.native_chat.steward_reports import ManagedSessionReports + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + from loopx.control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable + from loopx.control_plane.quota.settlement import ReceiptBoundReplayPhase, read_heartbeat_settlement + from loopx.control_plane.turn_driver import executor, host_candidate + from loopx.control_plane.turn_driver.journal_store import ( + find_loopx_turn_key_by_settlement_identity, load_turn_journal, turn_journal_path, + ) + + session, reports, _, _ = configured_local_report(steward, tmp_path, monkeypatch, provider) + store, runtime, bot, _, _, capture, _ = steward + sid = session["session_id"] + actor = reports.selected(sid)["actor"] + goal_ref = {"goal_id": actor["goal_id"], "goal_instance_id": actor["goal_instance_id"]} + install_model(steward, tmp_path, sid) + runtime.project_contexts.conversation_bindings = None + with monkeypatch.context() as old_parser: + old_parser.setattr(host_candidate, "parse_model_json", lambda _text: None) + first = finish(runtime, pump_steward_monitors(runtime)[0]) + native, request = first["response"]["native_loopx"], first["loopx_request"] + assert native["status"] == "stopped" and native["result_kind"] == "wait" + _, window = reports.current_window(sid) + folder = attempt_directory(reports.root, window) + original_journal = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"]) + original_journal_bytes = original_journal.read_bytes() + saved_instance = request["turn_instance_id"] + ".saved-candidate" + + def readback(turn_instance): + return read_heartbeat_settlement(reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], + todo_id=request["todo_id"], turn_instance_id=turn_instance, + registry_path=runtime.registry_path, goal_ref=goal_ref) + + original = readback(request["turn_instance_id"]) + assert original is not None and original.original_prepared_receipt is not None + details = original.original_prepared_receipt["details"] + # The actual first-run producer publishes full modern facts, not an ensure + # upgrade or hand-written receipt fields. A first admission has no parent. + assert details["closeout_required"] is True and details["prepared_lease_required"] is True + assert details["ok"] is True and details["should_run"] is True and details["must_attempt_work"] is True + assert details["delivery_allowed"] is True and details["quiet_noop_allowed"] is False + assert details["settlement_effect_id"] == original.identity.value.effect_id + assert "settlement_receipt_revision" not in details + assert original.original_prepared_receipt.get("source_event_id") is None + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) + calls, writes = capture.read_bytes(), list(bot.writes) + with pytest.raises(RefreshTaskLeaseUnavailable): + StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + assert readback(saved_instance) is None + assert not (folder / "private-generation.json").exists() + assert original_journal.read_bytes() == original_journal_bytes + assert capture.read_bytes() == calls and bot.writes == writes + + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + p1 = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + assert original.prepared_lease == { + "prepared_lease_key_sha256": sha256(p1["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": p1["version"], "prepared_lease_epoch": p1["lease_epoch"], + } + native_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], + "--idempotency-key", p1["idempotency_key"], "--expected-version", str(p1["version"])]) + released = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + p2 = native_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", "saved-report-control-writer", "--ttl-seconds", "3600", + "--expected-version", str(released["version"])])["lease"] + assert p2["lease_epoch"] > p1["lease_epoch"] and p2["idempotency_key"] != p1["idempotency_key"] + # The supported administrative owner explicitly settles only P1's debt. + # P2's control proof does not by itself authorize the saved business Run. + closed = cli(runtime, "refresh-state", "--goal-id", actor["goal_id"], + "--goal-instance-id", actor["goal_instance_id"], "--agent-id", actor["agent_id"], + "--todo-id", request["todo_id"], "--turn-instance-id", request["turn_instance_id"], + "--execution-loss-closeout", "--classification", "execution_loss_closeout", + "--delivery-batch-scale", "single_surface", "--delivery-outcome", "outcome_gap", + "--delivery-boundary", "semantic_closeout", "--progress-result-class", "blocked", + "--progress-work-item-id", request["todo_id"], "--progress-blocker-id", "original-execution-lost", + "--progress-evidence-id", "evidence:original-lease-released", "--no-global-sync", "--suppress-external-sinks", + "--task-lease-idempotency-key", p2["idempotency_key"], "--task-lease-expected-version", str(p2["version"])) + assert closed["settlement_progress"]["state"] == "settled" + settled_original = readback(request["turn_instance_id"]) + assert settled_original.replay_phase is ReceiptBoundReplayPhase.SETTLED + assert settled_original.prepared_lease == original.prepared_lease + assert settled_original.original_prepared_receipt == original.original_prepared_receipt + assert settled_original.spend_run is None and settled_original.completion_event is None + assert original_journal.read_bytes() == original_journal_bytes and not (folder / "private-generation.json").exists() + + frozen_after_admission = [] + actual_freeze = ManagedSessionReports.freeze + + def observe_freeze(service, report_request, candidate): + admitted = readback(saved_instance) + assert admitted is not None and admitted.original_prepared_receipt is not None + assert admitted.identity.value.turn_instance_id == saved_instance + assert admitted.original_prepared_receipt["details"]["closeout_required"] is True + assert admitted.prepared_lease == { + "prepared_lease_key_sha256": sha256(p2["idempotency_key"].encode()).hexdigest(), + "prepared_lease_version": p2["version"], "prepared_lease_epoch": p2["lease_epoch"], + } + frozen_after_admission.append(admitted.identity.value.effect_id) + return actual_freeze(service, report_request, candidate) + + monkeypatch.setattr(ManagedSessionReports, "freeze", observe_freeze) + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) + actual_writer = executor._write_journal + interrupted = False + + def interrupt_saved_checkpoint(path, journal, **kwargs): + nonlocal interrupted + actual_writer(path, journal, **kwargs) + identity = journal["plan"]["transaction"]["settlement_plan"]["identity"] + if (saved_lease_loss is not None and not interrupted + and identity["turn_instance_id"] == saved_instance + and (journal.get("task_validation") or {}).get("status") == "passed"): + interrupted = True + raise OSError("saved report interrupted after its real validation checkpoint") + + with monkeypatch.context() as journal_fault: + journal_fault.setattr(executor, "_write_journal", interrupt_saved_checkpoint) + StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + assert len(frozen_after_admission) == 1 and validate_private_generation(folder) + saved = readback(saved_instance) + assert saved.identity.value.turn_instance_id == saved_instance + assert saved.identity.value.effect_id != original.identity.value.effect_id + assert saved.prepared_lease != original.prepared_lease + saved_key = find_loopx_turn_key_by_settlement_identity(reports.root, goal_id=actor["goal_id"], + agent_id=actor["agent_id"], todo_id=request["todo_id"], turn_instance_id=saved_instance) + saved_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=saved_key) + saved_journal = load_turn_journal(saved_path) + if saved_lease_loss is not None: + from dataclasses import replace + from loopx.capabilities.native_chat import steward_commands + from loopx.control_plane.quota import settlement + + assert interrupted and saved_journal["task_validation"]["status"] == "passed" + assert saved_journal["host_attempt_count"] == 1 + assert saved.writeback_run is None and saved.spend_run is None and saved.completion_event is None + active_saved = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + native_command(runtime, ["task-lease", "release", *route, "--owner", actor["agent_id"], + "--idempotency-key", active_saved["idempotency_key"], + "--expected-version", str(active_saved["version"])]) + retained = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + if saved_lease_loss == "replacement": + retained = native_command(runtime, ["task-lease", "acquire", *route, "--owner", actor["agent_id"], + "--idempotency-key", "saved-report-replacement", "--ttl-seconds", "3600", + "--expected-version", str(retained["version"])])["lease"] + assert retained["lease_epoch"] > active_saved["lease_epoch"] + mutations, readonly_reads = [], [] + actual_command = steward_commands.native_command + actual_readback = settlement.read_heartbeat_settlement + + def observe_command(controller, argv): + if argv[:2] in (["task-lease", "acquire"], ["task-lease", "renew"], ["task-lease", "release"]): + mutations.append(list(argv)) + return actual_command(controller, argv) + + def read_recovery_attachment(*args, **kwargs): + result = actual_readback(*args, **kwargs) + if kwargs.get("include_readonly_recovery") is True: + readonly_reads.append(kwargs) + # Alter only the optional display attachment after the real + # owner read. Execution qualification and proof stay real. + if recovery_attachment == "missing": + return replace(result, settlement_recovery=None) + if recovery_attachment == "malformed": + return replace(result, settlement_recovery={**result.settlement_recovery, + "next_cli_actions_argv": [["todo", "claim"]]}) + return result + + monkeypatch.setattr(steward_commands, "native_command", observe_command) + monkeypatch.setattr(settlement, "read_heartbeat_settlement", read_recovery_attachment) + protected = {path: path.read_bytes() for path in [runtime.registry_path, original_journal, saved_path, + store._turn_path(sid, first["turn_id"])]} + goal_folder = reports.root / "goals" / actor["goal_id"] + def goal_bytes(): + return {path.relative_to(goal_folder): path.read_bytes() for path in goal_folder.rglob("*") + if path.is_file() and path.suffix in {".json", ".jsonl", ".md"}} + before = goal_bytes() + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) + with pytest.raises(RefreshTaskLeaseUnavailable) as refused: + StewardTurnHost(runtime).recover_report_settlement(actor, window, task) + payload = refused.value.payload + assert payload["resume_turn_key"] == saved_key and payload["effects_scope"] == "current_invocation" + assert payload["original_ts_refusal"]["code"] == RefreshTaskLeaseUnavailable.code + assert all(effect is False for effect in payload["effects"].values()) + for _ in range(2): + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + assert readonly_reads and all(row["turn_instance_id"] == saved_instance + and row["goal_ref"] == saved_journal["plan"]["goal_ref"] for row in readonly_reads) + if recovery_attachment == "exact": + gate = payload["settlement_recovery"] + assert gate["settlement_identity"] == saved.identity.value.as_dict() + assert gate["goal_ref"] == saved_journal["plan"]["goal_ref"] + assert gate["next_cli_actions_argv"][0] == ["turn", "inspect-journal", "--goal-id", actor["goal_id"], + "--agent-id", actor["agent_id"], "--turn-key", saved_key] + assert gate["original_settlement_readback"]["prepared_lease"] == saved.prepared_lease + assert gate["original_settlement_readback"]["original_prepared_receipt"] == saved.original_prepared_receipt + else: + assert "settlement_recovery" not in payload and "settlement_recovery_error" in payload + health = reports.health(sid) + assert health["recovery_failure"] == payload and health["execution_failed"] + notices = [row for row in store.messages(sid) if row.get("message_id") == health["recovery_message_id"]] + assert len(notices) == 1 and notices[0]["turn_id"] == first["turn_id"] and notices[0]["role"] == "error" + if recovery_attachment == "exact": + assert all(command in notices[0]["text"] for command in gate["next_cli_actions"]) + else: + assert "```sh" not in notices[0]["text"] + assert len([row for row in store.events_after(sid, first["turn_id"], None) + if row["kind"] == "native_loopx.result" and row["payload"] == payload]) == 1 + assert mutations == [] and len(frozen_after_admission) == 1 + assert actual_command(runtime, ["task-lease", "inspect", *route])["lease"] == retained + assert goal_bytes() == before and {path: path.read_bytes() for path in protected} == protected + after_saved, after_original = readback(saved_instance), readback(request["turn_instance_id"]) + assert after_saved.original_prepared_receipt == saved.original_prepared_receipt + assert after_saved.prepared_lease == saved.prepared_lease + assert after_saved.writeback_run is None and after_saved.spend_run is None and after_saved.completion_event is None + assert after_original.replay_phase is ReceiptBoundReplayPhase.SETTLED + assert after_original.prepared_lease == original.prepared_lease and after_original.spend_run is None + assert after_original.original_prepared_receipt == original.original_prepared_receipt + assert after_original.completion_event is None and original_journal.read_bytes() == original_journal_bytes + assert capture.read_bytes() == calls and bot.writes == writes + return + assert saved.replay_phase is ReceiptBoundReplayPhase.SETTLED and saved.spend_run is not None + assert saved_journal["status"] == "committed" + done = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) + assert done["status"] == "done" and done["completion_turn_key"] == saved_instance + after = readback(request["turn_instance_id"]) + assert after.prepared_lease == original.prepared_lease and after.original_prepared_receipt == original.original_prepared_receipt + assert after.spend_run is None and after.completion_event is None + assert original_journal.read_bytes() == original_journal_bytes + assert capture.read_bytes() == calls and bot.writes == writes + + def test_stopped_report_without_valid_saved_editorial_candidate_stays_closed(steward, tmp_path, monkeypatch): from loopx.capabilities.native_chat.steward_monitor import StewardMonitor from loopx.control_plane.turn_driver import host_candidate From 08faeb080379e48d560d41f9b5083c56a7c969e8 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 18:54:04 +0800 Subject: [PATCH 11/12] fix(recovery): preserve selection and semantic operation scopes Keep ordinary selection reservations causal, separate automatic workspace provenance from actual operations, and preserve cached and accounting-only recovery. Qualify physical and indexed writeback records and retain the original saved report owner. Record the pending aggregate source qualification in the paired RFC checkpoints. Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 6 + .../monorepo-distribution-split-v0.zh-CN.md | 5 + loopx/cli_commands/turn_run_once.py | 54 +++- .../quota/heartbeat_receipt_identity.ts | 46 +++- loopx/state_refresh.py | 2 +- .../test_quota_execution_preparation.py | 115 +++++++- .../quota_settlement_readback.test.ts | 252 +++++++++++++++++- tests/test_codex_turn_completion.py | 65 +++++ tests/test_native_steward_reports.py | 61 ++++- 9 files changed, 566 insertions(+), 40 deletions(-) diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 3ef0805835..22fc4b196e 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4846,6 +4846,12 @@ view shows its independently checked result in the same conversation. remain outside that guarantee. Uncaused repeats preserve the complete typed review/capability source and earliest admission; the same obligation id cannot authorize a source rewrite. + Integration remains held for final writeback of aggregate replan duties + whose original receipt has no scalar review source. Early and final + qualification must share a complete canonical-source reader and the existing + TS semantic owner; a caller-authored obligation or a digest-only snapshot + cannot replace that provenance. This unresolved path is not qualified by + passing claim, report, cached-result or TypeScript controls. - **Recovery readback:** a frozen report retains the typed original-lease refusal. An optional, default-off settlement attachment verifies its original owner, identity and journal before providing journal and task-lease inspection diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index 38f611ece5..adf0265e8b 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3742,6 +3742,11 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 写仍不属于此保证。 无因果链接的重复记录保留完整 typed review/capability 来源与最早准入;相同 obligation id 不能授权来源改写。 + 原 receipt 没有 scalar review source 的 aggregate replan duty,其最终写回 + 仍阻塞整合。早期与最终资格核验必须共用完整 canonical-source reader 和既有 + TS semantic owner;caller 自行构造的 obligation 或只有 digest 的 snapshot + 不能替代来源证明。领取、报告、缓存结果或 TypeScript 控制通过,不构成这条 + 未解决路径的资格。 - **恢复回读:**frozen report 保留 typed 原租约拒绝。默认关闭的可选 settlement attachment 核验原 owner、identity 与 journal 后,仅提供 journal 与 task-lease 检查命令;证据缺失或冲突时不提供命令。receiver 记录幂等的本地提示与同一 diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 7805b34098..b70797a937 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -31,6 +31,10 @@ SettlementStepKind, read_heartbeat_settlement, ) +from ..control_plane.quota.settlement_workspace_causality import ( + build_delivery_workspace_causality, + resolve_settlement_workspace_requirement, +) from ..control_plane.quota.spend_sources import quota_spend_source_for_execution_context from ..control_plane.work_items.autonomous_replan_obligation import ( replan_obligation_id_from_packet, @@ -215,15 +219,17 @@ def current_execution_lease( raise RefreshTaskLeaseUnavailable("The qualified Turn has no current task lease proof.") return dict(proof) - if resume_execution: + execution_preparation_needed = True + if args.execute and getattr(args, "resume_turn_key", None): inspection = inspect_loopx_turn_journal( runtime_root, goal_id=args.goal_id, agent_id=args.agent_id, turn_key=args.resume_turn_key, retry_failed=bool(args.retry_failed_turn), ) recovery = inspection.get("recovery_decision") # Retained results and the typed continuation to accounting need - # no execution resolver. Spend still uses its fresh receipt owner. - if not isinstance(recovery, Mapping) or not ( + # no execution lease or workspace preparation. Spend still uses + # its fresh receipt owner; this grants no workspace contract waiver. + execution_preparation_needed = not (isinstance(recovery, Mapping) and ( recovery.get("action") == "return_existing" or ( recovery.get("action") == "continue" @@ -231,8 +237,9 @@ def current_execution_lease( and recovery.get("reinvoke_host") is False and recovery.get("resume_from") == "quota_spend" ) - ): - current_execution_lease() + )) + if resume_execution and execution_preparation_needed: + current_execution_lease() replan_guard_scoped = execution_context["replan_guard_scoped"] replan_obligation_id = ( replan_obligation_id_from_packet( @@ -272,13 +279,36 @@ def append_settlement_event( ) writeback_contract = execution_context["writeback"] - delivery_workspace_path = ( - project - if isinstance( - writeback_contract.get("delivery_workspace_causality"), dict - ) and not (settlement_identity.replan_obligation_id and not selected_todo) - else None - ) + delivery_workspace_path = None + if execution_preparation_needed: + if getattr(args, "resume_turn_key", None): + # Recovery uses the journal's original identity and saved Goal Ref, + # never the current Todo contract or a presentation hint. + original_settlement = read_heartbeat_settlement( + runtime_root, goal_id=settlement_identity.goal_id, + agent_id=settlement_identity.agent_id, + todo_id=settlement_identity.todo_id, + turn_instance_id=settlement_identity.turn_instance_id, + replan_obligation_id=settlement_identity.replan_obligation_id, + registry_path=registry_path, goal_ref=goal_ref, + ) + workspace_causality = ( + original_settlement.workspace_causality + if original_settlement is not None else None + ) + else: + workspace_causality = build_delivery_workspace_causality(selected_todo) + workspace_requirement = resolve_settlement_workspace_requirement( + workspace_causality, + settlement_binding_kind=settlement_identity.binding_kind.value, + )["requirement"] + delivery_workspace_path = ( + project + if workspace_requirement != "not_required" and isinstance( + writeback_contract.get("delivery_workspace_causality"), dict + ) and not (settlement_identity.replan_obligation_id and not selected_todo) + else None + ) completion_delivery_workspace = ( capture_delivery_workspace( delivery_workspace_path, diff --git a/loopx/control_plane/quota/heartbeat_receipt_identity.ts b/loopx/control_plane/quota/heartbeat_receipt_identity.ts index bcd474ab32..36857a7fda 100644 --- a/loopx/control_plane/quota/heartbeat_receipt_identity.ts +++ b/loopx/control_plane/quota/heartbeat_receipt_identity.ts @@ -384,11 +384,13 @@ function heartbeatSemanticWorkAdmission(event: JsonObject, identity: SettlementI const fact = heartbeatReceiptFactFromEvent(event), receiptDetails = heartbeatReceiptDetails(event); const binding = heartbeatReceiptBinding(identity.goal_id, identity.agent_id, fact); const selected = normalizeHeartbeatReplanObligationId(receiptDetails.semantic_replan_obligation_id); + // Business-delivery closeout debt is a recorded boolean. Semantic report + // permission comes from the exact obligation and typed admission below. if (selected === null || event.schema_version !== "loopx_rollout_event_v0" || event.event_kind !== "quota_should_run" || event.goal_id !== identity.goal_id || event.agent_id !== identity.agent_id || fact.run_id !== identity.turn_instance_id || fact.todo_id !== identity.todo_id || binding?.binding_kind !== "todo" - || binding.binding_id !== identity.todo_id || fact.closeout_required !== true + || binding.binding_id !== identity.todo_id || typeof receiptDetails.closeout_required !== "boolean" || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id)) || receiptDetails.interaction_mode !== "autonomous_replan" || receiptDetails.delivery_allowed !== false || receiptDetails.ok !== true || receiptDetails.should_run !== true @@ -404,6 +406,42 @@ export function heartbeatSemanticAdmissionEvent(events: readonly JsonObject[], r return selected !== null && heartbeatSemanticWorkAdmission(selected, identity) !== null ? selected : null; } +function ordinarySelectionReservationParent(parent: JsonObject, upgrade: JsonObject, + identity: SettlementIdentity, allowLegacyPrepared: boolean): boolean { + const reservation = heartbeatReceiptDetails(parent), bound = heartbeatReceiptDetails(upgrade); + const empty = (value: unknown): boolean => value === undefined || value === null || value === ""; + const workFacts = (fields: JsonObject): boolean => fields.command === "quota" + && fields.quota_command === "should-run" && fields.turn_instance_id === identity.turn_instance_id + && fields.ok === true && fields.should_run === true && fields.must_attempt_work === true + && fields.selection_required === true && fields.quiet_noop_allowed === false + && ["request_id", "request_revision", "request_digest", "execution_scope_sha256"] + .every(field => !Object.hasOwn(fields, field)); + const negative = (fields: JsonObject): boolean => workFacts(fields) + && fields.delivery_allowed === false && fields.closeout_required === false + && ["prepared_lease_key_sha256", "prepared_lease_version", "prepared_lease_epoch"] + .every(field => !Object.hasOwn(fields, field)); + // quota_rollout_details records selected_todo_id before a portfolio choice. + // It is a recommendation, not this identity's eventual settlement binding. + return identity.binding_kind === "todo" && parent.schema_version === "loopx_rollout_event_v0" + && upgrade.schema_version === "loopx_rollout_event_v0" && parent.status === "normal_run" + && negative(reservation) && workFacts(bound) && reservation.interaction_mode === "bounded_delivery" + && (empty(reservation.selected_todo_id) + || normalizeHeartbeatTodoId(reservation.selected_todo_id) === reservation.selected_todo_id) + && empty(parent.todo_id) && ["todo_id", "replan_obligation_id", "settlement_effect_id", + "semantic_replan_obligation_id"].every(field => empty(reservation[field])) + && ["settlement_receipt_revision", "prepared_lease_required", "pending_action_selection_state", + "pending_action_selection_todo_id"].every(field => !Object.hasOwn(reservation, field)) + && !Object.hasOwn(parent, "causality") && bound.todo_id === identity.todo_id + && bound.selected_todo_id === identity.todo_id + && typeof bound.settlement_effect_id === "string" && effectIdsMatch(bound.settlement_effect_id, identity.effect_id) + && empty(bound.replan_obligation_id) && empty(bound.semantic_replan_obligation_id) + && (!Object.hasOwn(bound, "prepared_lease_required") || bound.prepared_lease_required === true) + // An already-held canonical lease can make the first identity binding + // positive. Reuse the strict prepared-admission owner for that case. + && (negative(bound) || (bound.prepared_lease_required === true + && heartbeatPreparedWorkAdmission(upgrade, identity, allowLegacyPrepared) !== null)); +} + function selectHeartbeatValidationEvent( events: readonly JsonObject[], receipt: JsonObject, identity: SettlementIdentity, allowLegacyPrepared: boolean, semanticAdmission: boolean, @@ -505,7 +543,11 @@ function selectHeartbeatValidationEvent( && parentDetails.pending_action_selection_state === "deferred" && parentDetails.pending_action_selection_todo_id === identity.todo_id && normalizeHeartbeatTodoId(parentDetails.pending_action_selection_todo_id) === identity.todo_id; - if ((!pendingParent && fact.todo_id !== identity.todo_id) + const selectionParent = revision === "identity_upgrade" + && ordinarySelectionReservationParent(parent, validation, identity, allowLegacyPrepared); + // A first binding cannot skip an earlier positive prepared admission. + if (selectionParent && [...positivePrepared.keys()].some(index => index < position)) return null; + if ((!pendingParent && !selectionParent && fact.todo_id !== identity.todo_id) || (fact.settlement_effect_id !== null && !effectIdsMatch(fact.settlement_effect_id, identity.effect_id))) return null; validation = parent; position = parentPositions[0]!; diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index 03d42e8b69..727d68395e 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -1634,7 +1634,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None): normalized_delivery_boundary == "semantic_closeout" and completion_todo_id is None and completion_turn_key is None - and not isinstance(record.get("delivery_workspace"), Mapping) + and delivery_workspace_path is None ): # Project the final Run without replacing its observed # work identity, semantic delta or recovery metadata. diff --git a/tests/control_plane/test_quota_execution_preparation.py b/tests/control_plane/test_quota_execution_preparation.py index 25d0d329b8..26c941569b 100644 --- a/tests/control_plane/test_quota_execution_preparation.py +++ b/tests/control_plane/test_quota_execution_preparation.py @@ -1994,22 +1994,102 @@ def test_todo_semantic_writeback_is_operation_scoped_without_business_lease( refresh = refresh.replace( "", str(vision), ) - for key, value in { - "": "advanced", - "": "semantic-writeback-surface", - "": "semantic-writeback-hypothesis", - "": "semantic-writeback-probe", - "": "evidence:semantic-writeback", - }.items(): - refresh = refresh.replace(key, value) + # The projected recipe owns settlement identity and semantic boundary. + # Author the actual observation independently through the existing CLI. + refresh_args = ( + *_projected_cli_args(refresh, turn_instance_id=turn), + "--progress-result-class", "advanced", + "--progress-work-item-id", SELECTED_REPLAN_TODO_ID, + "--progress-surface-id", "semantic-writeback-surface", + "--progress-hypothesis-id", "semantic-writeback-hypothesis", + "--progress-probe-kind", "semantic-writeback-probe", + "--progress-evidence-id", "evidence:semantic-writeback", + ) + context = { + "project": project, "runtime": runtime, "registry": registry, + "state": project / f".codex/goals/{GOAL_ID}/ACTIVE_GOAL_STATE.md", + "goal_id": GOAL_ID, "agent_id": AGENT_ID, + "todo": SELECTED_REPLAN_TODO_ID, "turn": turn, + } + before = _writeback_side_effect_snapshot(context) + assert before["authority"]["leases"] == [] + assert before["original_prepared"] is None + # An explicit producing-workspace operation still needs business authority; + # it cannot borrow this Turn's planning-only semantic writeback scope. + refused_code, refused = _run_cli( + registry, runtime, *refresh_args, + "--delivery-workspace-path", str(project), cwd=project, + ) + assert refused_code == 1 and refused["error_code"] == "refresh_task_lease_unavailable", refused + assert refused["appended"] is False + assert _writeback_side_effect_snapshot(context) == before code, result = _run_cli( - registry, runtime, *_projected_cli_args(refresh, turn_instance_id=turn), cwd=project, + registry, runtime, *refresh_args, cwd=project, ) assert code == 0 and result["settlement_result"]["ok"] is True, result - run = json.loads(Path(result["json_path"]).read_text(encoding="utf-8")) - semantic = run["autonomous_replan_ack"]["semantic_delta"] - assert semantic["accepted"] is True - assert semantic["observation_fingerprint"] == run["progress_observation"]["fingerprint"] + runs_dir = runtime / "goals" / GOAL_ID / "runs" + run_path = Path(result["json_path"]) + markdown_path = Path(result["markdown_path"]) + assert run_path.parent == runs_dir + assert list(runs_dir.glob("*.json")) == [run_path] + assert markdown_path == run_path.with_suffix(".md") + assert markdown_path.exists() + run = json.loads(run_path.read_text(encoding="utf-8")) + index_path = runs_dir / "index.jsonl" + assert Path(result["index_path"]) == index_path + [index_row] = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + assert Path(index_row["json_path"]) == run_path + assert Path(index_row["markdown_path"]) == markdown_path + assert run["state"]["path"] == str(context["state"]) + expected_identity = { + "schema_version": "quota_settlement_identity_v0", + "effect_id": ( + "settlement-cli-fixture:codex-settlement-cli:todo_chain_000000000000:" + "turn-operation-scoped-semantic-writeback" + ), + "goal_id": "settlement-cli-fixture", + "agent_id": "codex-settlement-cli", + "todo_id": "todo_chain_000000000000", + "turn_instance_id": "turn-operation-scoped-semantic-writeback", + } + assert admitted["heartbeat_receipt"]["settlement_identity"] == expected_identity + # The physical Run retains adapter presentation. Settlement readback owns + # the persisted index row; qualify both carriers against the authored slice. + for record in (run, index_row): + assert record["settlement_identity"] == expected_identity + assert record["goal_id"] == "settlement-cli-fixture" + assert record["agent_id"] == "codex-settlement-cli" + assert record["todo_id"] == "todo_chain_000000000000" + assert record["turn_instance_id"] == "turn-operation-scoped-semantic-writeback" + assert record["delivery_outcome"] == "outcome_progress" + assert record["delivery_batch_scale"] == "single_surface" + progress = record["progress_observation"] + assert progress["schema_version"] == "typed_progress_observation_v0" + assert progress["result_class"] == "advanced" + assert progress["work_item_id"] == "todo_chain_000000000000" + assert progress["surface_id"] == "semantic-writeback-surface" + assert progress["hypothesis_id"] == "semantic-writeback-hypothesis" + assert progress["probe_kind"] == "semantic-writeback-probe" + assert progress["evidence_ids"] == ["evidence:semantic-writeback"] + assert record["delivery_workspace"] == { + "schema_version": "delivery_workspace_v1", + "workspace_identity": "loopx:settlement-cli-fixture", + "identity_kind": "local_goal", + "task_repository": None, + "repository_source": "goal_id_fallback", + "workspace_kind": "local_goal_workspace", + "peer_independent_worktree_required": False, + } + ack = record["autonomous_replan_ack"] + assert ack["schema_version"] == "autonomous_replan_ack_v0" + assert ack["recorded"] is True and ack["source"] == "refresh_state" + semantic = ack["semantic_delta"] + assert semantic["schema_version"] == "replan_semantic_delta_v0" + assert semantic["accepted"] is True + assert "new_surface" in semantic["outcomes"] + assert semantic["observation_fingerprint"] == progress["fingerprint"] + assert index_row["progress_observation"] == run["progress_observation"] + assert index_row["autonomous_replan_ack"] == run["autonomous_replan_ack"] readback = read_heartbeat_settlement( runtime, goal_id=GOAL_ID, agent_id=AGENT_ID, todo_id=SELECTED_REPLAN_TODO_ID, turn_instance_id=turn, @@ -2017,8 +2097,10 @@ def test_todo_semantic_writeback_is_operation_scoped_without_business_lease( ) assert readback is not None assert readback.prepared_lease is None + assert readback.original_prepared_receipt is None + assert readback.resume_lease_qualification is None assert readback.completion_event is None - assert readback.writeback_run == run + assert readback.writeback_run == index_row with pytest.raises(RefreshTaskLeaseUnavailable): read_heartbeat_settlement( runtime, goal_id=GOAL_ID, agent_id=AGENT_ID, @@ -2026,6 +2108,11 @@ def test_todo_semantic_writeback_is_operation_scoped_without_business_lease( registry_path=registry, resume_lease_candidate={"requested_proof": None}, ) + after = _writeback_side_effect_snapshot(context) + assert after["authority"] == before["authority"] + assert after["authority"]["leases"] == [] + assert after["original_prepared"] is None + assert after["goal_file"] == before["goal_file"] def test_scoped_user_gate_fallback_execution_preparation_excludes_deferred_replan(monkeypatch): diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index f1da635b1e..154f87f243 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -37,7 +37,7 @@ import { PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, preflightPriorHostTurnCloseout, } from "../../loopx/control_plane/quota/unsettled_host_turn_recovery.ts"; -import {heartbeatSemanticAdmissionEvent} from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; +import {heartbeatPreparedWorkAdmission, heartbeatSemanticAdmissionEvent, heartbeatValidationEvent} from "../../loopx/control_plane/quota/heartbeat_receipt_identity.ts"; import {QUOTA_NATIVE_ADMISSION_SCHEMA} from "../../loopx/control_plane/quota/source_admission.ts"; import {shadowMaintenanceLockPath} from "../../loopx/control_plane/coordination/shadow_management.ts"; import {buildVisionCheckpoint, VISION_REFRESH_REQUEST_SCHEMA, visionAuthoringContract} @@ -640,6 +640,222 @@ function request(runtimeRoot: string, overrides: Record = {}) { }; } +function ordinarySelectionReceipts(chosenTodo = todoId) { + const chosenIdentity = settlementIdentity({goal_id: goalId, agent_id: agentId, + todo_id: chosenTodo, turn_instance_id: turnId}); + // Public-safe quota_rollout_details shapes: the first row recommends Todo A + // while requiring an explicit choice; the second binds that actual choice. + const reservation: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", + event_id: "selection-reservation", goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", + details: {command: "quota", quota_command: "should-run", turn_instance_id: turnId, + ok: true, should_run: true, must_attempt_work: true, selection_required: true, + delivery_allowed: false, quiet_noop_allowed: false, closeout_required: false, + interaction_mode: "bounded_delivery", todo_id: null, replan_obligation_id: null, + semantic_replan_obligation_id: null, settlement_effect_id: null, selected_todo_id: todoId}}; + const bound: JsonObject = {schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", + event_id: "selection-bound", goal_id: goalId, agent_id: agentId, run_id: turnId, status: "normal_run", + todo_id: chosenTodo, causality: {source_event_id: reservation.event_id, caused_by: reservation.event_id}, + details: {command: "quota", quota_command: "should-run", turn_instance_id: turnId, + ok: true, should_run: true, must_attempt_work: true, selection_required: true, + delivery_allowed: false, quiet_noop_allowed: false, closeout_required: false, + interaction_mode: "skip", todo_id: chosenTodo, selected_todo_id: chosenTodo, + replan_obligation_id: null, semantic_replan_obligation_id: null, + settlement_effect_id: chosenIdentity.effect_id, settlement_receipt_revision: "identity_upgrade", + prepared_lease_required: true}}; + return {reservation, bound, chosenIdentity}; +} + +for (const chosenTodo of [todoId, "todo_second_candidate"]) { + test(`ordinary negative selection reservation admits the explicit ${chosenTodo} and preserves first preparation`, async () => { + const root = await fixture({guard: false}); + try { + const {reservation, bound, chosenIdentity} = ordinarySelectionReceipts(chosenTodo); + const events: JsonObject[] = [reservation, bound]; + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const save = () => writeFile(log, events.map(event => JSON.stringify(event)).join("\n") + "\n"); + await save(); + const originalBytes = await readFile(log, "utf8"); + const input = request(root, {todo_id: chosenTodo}); + const before = await readQuotaSettlement(input); + assert.deepEqual(requireJsonObject(requireJsonObject(before.identity, "identity").result, "result").value, + settlementIdentityPayload(chosenIdentity)); + assert.deepEqual(before.heartbeat_receipt, bound); + assert.equal(before.original_prepared_receipt, null); + assert.equal(Object.hasOwn(before, "prepared_lease"), false); + assert.equal(await readFile(log, "utf8"), originalBytes); + + const proof = {prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 7, prepared_lease_epoch: 3}; + const ready = {...requireJsonObject(bound.details, "bound details"), ...proof, + interaction_mode: "bounded_delivery", delivery_allowed: true, closeout_required: true}; + const afterClaim = await readQuotaSettlement({...input, heartbeat_reentry_guard: ready}); + const qualification = requireJsonObject(afterClaim.heartbeat_reentry_qualification, "qualification"); + assert.equal(qualification.append, true); + const admittedDetails = requireJsonObject(qualification.details, "admitted details"); + assert.equal(admittedDetails.settlement_receipt_revision, "work_admission"); + assert.equal(admittedDetails.todo_id, chosenTodo); + assert.equal(admittedDetails.settlement_effect_id, chosenIdentity.effect_id); + assert.equal(afterClaim.original_prepared_receipt, null); + assert.equal(await readFile(log, "utf8"), originalBytes); + + const admitted: JsonObject = {...bound, event_id: "selection-work", status: "normal_run", details: admittedDetails, + causality: {source_event_id: bound.event_id, caused_by: bound.event_id}}; + events.push(admitted); + await save(); + const replay = await readQuotaSettlement({...input, heartbeat_reentry_guard: ready}); + assert.equal(requireJsonObject(replay.heartbeat_reentry_qualification, "replay qualification").append, false); + assert.deepEqual(replay.original_prepared_receipt, admitted); + assert.deepEqual(replay.prepared_lease, proof); + + const readiness: JsonObject = {...admitted, event_id: "selection-readiness", details: {...admittedDetails, + prepared_lease_version: 8, settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: admitted.event_id, caused_by: admitted.event_id}}; + events.push(readiness); + await save(); + const renewed = await readQuotaSettlement(input); + assert.deepEqual(renewed.original_prepared_receipt, admitted); + assert.deepEqual(renewed.prepared_lease, proof); + const replacement = {...readiness, event_id: "selection-replacement", details: {...requireJsonObject(readiness.details, "readiness"), + prepared_lease_epoch: 4}}; + assert.throws(() => heartbeatValidationEvent([...events, replacement], replacement, chosenIdentity), + {code: "heartbeat_receipt_identity_conflict"}); + } finally {await rm(root, {recursive: true, force: true});} + }); +} + +test("ordinary selection lineage rejects incomplete, bound or positive reservations", () => { + const {reservation, bound, chosenIdentity} = ordinarySelectionReceipts(); + const details = requireJsonObject(reservation.details, "reservation"); + for (const patch of [ + {selection_required: undefined}, {selection_required: "true"}, {ok: false}, {should_run: false}, + {must_attempt_work: undefined}, {quiet_noop_allowed: true}, {delivery_allowed: true}, + {closeout_required: undefined}, {closeout_required: "false"}, {closeout_required: true}, + {command: "turn"}, {quota_command: "ensure"}, {turn_instance_id: "foreign-turn"}, + {interaction_mode: "autonomous_replan"}, {todo_id: []}, {todo_id: "todo_foreign"}, + {settlement_effect_id: []}, {settlement_effect_id: "foreign-effect"}, + {replan_obligation_id: "replan-0000000000000001"}, {semantic_replan_obligation_id: "replan-0000000000000001"}, + {selected_todo_id: "malformed"}, {prepared_lease_required: false}, + {prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 7, prepared_lease_epoch: 3}, + {settlement_receipt_revision: "work_admission"}, {pending_action_selection_state: "bound"}, + ]) { + const invalid = {...reservation, details: {...details, ...patch}}; + if (patch.settlement_effect_id === "foreign-effect") { + assert.throws(() => heartbeatValidationEvent([invalid, bound], bound, chosenIdentity), + {code: "heartbeat_receipt_identity_conflict"}); + } else assert.equal(heartbeatValidationEvent([invalid, bound], bound, chosenIdentity), null, JSON.stringify(patch)); + } + for (const patch of [{schema_version: "unknown"}, {status: "turn_run_once"}, {todo_id: todoId}, + {goal_id: "foreign-goal"}, {agent_id: "foreign-agent"}, {run_id: "foreign-turn"}, + {goal_ref: {goal_id: goalId, goal_instance_id: instanceB}}, + {causality: {source_event_id: "earlier-work", caused_by: "earlier-work"}}]) { + const invalid = {...reservation, ...patch}; + assert.equal(heartbeatValidationEvent([invalid, bound], bound, chosenIdentity), null, JSON.stringify(patch)); + } +}); + +test("ordinary selection upgrade rejects partial work facts and requires one exact causal parent", () => { + const {reservation, bound, chosenIdentity} = ordinarySelectionReceipts(); + for (const patch of [{delivery_allowed: true}, {closeout_required: true}, {selection_required: undefined}, + {selected_todo_id: "todo_second_candidate"}, {prepared_lease_required: false}, + {prepared_lease_key_sha256: "a".repeat(64), prepared_lease_version: 7, prepared_lease_epoch: 3}]) { + const invalid = {...bound, details: {...requireJsonObject(bound.details, "bound"), ...patch}}; + assert.equal(heartbeatValidationEvent([reservation, invalid], invalid, chosenIdentity), null, JSON.stringify(patch)); + } + for (const causality of [{source_event_id: "missing", caused_by: "missing"}, + {source_event_id: reservation.event_id, caused_by: "foreign-parent"}]) { + const invalid = {...bound, causality}; + assert.equal(heartbeatValidationEvent([reservation, invalid], invalid, chosenIdentity), null); + } + for (const patch of [{schema_version: "unknown"}, {goal_id: "foreign-goal"}, {agent_id: "foreign-agent"}, + {run_id: "foreign-turn"}, {goal_ref: {goal_id: goalId, goal_instance_id: instanceB}}]) { + const invalid = {...bound, ...patch}; + assert.equal(heartbeatValidationEvent([reservation, invalid], invalid, chosenIdentity), null, JSON.stringify(patch)); + } + const foreignEffect = {...bound, details: {...requireJsonObject(bound.details, "bound"), settlement_effect_id: "foreign-effect"}}; + assert.equal(heartbeatValidationEvent([reservation, foreignEffect], foreignEffect, chosenIdentity), null); + assert.equal(heartbeatValidationEvent([reservation, reservation, bound], bound, chosenIdentity), null); + assert.equal(heartbeatValidationEvent([bound, reservation], bound, chosenIdentity), null); + + const deferred: JsonObject = {...reservation, status: "action_selection_deferred", details: { + pending_action_selection_state: "deferred", pending_action_selection_todo_id: todoId, + todo_id: "", replan_obligation_id: "", settlement_effect_id: ""}}; + assert.deepEqual(heartbeatValidationEvent([deferred, bound], bound, chosenIdentity), deferred); +}); + +test("selecting an already-prepared candidate permits its first positive identity binding", async () => { + for (const chosenTodo of [todoId, "todo_second_candidate"]) { + const root = await fixture({guard: false}); + try { + const {reservation, bound, chosenIdentity} = ordinarySelectionReceipts(chosenTodo); + const proof = {prepared_lease_key_sha256: "b".repeat(64), prepared_lease_version: 12, prepared_lease_epoch: 6}; + const admitted: JsonObject = {...bound, details: {...requireJsonObject(bound.details, "bound"), ...proof, + interaction_mode: "bounded_delivery", delivery_allowed: true, closeout_required: true}}; + const events = [reservation, admitted]; + const original = structuredClone(events); + assert.deepEqual(heartbeatValidationEvent(events, admitted, chosenIdentity), admitted); + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + await writeFile(log, events.map(event => JSON.stringify(event)).join("\n") + "\n"); + const encoded = await readFile(log, "utf8"); + const result = await readQuotaSettlement(request(root, {todo_id: chosenTodo})); + assert.deepEqual(requireJsonObject(requireJsonObject(result.identity, "identity").result, "result").value, + settlementIdentityPayload(chosenIdentity)); + assert.deepEqual(result.original_prepared_receipt, admitted); + assert.deepEqual(result.prepared_lease, proof); + assert.equal(await readFile(log, "utf8"), encoded); + assert.deepEqual(events, original); + + const readiness: JsonObject = {...admitted, event_id: "active-selection-readiness", details: { + ...requireJsonObject(admitted.details, "admitted"), settlement_receipt_revision: "lease_readiness", + prepared_lease_version: 13}, causality: {source_event_id: admitted.event_id, caused_by: admitted.event_id}}; + assert.deepEqual(heartbeatValidationEvent([...events, readiness], readiness, chosenIdentity), admitted); + const rebound = {...readiness, event_id: "second-identity-upgrade", details: {...requireJsonObject(readiness.details, "readiness"), + settlement_receipt_revision: "identity_upgrade"}, + causality: {source_event_id: reservation.event_id, caused_by: reservation.event_id}}; + assert.equal(heartbeatValidationEvent([...events, readiness, rebound], rebound, chosenIdentity), null); + + const partial = {...admitted, details: {...requireJsonObject(bound.details, "bound"), + interaction_mode: "bounded_delivery", delivery_allowed: true, closeout_required: true, + prepared_lease_key_sha256: proof.prepared_lease_key_sha256}}; + assert.throws(() => heartbeatValidationEvent([reservation, partial], partial, chosenIdentity), + {code: "malformed_settlement_state"}); + } finally {await rm(root, {recursive: true, force: true});} + } +}); + +test("selection histories retain no-epoch exact-version proofs and existing alias legacy policy", async () => { + const root = await fixture({guard: false}); + try { + const {reservation, bound, chosenIdentity} = ordinarySelectionReceipts("todo_second_candidate"); + const proof = {prepared_lease_key_sha256: "c".repeat(64), prepared_lease_version: 5}; + const modern: JsonObject = {...bound, details: {...requireJsonObject(bound.details, "bound"), ...proof, + interaction_mode: "bounded_delivery", delivery_allowed: true, closeout_required: true}}; + const legacy: JsonObject = {...bound, event_id: "selection-legacy-prepared", status: "turn_run_once", + details: {source: "loopx_turn_run_once", turn_instance_id: turnId, todo_id: chosenIdentity.todo_id, + settlement_effect_id: chosenIdentity.effect_id, prepared_lease_required: true, ...proof}}; + const log = join(root, "goals", goalId, "rollout-event-log.jsonl"); + for (const admitted of [modern, legacy]) { + await writeFile(log, [reservation, admitted].map(event => JSON.stringify(event)).join("\n") + "\n"); + const encoded = await readFile(log, "utf8"); + const result = await readQuotaSettlement(request(root, {todo_id: chosenIdentity.todo_id})); + assert.deepEqual(result.original_prepared_receipt, admitted); + assert.deepEqual(result.prepared_lease, proof); + assert.equal(Object.hasOwn(requireJsonObject(result.prepared_lease, "prepared"), "prepared_lease_epoch"), false); + assert.equal(await readFile(log, "utf8"), encoded); + const readiness: JsonObject = {...admitted, event_id: "same-version-readiness", details: { + ...requireJsonObject(admitted.details, "admitted"), settlement_receipt_revision: "lease_readiness"}, + causality: {source_event_id: admitted.event_id, caused_by: admitted.event_id}}; + assert.deepEqual(heartbeatValidationEvent([reservation, admitted, readiness], readiness, chosenIdentity, true), admitted); + const replacement = {...readiness, details: {...requireJsonObject(readiness.details, "readiness"), prepared_lease_version: 6}}; + assert.throws(() => heartbeatValidationEvent([reservation, admitted, replacement], replacement, chosenIdentity, true), + {code: "heartbeat_receipt_identity_conflict"}); + } + assert.deepEqual(heartbeatPreparedWorkAdmission(modern, chosenIdentity, false), proof); + assert.equal(heartbeatPreparedWorkAdmission(legacy, chosenIdentity, false), null); + assert.deepEqual(heartbeatPreparedWorkAdmission(legacy, chosenIdentity, true), proof); + assert.equal(heartbeatPreparedWorkAdmission({...legacy, details: {...requireJsonObject(legacy.details, "legacy"), + delivery_allowed: false}}, chosenIdentity, true), null); + } finally {await rm(root, {recursive: true, force: true});} +}); + test("pending closeout survives maintenance and reads one original receipt snapshot", async t => { for (const entry of [ {state: "writeback_required"}, @@ -3185,6 +3401,24 @@ for (const provider of ["file", "sqlite"] as const) { try { const logBefore = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; + for (const closeoutRequired of [false, true]) { + const recorded = {...f.admission, details: {...requireJsonObject(f.admission.details, "admission"), + closeout_required: closeoutRequired}}; + await writeFile(f.log, JSON.stringify(recorded) + "\n"); + const original = await readFile(f.log); + const qualified = await readQuotaSettlement(input); + assert.deepEqual(requireJsonObject(requireJsonObject(qualified.writeback_qualification, "qualification").result, "result").value, { + step_kind: "durable_writeback", effect_ref: `${identity.effect_id}#durable_writeback`, + delivery_boundary: "semantic_closeout", original_prepared: null, + selected_obligation_id: selectedWritebackObligation}); + assert.equal(qualified.original_prepared_receipt, null); + assert.equal(Object.hasOwn(qualified, "prepared_lease"), false); + assert.equal(qualified.resume_lease_qualification, undefined); + assert.equal(qualified.completion_event, null); + assert.deepEqual(await readFile(f.log), original); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); + } + await writeFile(f.log, logBefore); const result = await readQuotaSettlement(input); assert.deepEqual((result.writeback_qualification as any).result.value, { step_kind: "durable_writeback", effect_ref: `${identity.effect_id}#durable_writeback`, @@ -3257,13 +3491,27 @@ for (const provider of ["file", "sqlite"] as const) { try { const input: JsonObject = {...request(f.root), ...todoSemanticWriteback()}; for (const patch of [{must_attempt_work: false}, {should_run: false}, {ok: false}, - {quiet_noop_allowed: true}, {closeout_required: false}, {delivery_allowed: true}, + {quiet_noop_allowed: true}, {closeout_required: undefined}, {closeout_required: null}, + {closeout_required: "false"}, {closeout_required: 0}, {delivery_allowed: true}, {interaction_mode: "control_plane_self_repair"}, {semantic_replan_obligation_id: ""}, {semantic_replan_review_kind: undefined}, {prepared_lease_required: true}, {prepared_lease_key_sha256: "a".repeat(64)}, {prepared_lease_version: 119}, {prepared_lease_epoch: 50}]) { await writeFile(f.log, JSON.stringify({...f.admission, details: {...f.admission.details as JsonObject, ...patch}}) + "\n"); await assert.rejects(readQuotaSettlement(input), JSON.stringify(patch)); } + const business: JsonObject = {...f.admission, event_id: "earlier-business-preparation", details: { + ...requireJsonObject(f.admission.details, "admission"), delivery_allowed: true, closeout_required: true, + prepared_lease_required: true, prepared_lease_key_sha256: "a".repeat(64), + prepared_lease_version: 119, prepared_lease_epoch: 50}}; + const review: JsonObject = {...f.admission, event_id: "later-semantic-review", details: { + ...requireJsonObject(f.admission.details, "admission"), closeout_required: false, + settlement_receipt_revision: "review_guard_upgrade"}, + causality: {source_event_id: business.event_id, caused_by: business.event_id}}; + await writeFile(f.log, [business, review].map(event => JSON.stringify(event)).join("\n") + "\n"); + const before = await readFile(f.log), authorityBefore = await f.store.loadAuthority(); + await assert.rejects(readQuotaSettlement(input), {code: "refresh_task_lease_unavailable"}); + assert.deepEqual(await readFile(f.log), before); + assert.deepEqual(await f.store.loadAuthority(), authorityBefore); } finally {await rm(f.root, {recursive: true, force: true});} const {todo} = await import("./shadow_file_fixture.ts"); for (const selected of [{...todo(todoId), claimed_by: "foreign-agent"}, diff --git a/tests/test_codex_turn_completion.py b/tests/test_codex_turn_completion.py index 4f2574eeee..54fabd3881 100644 --- a/tests/test_codex_turn_completion.py +++ b/tests/test_codex_turn_completion.py @@ -881,6 +881,71 @@ def record_spend(*args, **kwargs): if accounting_only: assert final_readback.writeback_run == original_readback.writeback_run assert final_readback.spend_event is not None and final_readback.spend_run is not None + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + + turns_dir = runtime / "goals" / goal_id / "turns" + [committed_journal_path] = list(turns_dir.glob("*.json")) + committed_journal_bytes = committed_journal_path.read_bytes() + committed_journal = json.loads(committed_journal_bytes) + assert committed_journal["status"] == "committed" + assert committed_journal["host_attempt_count"] == 1 + assert committed_journal["host_result"] == frozen_result + capture_path = turns_dir / "decisions" / committed_journal_path.name + capture_bytes = capture_path.read_bytes() + committed_receipt_bytes = event_path.read_bytes() + state_path = project / ".codex" / "goals" / goal_id / "ACTIVE_GOAL_STATE.md" + state_bytes, registry_bytes = state_path.read_bytes(), registry.read_bytes() + assert json.loads(registry_bytes)["goals"][0]["status"] == "active" + run_files = {path: path.read_bytes() for path in index_path.parent.iterdir() + if path.suffix in {".json", ".md"}} + calls_before_cache = ( + len(host_calls), len(resolver_calls), len(resolver_errors), + len(qualification_requests), len(qualifications), len(completion_calls), + len(refresh_requests), len(spend_requests), + ) + index_bytes = index_path.read_bytes() + assert index_bytes.endswith(b"\n") + # This unrelated, newline-terminated malformed row must remain visible. + # Returning a committed Turn does not require or repair its Run history. + malformed_row = b'{"goal_id":"unrelated-cached-resume-fixture","classification": }\n' + with pytest.raises(json.JSONDecodeError): + json.loads(malformed_row) + with index_path.open("ab") as index: + index.write(malformed_row) + + actual_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + lifetime_checks = [] + + @contextlib.contextmanager + def observe_current_lifetime(admission, *, operation): + with actual_lifetime(admission, operation=operation): + lifetime_checks.append(operation) + yield + + with monkeypatch.context() as cached_observation: + cached_observation.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", observe_current_lifetime) + code, cached = invoke("--resume-turn-key", committed_journal["turn_key"]) + assert code == 0 and cached["status"] == "committed" and cached["replayed"] is True, cached + assert cached["resume_turn_key"] == committed_journal["turn_key"] + assert cached["receipt"] == committed_journal["receipt"] + assert not any(cached["effects"].values()) + assert "turn_recovery_lifetime" in lifetime_checks + assert committed_journal_path.read_bytes() == committed_journal_bytes + assert _turn_journal(runtime)["plan"] == committed_journal["plan"] + assert capture_path.read_bytes() == capture_bytes + assert event_path.read_bytes() == committed_receipt_bytes + assert original_receipt_lines() == receipt_bytes + assert state_path.read_bytes() == state_bytes and registry.read_bytes() == registry_bytes + assert index_path.read_bytes() == index_bytes + malformed_row + assert {path: path.read_bytes() for path in index_path.parent.iterdir() + if path.suffix in {".json", ".md"}} == run_files + assert calls_before_cache == ( + len(host_calls), len(resolver_calls), len(resolver_errors), + len(qualification_requests), len(qualifications), len(completion_calls), + len(refresh_requests), len(spend_requests), + ) + assert host_calls == [committed_journal["turn_key"]] + assert _turn_journal(runtime)["host_attempt_count"] == 1 def test_builtin_codex_advertises_explicit_completion_without_promoting_progress() -> ( diff --git a/tests/test_native_steward_reports.py b/tests/test_native_steward_reports.py index bccf9c20eb..99f7241f48 100644 --- a/tests/test_native_steward_reports.py +++ b/tests/test_native_steward_reports.py @@ -2365,17 +2365,22 @@ def test_saved_report_requires_real_original_closeout_then_its_own_fresh_native_ from loopx.capabilities.native_chat.steward_reports import ManagedSessionReports from loopx.capabilities.native_chat.steward_turn import StewardTurnHost from loopx.control_plane.quota.execution_preparation import RefreshTaskLeaseUnavailable - from loopx.control_plane.quota.settlement import ReceiptBoundReplayPhase, read_heartbeat_settlement + from loopx.control_plane.goals.goal_ref_validation import exact_goal_ref + from loopx.control_plane.projects.registry_codec import SOURCE_SESSION_PROFILE_ID + from loopx.control_plane.quota.settlement import ( + ReceiptBoundReplayPhase, SettlementFailureKind, read_heartbeat_settlement, + ) + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path from loopx.control_plane.turn_driver import executor, host_candidate from loopx.control_plane.turn_driver.journal_store import ( - find_loopx_turn_key_by_settlement_identity, load_turn_journal, turn_journal_path, + find_loopx_turn_key_by_settlement_identity, load_loopx_turn_plan_from_journal, + load_turn_journal, turn_journal_path, ) session, reports, _, _ = configured_local_report(steward, tmp_path, monkeypatch, provider) store, runtime, bot, _, _, capture, _ = steward sid = session["session_id"] actor = reports.selected(sid)["actor"] - goal_ref = {"goal_id": actor["goal_id"], "goal_instance_id": actor["goal_instance_id"]} install_model(steward, tmp_path, sid) runtime.project_contexts.conversation_bindings = None with monkeypatch.context() as old_parser: @@ -2383,10 +2388,26 @@ def test_saved_report_requires_real_original_closeout_then_its_own_fresh_native_ first = finish(runtime, pump_steward_monitors(runtime)[0]) native, request = first["response"]["native_loopx"], first["loopx_request"] assert native["status"] == "stopped" and native["result_kind"] == "wait" + actor = request["actor"] _, window = reports.current_window(sid) folder = attempt_directory(reports.root, window) original_journal = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"]) original_journal_bytes = original_journal.read_bytes() + original_plan = load_loopx_turn_plan_from_journal(reports.root, + goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=native["resume_turn_key"]) + goal_ref = original_plan.get("goal_ref") + registry = json.loads(runtime.registry_path.read_bytes()) + registered_goal = next(goal for goal in registry["goals"] if goal["id"] == actor["goal_id"]) + if goal_ref is None: + # This fixture's registered bounded alias has no incarnation. A + # missing exact Source reference must never take this alias branch. + assert "goal_ref" not in original_plan + assert registry.get("profile_id") != SOURCE_SESSION_PROFILE_ID + assert not registered_goal.get("goal_instance_id") + assert actor.get("goal_instance_id") is None + else: + assert goal_ref == exact_goal_ref(actor["goal_id"], goal_ref["goal_instance_id"]) + assert actor["goal_instance_id"] == goal_ref["goal_instance_id"] saved_instance = request["turn_instance_id"] + ".saved-candidate" def readback(turn_instance): @@ -2407,15 +2428,33 @@ def readback(turn_instance): assert original.original_prepared_receipt.get("source_event_id") is None task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == request["todo_id"]) calls, writes = capture.read_bytes(), list(bot.writes) + route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] + p1 = native_command(runtime, ["task-lease", "inspect", *route])["lease"] + event_path = rollout_event_log_path(reports.root, actor["goal_id"]) + events_before = event_path.read_bytes() with pytest.raises(RefreshTaskLeaseUnavailable): StewardTurnHost(runtime).recover_report_settlement(actor, window, task) - assert readback(saved_instance) is None + absent = readback(saved_instance) + assert absent is not None and absent.identity.value is None + assert absent.identity.failure is not None and absent.identity.failure.kind is SettlementFailureKind.RECEIPT_MISSING + assert absent.progress["state"] == "identity_required" + assert absent.progress["next_step"] == "validation" + assert absent.heartbeat_receipt is None and absent.original_prepared_receipt is None + assert absent.prepared_lease is None and absent.prepared_lease_required is False + assert absent.resume_lease_qualification is None and absent.native_child_admission is None + assert absent.writeback_run is None and absent.writeback_event is None + assert absent.spend_run is None and absent.spend_event is None and absent.completion_event is None + assert event_path.read_bytes() == events_before + assert all(event.get("run_id") != saved_instance + and (event.get("details") or {}).get("turn_instance_id") != saved_instance + for event in load_rollout_events(event_path)) + assert find_loopx_turn_key_by_settlement_identity(reports.root, goal_id=actor["goal_id"], + agent_id=actor["agent_id"], todo_id=request["todo_id"], turn_instance_id=saved_instance) is None + assert native_command(runtime, ["task-lease", "inspect", *route])["lease"] == p1 assert not (folder / "private-generation.json").exists() assert original_journal.read_bytes() == original_journal_bytes assert capture.read_bytes() == calls and bot.writes == writes - route = ["--goal-id", actor["goal_id"], "--todo-id", request["todo_id"]] - p1 = native_command(runtime, ["task-lease", "inspect", *route])["lease"] assert original.prepared_lease == { "prepared_lease_key_sha256": sha256(p1["idempotency_key"].encode()).hexdigest(), "prepared_lease_version": p1["version"], "prepared_lease_epoch": p1["lease_epoch"], @@ -2430,7 +2469,8 @@ def readback(turn_instance): # The supported administrative owner explicitly settles only P1's debt. # P2's control proof does not by itself authorize the saved business Run. closed = cli(runtime, "refresh-state", "--goal-id", actor["goal_id"], - "--goal-instance-id", actor["goal_instance_id"], "--agent-id", actor["agent_id"], + *(["--goal-instance-id", goal_ref["goal_instance_id"]] if goal_ref is not None else []), + "--agent-id", actor["agent_id"], "--todo-id", request["todo_id"], "--turn-instance-id", request["turn_instance_id"], "--execution-loss-closeout", "--classification", "execution_loss_closeout", "--delivery-batch-scale", "single_surface", "--delivery-outcome", "outcome_gap", @@ -2488,6 +2528,8 @@ def interrupt_saved_checkpoint(path, journal, **kwargs): agent_id=actor["agent_id"], todo_id=request["todo_id"], turn_instance_id=saved_instance) saved_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=saved_key) saved_journal = load_turn_journal(saved_path) + assert ("goal_ref" in saved_journal["plan"]) == ("goal_ref" in original_plan) + assert saved_journal["plan"].get("goal_ref") == goal_ref if saved_lease_loss is not None: from dataclasses import replace from loopx.capabilities.native_chat import steward_commands @@ -2548,11 +2590,12 @@ def goal_bytes(): runtime._steward_monitor_next_check.clear() assert not pump_steward_monitors(runtime) assert readonly_reads and all(row["turn_instance_id"] == saved_instance - and row["goal_ref"] == saved_journal["plan"]["goal_ref"] for row in readonly_reads) + and row["goal_ref"] == saved_journal["plan"].get("goal_ref") for row in readonly_reads) if recovery_attachment == "exact": gate = payload["settlement_recovery"] assert gate["settlement_identity"] == saved.identity.value.as_dict() - assert gate["goal_ref"] == saved_journal["plan"]["goal_ref"] + assert ("goal_ref" in gate) == ("goal_ref" in saved_journal["plan"]) + assert gate.get("goal_ref") == saved_journal["plan"].get("goal_ref") assert gate["next_cli_actions_argv"][0] == ["turn", "inspect-journal", "--goal-id", actor["goal_id"], "--agent-id", actor["agent_id"], "--turn-key", saved_key] assert gate["original_settlement_readback"]["prepared_lease"] == saved.prepared_lease From 28c7992ba3fc0c6c8d687169887a4ed5f80fb506 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 20:28:50 +0800 Subject: [PATCH 12/12] fix(recovery): resume validated report checkpoints through strict inspection Bind compact journal inspection to the recorded result and request identity, preserve full executor and lease validation, and accept strict paired optional fields in the Python transport. Cover real TS-to-Python decoding, malformed carriers and saved report recovery; update both RFC checkpoints. Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- .../rfcs/monorepo-distribution-split-v0.md | 8 + .../monorepo-distribution-split-v0.zh-CN.md | 7 + .../collaboration/steward_managed_mode.ts | 26 +- .../control_plane/turn_driver/turn_journal.ts | 61 +++++ .../turn_driver/turn_journal_runtime.py | 81 +++++- .../control_plane_ts/steward_reports.test.ts | 146 ++++++++++ tests/control_plane_ts/turn_journal.test.ts | 92 +++++++ tests/test_loopx_turn_journal_inspection.py | 251 ++++++++++++++++++ 8 files changed, 670 insertions(+), 2 deletions(-) diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 22fc4b196e..04c2ae1b9e 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4861,6 +4861,14 @@ view shows its independently checked result in the same conversation. positive prepared admission may explain the existing explicit administrative closeout and its separate current-control-writer requirement; bounded legacy evidence supports inspection only. +- **Validated report checkpoint:** an open Todo with a consistent in-progress + journal, complete validation prefix and cached material result may continue + only its original durable writeback without a model. Compact inspection + binds the Goal, Agent, Todo, Turn and inspected key to the packet headers + and recorded passed task receipt. The executor's full result validation and + original saved-generation lease qualification still precede business effects. + Python transport accepts the old inspection shape or the strict paired + optional fields; it neither decides continuation nor supplies authority. - **Fresh native admission:** a new native Turn records its own genuine typed work admission before Host execution. The shared owner checks prior debt before preparation and rereads it with current CAS at publication. A saved diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index adf0265e8b..0b3496c024 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -3753,6 +3753,13 @@ workspace 批准和脚本模型工作。桌面视图在采用检查后显示原 health packet;检查不授予 execution 或行政写权限。现代 positive prepared admission 可以说明既有显式行政收尾及其独立的当前 control-writer 要求; 有限 legacy 证据仅支持检查。 +- **已验证报告 checkpoint:**开放 Todo 的 in-progress journal 若一致、具备 + 完整 validation prefix 和缓存 material result,仅可在不调用模型的情况下 + 继续原 durable writeback。compact inspection 将 Goal、Agent、Todo、Turn + 与被检查 key 绑定到 packet header 和已记录的 passed task receipt;业务 + effect 前仍须通过 executor 完整结果验证和原 saved-generation lease 资格。 + Python transport 仅接受旧 inspection shape 或严格成对的可选字段,不决定 + continuation,也不提供权限。 - **首次 native 准入:**新 native Turn 在 Host 执行前记录自己的真实 typed work admission。共享 owner 在 preparation 前检查原债务,并在 publication 时连同当前 CAS 复读。saved candidate 仅提供内容证据,不能借原 Turn 的 diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index fe40887e83..1645a97587 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -535,10 +535,34 @@ function reportSettlementOnly(params: JsonObject, request: JsonObject, todo: Jso && [request.turn_instance_id, `${String(request.turn_instance_id)}.saved-candidate`].includes(String(todo.completion_turn_key)) && (decision.action === "continue" && ["durable_writeback", "quota_spend", "scheduler_apply", "scheduler_ack"].includes(String(decision.resume_from)) || decision.action === "return_existing" && recovery.journal_status === "committed"); + const cachedResult = jsonObject(recovery.cached_validation_state); + const taskValidation = jsonObject(cachedResult?.task_validation); + const recorded = jsonObject(recovery.recorded_effects); + const actor = requireJsonObject(request.actor, "original report actor"); + // The journal owner attests only the cached packet headers and recorded + // passed task receipt. The executor's full result validation and this Run's + // independent execution-lease qualification still precede every effect. + const pendingValidatedGeneration = todo.status === "open" && recovery.journal_status === "in_progress" + && decision.action === "continue" && decision.can_continue === true + && decision.resume_from === "durable_writeback" && decision.reinvoke_host === false + && Array.isArray(recovery.completed_phases) && recovery.completed_phases.length === 3 + && recovery.completed_phases[0] === "host_execute" && recovery.completed_phases[1] === "typed_result" + && recovery.completed_phases[2] === "validation" + && typeof recovery.turn_key === "string" && ENVELOPED_SHA256_PATTERN.test(recovery.turn_key) + && cachedResult !== null && cachedResult.turn_key === recovery.turn_key + && cachedResult.goal_id === actor.goal_id && cachedResult.agent_id === actor.agent_id + && cachedResult.todo_id === request.todo_id && cachedResult.todo_id === todo.todo_id + && [request.turn_instance_id, `${String(request.turn_instance_id)}.saved-candidate`].includes(cachedResult.turn_instance_id) + && typeof cachedResult.result_kind === "string" + && ["validated_progress", "validated_completion"].includes(cachedResult.result_kind) + && taskValidation?.schema_version === "loopx_turn_task_validation_v0" + && taskValidation.ok === true && taskValidation.status === "passed" + && recorded?.host_invoked === true && recorded.state_written === false + && recorded.quota_spent === false && recorded.scheduler_acknowledged === false; const accepted = recovery.schema_version === "loopx_turn_journal_inspection_v1" && recovery.ok === true && recovery.goal_matches === true && recovery.owner_matches === true && recovery.turn_key_matches === true && recovery.journal_consistent === true && decision.reinvoke_host === false - && (cachedGeneration || completedGeneration); + && (cachedGeneration || completedGeneration || pendingValidatedGeneration); if (!accepted || params.preparing_model_context === true) { throw new EffectRuntimeRequestError("original report recovery requires validated remaining effects without a model"); } diff --git a/loopx/control_plane/turn_driver/turn_journal.ts b/loopx/control_plane/turn_driver/turn_journal.ts index 8e0b5b0578..6fb433e624 100644 --- a/loopx/control_plane/turn_driver/turn_journal.ts +++ b/loopx/control_plane/turn_driver/turn_journal.ts @@ -18,6 +18,7 @@ import { recordedTurnEffects, type RecordedTurnEffects } from "./turn_journal_ef import { selectedWorkTodoId } from "../work_items/interaction_contract.ts"; import { turnPlanWorkSource, turnEnvelopeScopedSettlementIdentity } from "../quota/turn_envelope.ts"; import { sourceNumberTokens } from "../runtime_decode.ts"; +import { TURN_RESULT_KINDS, type TurnResultKind } from "./turn_contract_generated.ts"; export const TURN_JOURNAL_INSPECTION_SCHEMA_VERSION = "loopx_turn_journal_inspection_v1"; @@ -101,9 +102,27 @@ export interface TurnJournalInspection { recovery_decision: TurnRecoveryDecision; last_recovery: TurnRecoveryAudit | null; recorded_effects: RecordedTurnEffects; + turn_key?: string; + cached_validation_state?: CachedTurnValidationState; effects: []; } +/** Structural cache/receipt readback, not full Host-result validation or a lease. + * The executor still revalidates the cached packet before remaining effects. */ +export interface CachedTurnValidationState { + goal_id: string; + agent_id: string; + todo_id: string | null; + turn_instance_id: string; + turn_key: string; + result_kind: TurnResultKind; + task_validation: { + schema_version: "loopx_turn_task_validation_v0"; + ok: true; + status: "passed"; + }; +} + export interface TurnJournalEffectContext { replay_legal: boolean; goal_matches: boolean; @@ -118,6 +137,8 @@ export interface TurnJournalEffectContext { recovery_decision: TurnRecoveryDecision; last_recovery: TurnRecoveryAudit | null; recorded_effects: RecordedTurnEffects; + turn_key?: string; + cached_validation_state?: CachedTurnValidationState; } // Replay has its own verdict. It is not a quota decision and must not manufacture @@ -170,6 +191,40 @@ function asObject(value: unknown): JsonObject { : {}; } +function cachedValidationState( + request: TurnJournalInspectionRequest, journal: JsonObject, journalConsistent: boolean, + completedPhases: readonly string[], +): CachedTurnValidationState | undefined { + if (!journalConsistent || journal.status !== "in_progress" + || JSON.stringify(completedPhases) !== JSON.stringify(transactionPhases.slice(0, 3))) return undefined; + const result = asObject(journal.host_result); + const validation = asObject(journal.task_validation); + const kind = result.result_kind; + // These are the existing result packet and independent validation receipt + // codecs. Do not infer a cached result or a passed receipt from phases alone. + if (result.schema_version !== "loopx_turn_result_v0" || result.turn_key !== request.turn_key + || typeof kind !== "string" || !TURN_RESULT_KINDS.includes(kind as TurnResultKind) + || !["validated_progress", "validated_completion", "repair_required", "replan_required"].includes(kind) + || journal.result_kind !== kind + || JSON.stringify(result.completed_phases) !== JSON.stringify(transactionPhases.slice(0, 2)) + || journal.validation_stage !== "task_postcondition" + || validation.schema_version !== "loopx_turn_task_validation_v0" + || validation.ok !== true || validation.status !== "passed" + || validation.recovery_kind !== null || ![null, 0].includes(validation.exit_code as null | number) + || !Array.isArray(validation.errors) || validation.errors.length !== 0 + || typeof validation.validator_kind !== "string" || !validation.validator_kind.trim() + || validation.validator_kind.length > 80 + || typeof validation.summary !== "string" || !validation.summary.trim() || validation.summary.length > 240) { + return undefined; + } + const identity = settlementIdentityFromPlan(asObject(asObject(journal.plan).transaction)).value; + if (identity === null) return undefined; + return {goal_id: request.goal_id, agent_id: request.agent_id, todo_id: identity.todo_id, + turn_instance_id: identity.turn_instance_id, + turn_key: request.turn_key, result_kind: kind as TurnResultKind, + task_validation: {schema_version: "loopx_turn_task_validation_v0", ok: true, status: "passed"}}; +} + export function parseTurnJournalGoalBinding( journal: JsonObject, ): TurnJournalGoalBinding { @@ -678,6 +733,7 @@ export function interpretTurnJournalEffect( journalStatus, journalConsistent, ); + const cachedResult = cachedValidationState(request, journal, journalConsistent, completedPhases); return { request: { kind: "turn_journal", @@ -701,6 +757,7 @@ export function interpretTurnJournalEffect( recorded_effects: recordedTurnEffects( journal, completedPhases, lineageConsistent, attemptViolation === null, ), + ...(cachedResult === undefined ? {} : {turn_key: request.turn_key, cached_validation_state: cachedResult}), }, }, interpretation: { @@ -754,6 +811,10 @@ export function projectTurnJournalInspection( recovery_decision: context.recovery_decision, last_recovery: context.last_recovery, recorded_effects: context.recorded_effects, + ...(context.cached_validation_state === undefined ? {} : { + turn_key: context.turn_key, + cached_validation_state: context.cached_validation_state, + }), effects: [], }; } diff --git a/loopx/control_plane/turn_driver/turn_journal_runtime.py b/loopx/control_plane/turn_driver/turn_journal_runtime.py index 0126337684..747fd60297 100644 --- a/loopx/control_plane/turn_driver/turn_journal_runtime.py +++ b/loopx/control_plane/turn_driver/turn_journal_runtime.py @@ -3,7 +3,11 @@ from collections.abc import Mapping from typing import Any +from ...turn_identity import normalize_turn_instance_id +from ..content_digest import ENVELOPED_SHA256_PATTERN from ..effect_runtime import effect_runtime_result, source_number_tokens +from ..todos.contract import normalize_todo_id +from .turn_contract_generated import LoopXTurnResultKind TURN_JOURNAL_INTERPRETATION_REQUEST_SCHEMA_VERSION = ( @@ -100,6 +104,72 @@ def _validate_recovery_audit(value: Any) -> bool: ) +def _validate_cached_validation_state( + payload: Mapping[str, Any], + journal: Mapping[str, Any], + *, + goal_id: str, + agent_id: str, + turn_key: str, +) -> bool: + """Check the compact wire carrier; TS retains continuation eligibility.""" + + cached = payload.get("cached_validation_state") + if not isinstance(cached, dict) or set(cached) != { + "goal_id", "agent_id", "todo_id", "turn_instance_id", "turn_key", + "result_kind", "task_validation", + }: + return False + receipt = cached.get("task_validation") + if not isinstance(receipt, dict) or set(receipt) != { + "schema_version", "ok", "status", + }: + return False + if ( + receipt.get("schema_version") != "loopx_turn_task_validation_v0" + or receipt.get("ok") is not True + or receipt.get("status") != "passed" + or any(not isinstance(cached.get(key), str) for key in ( + "goal_id", "agent_id", "turn_instance_id", "turn_key", "result_kind", + )) + or any(not cached[key].strip() for key in ("goal_id", "agent_id")) + or not isinstance(payload.get("turn_key"), str) + or ENVELOPED_SHA256_PATTERN.fullmatch(payload["turn_key"]) is None + or (cached["todo_id"] is not None and ( + not isinstance(cached["todo_id"], str) + or normalize_todo_id(cached["todo_id"]) != cached["todo_id"] + )) + ): + return False + try: + if normalize_turn_instance_id(cached["turn_instance_id"]) != cached["turn_instance_id"]: + return False + LoopXTurnResultKind(cached["result_kind"]) + except ValueError: + return False + + plan = journal.get("plan") + transaction = plan.get("transaction") if isinstance(plan, Mapping) else None + settlement = transaction.get("settlement_plan") if isinstance(transaction, Mapping) else None + identity = settlement.get("identity") if isinstance(settlement, Mapping) else None + result = journal.get("host_result") + if ( + not isinstance(transaction, Mapping) + or not isinstance(identity, Mapping) + or not isinstance(result, Mapping) + ): + return False + return bool( + cached["goal_id"] == goal_id == journal.get("goal_id") == identity.get("goal_id") + and cached["agent_id"] == agent_id == identity.get("agent_id") + and cached["todo_id"] == identity.get("todo_id") + and cached["turn_instance_id"] == identity.get("turn_instance_id") + and payload["turn_key"] == cached["turn_key"] == turn_key + == journal.get("turn_key") == transaction.get("turn_key") == result.get("turn_key") + and cached["result_kind"] == journal.get("result_kind") == result.get("result_kind") + ) + + def interpret_turn_journal_projection( journal: Mapping[str, Any], *, @@ -128,7 +198,10 @@ def interpret_turn_journal_projection( if tokens: request["source_number_tokens"] = tokens payload = effect_runtime_result("turn_journal.inspect", request) - if not isinstance(payload, dict) or set(payload) != _PROJECTION_KEYS: + if not isinstance(payload, dict) or set(payload) not in ( + _PROJECTION_KEYS, + _PROJECTION_KEYS | {"turn_key", "cached_validation_state"}, + ): raise RuntimeError( "TypeScript Turn-journal inspection projection shape mismatch" ) @@ -159,6 +232,12 @@ def interpret_turn_journal_projection( raise RuntimeError( "TypeScript Turn-journal inspection projection type mismatch" ) + if "cached_validation_state" in payload and not _validate_cached_validation_state( + payload, journal, goal_id=goal_id, agent_id=agent_id, turn_key=turn_key, + ): + raise RuntimeError( + "TypeScript Turn-journal inspection cached validation state mismatch" + ) if payload.get("effects") != []: raise RuntimeError("Turn-journal inspection must remain effect-free") return payload diff --git a/tests/control_plane_ts/steward_reports.test.ts b/tests/control_plane_ts/steward_reports.test.ts index cb762d67c3..8f2bfc6d4d 100644 --- a/tests/control_plane_ts/steward_reports.test.ts +++ b/tests/control_plane_ts/steward_reports.test.ts @@ -3,6 +3,152 @@ import test from "node:test"; import {planStewardReport} from "../../loopx/control_plane/collaboration/steward_reports.ts"; import {admitManagedSessionIdentity, admitManagedSessionTurn, admitStewardManagedTurn, bindStewardManagedActor, observeManagedWork} from "../../loopx/control_plane/collaboration/steward_managed_mode.ts"; import {resolveConversationScope} from "../../loopx/control_plane/collaboration/conversation_scope.ts"; +import {interpretTurnJournal, type TurnJournalInspectionRequest} from "../../loopx/control_plane/turn_driver/turn_journal.ts"; + +type JsonObject = Record; + +function localReportRecovery() { + const goal = {goal_id: "project", goal_instance_id: null, registered_agents: ["writer"], workspace_path: "/workspace"}; + const session = {goal_id: goal.goal_id, goal_instance_id: null, session_id: "a".repeat(32), + channel_id: "agent.writer", session_mode: "managed_runtime", managed_agent_id: "writer", agent_id: "codex", + executor_endpoint_id: "codex", adapter_kind: "codex_app_server", managed_workspace_path: goal.workspace_path}; + const actor = {schema_version: "loopx_managed_session_actor_v0", goal_id: goal.goal_id, goal_instance_id: null, + session_id: session.session_id, agent_id: "writer", workspace_path: goal.workspace_path}; + const request = {operation: "native_turn", host: "codex-app-server", work_kind: "report", actor, + turn_instance_id: "managed-session.report.original", todo_id: "todo_report", validation_command: null, + report_window_id: "original-window", report_subscription_revision: "original-subscription"}; + const window = {goal_id: goal.goal_id, agent_id: actor.agent_id, window_id: request.report_window_id, + subscription_revision: request.report_subscription_revision}; + const todo = {todo_id: request.todo_id, target_key: window.window_id, action_kind: "generate_periodic_report", + claimed_by: actor.agent_id, status: "open"}; + return {actor, request, todo, params: {session, goal, request, goal_active: true, queued_turn_count: 0, + report_window: window, report_todo: todo, report_subscription: {enabled: true, profile_preset: "weekly-progress", + route_ref: `managed-session:${session.session_id}`, effective_revision: request.report_subscription_revision}}}; +} + +function portfolioReportRecovery() { + const p = original(); + const window = {window_id: "cadence_original", subscription_revision: "sha256:original", + goal_id: p.actor.goal_id, agent_id: p.actor.agent_id}; + const request = {operation: "native_turn", host: "codex-app-server", actor: p.actor, + turn_instance_id: "report.cadence_original", todo_id: "todo_report", validation_command: ["validate"], + work_kind: "report", report_window_id: window.window_id, report_subscription_revision: window.subscription_revision}; + const todo = {todo_id: request.todo_id, target_key: window.window_id, action_kind: "generate_periodic_report", + claimed_by: p.actor.agent_id, status: "open"}; + return {actor: p.actor, request, todo, params: {...p, request, report_window: window, report_todo: todo, + report_subscription: {enabled: true, profile_preset: "portfolio-progress", + route_ref: `steward-private:${p.binding.binding_id}:${p.source}`, effective_revision: window.subscription_revision}}}; +} + +function pendingReportJournal(actor: JsonObject, todoId: string, turnInstance: string): TurnJournalInspectionRequest { + const goalId = actor.goal_id as string, agentId = actor.agent_id as string; + const key = "sha256:" + "d".repeat(64); + return {schema_version: "loopx_turn_journal_interpretation_request_v0", + goal_id: goalId, agent_id: agentId, turn_key: key, + journal: {schema_version: "loopx_turn_journal_v0", goal_id: goalId, turn_key: key, + status: "in_progress", completed_phases: ["host_execute", "typed_result", "validation"], + host_attempt_count: 1, result_kind: "validated_completion", validation_stage: "task_postcondition", + plan: {turn_envelope: {goal_id: goalId, agent_id: agentId, action: {selected_todo: {todo_id: todoId}}}, + transaction: {turn_key: key, turn_instance_id: turnInstance, + settlement_plan: {schema_version: "quota_settlement_plan_v1", identity: { + schema_version: "quota_settlement_identity_v0", effect_id: `${goalId}:${agentId}:${todoId}:${turnInstance}`, + goal_id: goalId, agent_id: agentId, todo_id: todoId, turn_instance_id: turnInstance}}}}, + host_result: {schema_version: "loopx_turn_result_v0", turn_key: key, result_kind: "validated_completion", + completed_phases: ["host_execute", "typed_result"], classification: "periodic_report_generation", + recommended_action: "Retain the original report.", next_action: "Read back the original report.", + delivery_batch_scale: "single_surface", delivery_outcome: "outcome_progress", + vision_unchanged_reason: "Keep the original reporting goal.", path_delta_mode: "unchanged", + summary: "private cached report body"}, + task_validation: {schema_version: "loopx_turn_task_validation_v0", ok: true, status: "passed", + validator_kind: "callback", summary: "private validation details", recovery_kind: null, exit_code: null, errors: []}}}; +} + +for (const host of [ + {name: "original managed Session", fixture: localReportRecovery, admit: admitManagedSessionTurn}, + {name: "original portfolio Session", fixture: portfolioReportRecovery, admit: admitStewardManagedTurn}, +]) { + test(`${host.name} admits only the inspected validated checkpoint's no-model continuation`, () => { + const fixture = host.fixture(); + for (const instance of [fixture.request.turn_instance_id, fixture.request.turn_instance_id + ".saved-candidate"]) { + const inspection = interpretTurnJournal(pendingReportJournal(fixture.actor, fixture.todo.todo_id, instance)); + const input = {...fixture.params, report_settlement_recovery: inspection}; + const before = structuredClone(input); + assert.equal(inspection.cached_validation_state?.turn_instance_id, instance); + assert.deepEqual(host.admit(input).actor, fixture.actor); + assert.equal(JSON.stringify(inspection).includes("private cached report body"), false); + assert.equal(JSON.stringify(inspection).includes("private validation details"), false); + assert.equal("current_task_lease_proof" in host.admit(input), false); + assert.deepEqual(input, before); + } + }); + + test(`${host.name} rejects mismatched, incomplete, mutating or model checkpoint recovery`, () => { + const fixture = host.fixture(); + const inspection = interpretTurnJournal(pendingReportJournal(fixture.actor, fixture.todo.todo_id, + fixture.request.turn_instance_id + ".saved-candidate")); + const recovery = (input: JsonObject) => input.report_settlement_recovery as JsonObject; + const cached = (input: JsonObject) => recovery(input).cached_validation_state as JsonObject; + const decision = (input: JsonObject) => recovery(input).recovery_decision as JsonObject; + const effects = (input: JsonObject) => recovery(input).recorded_effects as JsonObject; + const invalid: Array<[string, (input: JsonObject) => void]> = [ + ["wrong Goal", input => { cached(input).goal_id = "foreign-goal"; }], + ["wrong Agent", input => { cached(input).agent_id = "foreign-agent"; }], + ["wrong Todo", input => { cached(input).todo_id = "todo_other"; }], + ["foreign native Turn", input => { cached(input).turn_instance_id = "foreign-turn"; }], + ["missing inspected key", input => { delete recovery(input).turn_key; }], + ["different inspected key", input => { recovery(input).turn_key = "sha256:" + "a".repeat(64); }], + ["different packet key", input => { cached(input).turn_key = "sha256:" + "a".repeat(64); }], + ["missing cache state", input => { delete recovery(input).cached_validation_state; }], + ["nonmaterial packet", input => { cached(input).result_kind = "wait"; }], + ["non-string material kind", input => { cached(input).result_kind = ["validated_completion"]; }], + ["missing validation", input => { delete cached(input).task_validation; }], + ["unpassed validation", input => { (cached(input).task_validation as JsonObject).status = "not_required"; }], + ["invalid receipt", input => { (cached(input).task_validation as JsonObject).ok = false; }], + ["wrong receipt schema", input => { (cached(input).task_validation as JsonObject).schema_version = "unverified"; }], + ["Goal mismatch", input => { recovery(input).goal_matches = false; }], + ["owner mismatch", input => { recovery(input).owner_matches = false; }], + ["key mismatch", input => { recovery(input).turn_key_matches = false; }], + ["inconsistent journal", input => { recovery(input).journal_consistent = false; }], + ["wrong journal status", input => { recovery(input).journal_status = "failed"; }], + ["missing phase prefix", input => { delete recovery(input).completed_phases; }], + ["wrong phase", input => { recovery(input).completed_phases = ["host_execute", "validation", "typed_result"]; }], + ["already written phase", input => { recovery(input).completed_phases = ["host_execute", "typed_result", "validation", "durable_writeback"]; }], + ["cannot continue", input => { decision(input).can_continue = false; }], + ["missing continue fact", input => { delete decision(input).can_continue; }], + ["different resume phase", input => { decision(input).resume_from = "validation"; }], + ["different recovery action", input => { decision(input).action = "return_existing"; }], + ["Host replay", input => { decision(input).reinvoke_host = true; }], + ["Host not observed", input => { effects(input).host_invoked = false; }], + ["Host unknown", input => { effects(input).host_invoked = null; }], + ["state already written", input => { effects(input).state_written = true; }], + ["state unknown", input => { effects(input).state_written = null; }], + ["quota already spent", input => { effects(input).quota_spent = true; }], + ["quota unknown", input => { effects(input).quota_spent = null; }], + ["scheduler acknowledged", input => { effects(input).scheduler_acknowledged = true; }], + ["scheduler unknown", input => { effects(input).scheduler_acknowledged = null; }], + ["model context", input => { input.preparing_model_context = true; }], + ]; + for (const [name, change] of invalid) { + const input: JsonObject = structuredClone({...fixture.params, report_settlement_recovery: inspection}); + change(input); + assert.throws(() => host.admit(input), name); + } + for (const [name, change] of [ + ["missing cached packet", (journal: JsonObject) => { delete journal.host_result; }], + ["invalid packet schema", (journal: JsonObject) => { (journal.host_result as JsonObject).schema_version = "unverified"; }], + ["invalid packet kind", (journal: JsonObject) => { (journal.host_result as JsonObject).result_kind = "unverified"; }], + ["different packet key", (journal: JsonObject) => { (journal.host_result as JsonObject).turn_key = "sha256:" + "a".repeat(64); }], + ["missing actual validation", (journal: JsonObject) => { delete journal.task_validation; }], + ["contradictory actual validation", (journal: JsonObject) => { (journal.task_validation as JsonObject).errors = ["not passed"]; }], + ] as Array<[string, (journal: JsonObject) => void]>) { + const raw = pendingReportJournal(fixture.actor, fixture.todo.todo_id, fixture.request.turn_instance_id); + change(raw.journal); + const corrupted = interpretTurnJournal(raw); + assert.equal(corrupted.cached_validation_state, undefined, name); + assert.throws(() => host.admit({...fixture.params, report_settlement_recovery: corrupted}), name); + } + }); +} test("a managed work identity is registration-bound and never chosen from the executor or legacy history", () => { const goal = {goal_id: "project", goal_instance_id: "original", registered_agents: ["writer", "reviewer"], workspace_path: "/workspace"}; diff --git a/tests/control_plane_ts/turn_journal.test.ts b/tests/control_plane_ts/turn_journal.test.ts index 6582b60d82..3c52df93cf 100644 --- a/tests/control_plane_ts/turn_journal.test.ts +++ b/tests/control_plane_ts/turn_journal.test.ts @@ -65,6 +65,98 @@ function request(status = "committed"): TurnJournalInspectionRequest { }; } +function cachedResultRequest(): TurnJournalInspectionRequest { + const input = request("in_progress"); + input.journal.completed_phases = ["host_execute", "typed_result", "validation"]; + input.journal.host_attempt_count = 1; + input.journal.result_kind = "validated_completion"; + input.journal.validation_stage = "task_postcondition"; + input.journal.host_result = {schema_version: "loopx_turn_result_v0", turn_key: turnKey, + result_kind: "validated_completion", completed_phases: ["host_execute", "typed_result"], + classification: "periodic_report_generation", recommended_action: "Retain the original report.", + next_action: "Read back the original report.", delivery_batch_scale: "single_surface", + delivery_outcome: "outcome_progress", vision_unchanged_reason: "Keep the original reporting goal.", + path_delta_mode: "unchanged", summary: "do-not-expose"}; + input.journal.task_validation = {schema_version: "loopx_turn_task_validation_v0", ok: true, + status: "passed", validator_kind: "callback", summary: "do-not-expose", + recovery_kind: null, exit_code: null, errors: []}; + delete input.journal.receipt; + return input; +} + +test("a recorded checkpoint projects bounded cache and task receipt facts without execution authority", () => { + const input = cachedResultRequest(); + const before = structuredClone(input); + const result = interpretTurnJournal(input); + assert.equal(result.journal_consistent, true); + assert.equal(result.replay_legal, false); // No terminal replay is fabricated. + assert.equal(result.turn_key, turnKey); + assert.deepEqual(result.cached_validation_state, {goal_id: "fixture-goal", agent_id: "fixture-agent", todo_id: todoId, + turn_instance_id: turnKey, + turn_key: turnKey, result_kind: "validated_completion", + task_validation: {schema_version: "loopx_turn_task_validation_v0", ok: true, status: "passed"}}); + assert.equal(result.recovery_decision.action, "continue"); + assert.equal(result.recovery_decision.can_continue, true); + assert.equal(result.recovery_decision.resume_from, "durable_writeback"); + assert.equal(result.recovery_decision.reinvoke_host, false); + assert.deepEqual(result.recorded_effects, {host_invoked: true, state_written: false, + quota_spent: false, scheduler_acknowledged: false}); + assert.deepEqual(result.effects, []); + assert.equal(JSON.stringify(result).includes("do-not-expose"), false); + assert.deepEqual(input, before); + assert.equal("current_task_lease_proof" in result, false); + assert.equal("native_work_admission" in result, false); + assert.ok(Buffer.byteLength(JSON.stringify(result.cached_validation_state)) < 400); +}); + +test("cache readback cannot be inferred from a phase prefix or corrupt packet and task receipt", () => { + const result = (input: TurnJournalInspectionRequest) => input.journal.host_result as Record; + const validation = (input: TurnJournalInspectionRequest) => input.journal.task_validation as Record; + const invalid: Array<[string, (input: TurnJournalInspectionRequest) => void]> = [ + ["foreign Goal", input => { input.goal_id = "other-goal"; }], + ["foreign Agent", input => { input.agent_id = "other-agent"; }], + ["foreign key", input => { input.turn_key = "sha256:" + "b".repeat(64); }], + ["missing result", input => { delete input.journal.host_result; }], + ["missing result schema", input => { delete result(input).schema_version; }], + ["corrupt result schema", input => { result(input).schema_version = "unverified_result"; }], + ["wrong result key", input => { result(input).turn_key = "sha256:" + "b".repeat(64); }], + ["missing result kind", input => { delete result(input).result_kind; }], + ["unsupported result kind", input => { result(input).result_kind = "unverified_completion"; }], + ["stop result is not material", input => { result(input).result_kind = "wait"; input.journal.result_kind = "wait"; }], + ["contradictory kind", input => { input.journal.result_kind = "validated_progress"; }], + ["missing result phases", input => { delete result(input).completed_phases; }], + ["changed result phases", input => { result(input).completed_phases = ["typed_result", "host_execute"]; }], + ["extra result phase", input => { result(input).completed_phases = ["host_execute", "typed_result", "validation"]; }], + ["missing validation", input => { delete input.journal.task_validation; }], + ["corrupt validation schema", input => { validation(input).schema_version = "unverified_validation"; }], + ["validation not passed", input => { validation(input).status = "progress"; }], + ["failed validation", input => { validation(input).ok = false; }], + ["missing validation errors", input => { delete validation(input).errors; }], + ["validation errors", input => { validation(input).errors = ["not validated"]; }], + ["missing recovery fact", input => { delete validation(input).recovery_kind; }], + ["contradictory recovery", input => { validation(input).recovery_kind = "repair_required"; }], + ["missing exit fact", input => { delete validation(input).exit_code; }], + ["nonzero exit", input => { validation(input).exit_code = 1; }], + ["string exit", input => { validation(input).exit_code = "0"; }], + ["missing validator", input => { delete validation(input).validator_kind; }], + ["empty validator", input => { validation(input).validator_kind = " "; }], + ["missing validation summary", input => { delete validation(input).summary; }], + ["wrong validation stage", input => { input.journal.validation_stage = "host_result_contract"; }], + ["incomplete checkpoint", input => { input.journal.completed_phases = ["host_execute", "typed_result"]; }], + ["changed checkpoint", input => { input.journal.completed_phases = ["host_execute", "validation", "typed_result"]; }], + ]; + for (const [name, change] of invalid) { + const input = cachedResultRequest(); + change(input); + const before = structuredClone(input); + const inspected = interpretTurnJournal(input); + assert.equal(inspected.cached_validation_state, undefined, name); + assert.equal("turn_key" in inspected, false, name); + assert.deepEqual(inspected.effects, [], name); + assert.deepEqual(input, before, name); + } +}); + function failedHostRetryRequest({ attempt = 1, kind = "provider_capacity", diff --git a/tests/test_loopx_turn_journal_inspection.py b/tests/test_loopx_turn_journal_inspection.py index 85002841ff..883b1b6220 100644 --- a/tests/test_loopx_turn_journal_inspection.py +++ b/tests/test_loopx_turn_journal_inspection.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import copy import io import json from pathlib import Path @@ -20,6 +21,7 @@ TURN_KEY = "sha256:" + "a" * 64 +_MISSING_FIELD = object() COMPLETED_PHASES = [ "host_execute", "typed_result", @@ -68,6 +70,51 @@ def _journal(*, status: str = "committed") -> dict[str, Any]: } +def _cached_validation_journal( + *, + turn_instance_id: str = TURN_KEY, + result_kind: str = "validated_completion", +) -> dict[str, Any]: + journal = _journal(status="in_progress") + journal["completed_phases"] = COMPLETED_PHASES[:3] + journal["host_attempt_count"] = 1 + journal["result_kind"] = result_kind + journal["validation_stage"] = "task_postcondition" + transaction = journal["plan"]["transaction"] + transaction["turn_instance_id"] = turn_instance_id + identity = transaction["settlement_plan"]["identity"] + identity["turn_instance_id"] = turn_instance_id + identity["effect_id"] = ( + f"fixture-goal:fixture-agent:todo_fixture0001:{turn_instance_id}" + ) + journal["host_result"] = { + "schema_version": "loopx_turn_result_v0", + "turn_key": TURN_KEY, + "result_kind": result_kind, + "completed_phases": COMPLETED_PHASES[:2], + "classification": "fixture_completion", + "recommended_action": "Retain the original artifact.", + "next_action": "Read back the original artifact.", + "delivery_batch_scale": "single_surface", + "delivery_outcome": "outcome_progress", + "vision_unchanged_reason": "Keep the original goal.", + "path_delta_mode": "unchanged", + "summary": "do-not-expose", + } + journal["task_validation"] = { + "schema_version": "loopx_turn_task_validation_v0", + "ok": True, + "status": "passed", + "validator_kind": "callback", + "summary": "do-not-expose", + "recovery_kind": None, + "exit_code": None, + "errors": [], + } + del journal["receipt"] + return journal + + def _write_journal(runtime_root: Path, journal: dict[str, Any]) -> Path: path = runtime_root / "goals" / "fixture-goal" / "turns" / f"{'a' * 64}.json" path.parent.mkdir(parents=True) @@ -917,6 +964,210 @@ def test_typescript_runtime_rejects_malformed_projection_types( ) +@pytest.mark.parametrize("turn_instance_id", [TURN_KEY, "fixture-turn.saved-candidate"]) +@pytest.mark.parametrize("result_kind", [ + "validated_progress", "validated_completion", "repair_required", "replan_required", +]) +def test_real_typescript_checkpoint_crosses_python_inspection_adapter( + turn_instance_id: str, result_kind: str, +) -> None: + journal = _cached_validation_journal( + turn_instance_id=turn_instance_id, result_kind=result_kind, + ) + before = copy.deepcopy(journal) + + result = turn_journal_runtime.interpret_turn_journal_projection( + journal, goal_id="fixture-goal", agent_id="fixture-agent", turn_key=TURN_KEY, + ) + + assert result["turn_key"] == TURN_KEY + assert result["cached_validation_state"] == { + "goal_id": "fixture-goal", "agent_id": "fixture-agent", + "todo_id": "todo_fixture0001", "turn_instance_id": turn_instance_id, + "turn_key": TURN_KEY, "result_kind": result_kind, + "task_validation": { + "schema_version": "loopx_turn_task_validation_v0", "ok": True, "status": "passed", + }, + } + assert result["journal_consistent"] is True + assert result["replay_legal"] is False + assert result["recovery_decision"]["action"] == "continue" + assert result["recovery_decision"]["resume_from"] == "durable_writeback" + assert result["recovery_decision"]["reinvoke_host"] is False + assert result["recorded_effects"] == { + "host_invoked": True, "state_written": False, + "quota_spent": False, "scheduler_acknowledged": False, + } + assert result["effects"] == [] + assert "do-not-expose" not in json.dumps(result) + assert "current_task_lease_proof" not in result + assert "native_work_admission" not in result + assert journal == before + + +@pytest.mark.parametrize("status,count", [ + ("committed", 7), ("stopped", 0), ("in_progress", 2), ("failed", 0), +]) +def test_real_typescript_old_inspection_shape_retains_adapter_parity( + status: str, count: int, +) -> None: + journal = _journal(status=status) + journal["completed_phases"] = COMPLETED_PHASES[:count] + expected = turn_journal_runtime.effect_runtime_result("turn_journal.inspect", { + "schema_version": "loopx_turn_journal_interpretation_request_v0", + "journal": journal, "goal_id": "fixture-goal", "agent_id": "fixture-agent", + "turn_key": TURN_KEY, "retry_failed": False, "session_recovery_check": None, + }) + + result = turn_journal_runtime.interpret_turn_journal_projection( + journal, goal_id="fixture-goal", agent_id="fixture-agent", turn_key=TURN_KEY, + ) + + assert result == expected + assert "turn_key" not in result + assert "cached_validation_state" not in result + assert result["effects"] == [] + + +@pytest.fixture +def cached_validation_projection() -> tuple[dict[str, Any], dict[str, Any]]: + journal = _cached_validation_journal() + projection = turn_journal_runtime.interpret_turn_journal_projection( + journal, goal_id="fixture-goal", agent_id="fixture-agent", turn_key=TURN_KEY, + ) + assert "cached_validation_state" in projection + return journal, projection + + +@pytest.mark.parametrize("path,replacement", [ + (("turn_key",), _MISSING_FIELD), + (("cached_validation_state",), _MISSING_FIELD), + *[((key,), "unexpected") for key in ( + "body", "current_task_lease_proof", "native_work_admission", "execution_grant", + )], + (("cached_validation_state",), None), + (("cached_validation_state",), []), + *[(("cached_validation_state", key), _MISSING_FIELD) for key in ( + "goal_id", "agent_id", "todo_id", "turn_instance_id", "turn_key", + "result_kind", "task_validation", + )], + *[(("cached_validation_state", "task_validation", key), _MISSING_FIELD) + for key in ("schema_version", "ok", "status")], + *[(("cached_validation_state", key), "unexpected") for key in ( + "body", "current_task_lease_proof", "native_work_admission", "execution_grant", + )], + (("cached_validation_state", "task_validation", "summary"), "unexpected"), + (("cached_validation_state", "task_validation", "execution_grant"), True), + *[(("cached_validation_state", key), 1) for key in ( + "goal_id", "agent_id", "turn_instance_id", "turn_key", "result_kind", + )], + (("cached_validation_state", "todo_id"), {}), + (("cached_validation_state", "task_validation"), []), + (("cached_validation_state", "task_validation", "schema_version"), "unknown"), + (("cached_validation_state", "task_validation", "ok"), "true"), + (("cached_validation_state", "task_validation", "ok"), 1), + (("cached_validation_state", "task_validation", "status"), "progress"), + (("turn_key",), 1), + (("turn_key",), "sha256:" + "A" * 64), + (("turn_key",), "sha256:" + "b" * 64), + (("cached_validation_state", "turn_key"), "sha256:" + "b" * 64), + (("cached_validation_state", "goal_id"), "other-goal"), + (("cached_validation_state", "goal_id"), ""), + (("cached_validation_state", "agent_id"), "other-agent"), + (("cached_validation_state", "agent_id"), " "), + (("cached_validation_state", "todo_id"), "todo_other00001"), + (("cached_validation_state", "todo_id"), None), + (("cached_validation_state", "todo_id"), "TODO_FIXTURE0001"), + (("cached_validation_state", "todo_id"), "invalid"), + (("cached_validation_state", "turn_instance_id"), "other-turn"), + (("cached_validation_state", "turn_instance_id"), ""), + (("cached_validation_state", "turn_instance_id"), "a" * 129), + (("cached_validation_state", "turn_instance_id"), "unsafe/turn"), + (("cached_validation_state", "result_kind"), "unknown"), + (("cached_validation_state", "result_kind"), "validated_progress"), + (("effects",), [{"kind": "execute"}]), +]) +def test_python_inspection_adapter_rejects_corrupt_checkpoint_carriers( + monkeypatch: pytest.MonkeyPatch, + cached_validation_projection: tuple[dict[str, Any], dict[str, Any]], + path: tuple[str, ...], replacement: Any, +) -> None: + journal, projection = cached_validation_projection + before = copy.deepcopy(journal) + target = projection + for key in path[:-1]: + target = target[key] + if replacement is _MISSING_FIELD: + del target[path[-1]] + else: + target[path[-1]] = replacement + calls: list[str] = [] + + def rpc(method: str, _params: dict[str, Any]) -> dict[str, Any]: + calls.append(method) + return projection + + monkeypatch.setattr(turn_journal_runtime, "effect_runtime_result", rpc) + with pytest.raises( + RuntimeError, + match="projection shape mismatch|cached validation state mismatch|must remain effect-free", + ): + turn_journal_runtime.interpret_turn_journal_projection( + journal, goal_id="fixture-goal", agent_id="fixture-agent", turn_key=TURN_KEY, + ) + assert calls == ["turn_journal.inspect"] + assert journal == before + + +@pytest.mark.parametrize("path,replacement", [ + (("goal_id",), "other-goal"), + (("turn_key",), "sha256:" + "b" * 64), + (("plan", "transaction", "turn_key"), "sha256:" + "b" * 64), + (("host_result", "turn_key"), "sha256:" + "b" * 64), + (("result_kind",), "validated_progress"), + (("host_result", "result_kind"), "validated_progress"), + *[(("plan", "transaction", "settlement_plan", "identity", key), value) + for key, value in ( + ("goal_id", "other-goal"), ("agent_id", "other-agent"), + ("todo_id", "todo_other00001"), ("turn_instance_id", "other-turn"), + )], +]) +def test_python_inspection_adapter_binds_checkpoint_to_source_identity_and_keys( + monkeypatch: pytest.MonkeyPatch, + cached_validation_projection: tuple[dict[str, Any], dict[str, Any]], + path: tuple[str, ...], replacement: Any, +) -> None: + journal, projection = cached_validation_projection + target = journal + for key in path[:-1]: + target = target[key] + target[path[-1]] = replacement + monkeypatch.setattr(turn_journal_runtime, "effect_runtime_result", lambda *_args: projection) + + with pytest.raises(RuntimeError, match="cached validation state mismatch"): + turn_journal_runtime.interpret_turn_journal_projection( + journal, goal_id="fixture-goal", agent_id="fixture-agent", turn_key=TURN_KEY, + ) + + +@pytest.mark.parametrize("field,replacement", [ + ("goal_id", "other-goal"), ("agent_id", "other-agent"), + ("turn_key", "sha256:" + "b" * 64), +]) +def test_python_inspection_adapter_binds_checkpoint_to_requested_scope( + monkeypatch: pytest.MonkeyPatch, + cached_validation_projection: tuple[dict[str, Any], dict[str, Any]], + field: str, replacement: str, +) -> None: + journal, projection = cached_validation_projection + scope = {"goal_id": "fixture-goal", "agent_id": "fixture-agent", "turn_key": TURN_KEY} + scope[field] = replacement + monkeypatch.setattr(turn_journal_runtime, "effect_runtime_result", lambda *_args: projection) + + with pytest.raises(RuntimeError, match="cached validation state mismatch"): + turn_journal_runtime.interpret_turn_journal_projection(journal, **scope) + + @pytest.mark.parametrize("status,count", [ ("committed", 3), ("stopped", 0), ("scheduler_action_required", 3), ("in_progress", 7), ("failed", 0),