From 3e06c9edf68cce1727ba6e62151c540d2d146126 Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 20:47:12 +0800 Subject: [PATCH] fix(turn): retain admitted lease generation for Host retries Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- ...hanism-convergence-and-learning-loop-v0.md | 16 +++ ...-convergence-and-learning-loop-v0.zh-CN.md | 12 ++ loopx/cli_commands/turn_decision.py | 19 ++- loopx/cli_commands/turn_run_once.py | 14 ++- loopx/control_plane/quota/settlement_cli.py | 2 +- tests/test_codex_turn_completion.py | 116 ++++++++++++++---- 6 files changed, 147 insertions(+), 32 deletions(-) diff --git a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md index 3f8d2882d6..7d48e6434e 100644 --- a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md +++ b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md @@ -347,6 +347,22 @@ proof into their manual writeback, retaining pagination, recovery and once-only settlement assertions. This retires a duplicate read classifier; full C1/C4/C9 cutover, installed/native adoption and method/outcome benefit remain open. +**Source checkpoint: Host retry and settlement recovery (2026-11-21).** A failed +Host result that requires another model attempt now uses the existing TypeScript +heartbeat work-requalification owner through the +[launch adapter](../../../loopx/cli_commands/turn_decision.py). The +[Turn adapter](../../../loopx/cli_commands/turn_run_once.py) retains the recorded +preparation before launch and resolves a recovery lease only for a validated +settlement effect. This removes pre-launch lease replacement rather than adding +another generation rule. The +[File/SQLite controls](../../../tests/test_codex_turn_completion.py) distinguish +same-generation Host retry, replaced-generation refusal, failed task validation, +frozen-result completion, interruption after lease retirement and once-only +replay. On `main@3d15059fef14`, the replaced-generation counterexample reruns the +Host and settles; the candidate refuses it while retaining the other controls. +Full execution cutover, installed adoption, other-backend qualification and +independent method/outcome benefit remain open. + The first task needs an actual feedback or method-choice requirement. Reuse complete existing paths; real consumers must demonstrate gaps before repairing them rather than manufacturing a failure for the example. Experiments, training and procedural publication retain original authority. This document allocates neither another lane's quota nor a prescribed PR count. ## 12. Consequential choices to confirm diff --git a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md index c72216997d..9ebf67e9ad 100644 --- a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md +++ b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md @@ -325,6 +325,18 @@ Python 保留发布、字节预算与 fallback。仅用于本次调用的选项 只结算一次的断言。这退役一个重复读回分类器;完整 C1/C4/C9 切换、已安装/ 原生采用及方法/成果收益仍开放。 +**源码 checkpoint:Host 重试与结算恢复(2026-11-21)。** 失败的 Host 结果若需 +再次调用模型,现通过[启动适配器](../../../loopx/cli_commands/turn_decision.py) +复用既有 TypeScript heartbeat 工作再资格 owner。 +[Turn 适配器](../../../loopx/cli_commands/turn_run_once.py) 在启动前保留原 +preparation,仅在已验证的结算 effect 中读取恢复租约。这移除启动前的租约替换, +不新增代次判定规则。 +[File/SQLite 对照](../../../tests/test_codex_turn_completion.py) 区分同代 Host +重试、换代拒绝、任务验证失败、冻结结果完成、租约退役后的中断恢复和只结算一次 +的重放。`main@3d15059fef14` 的换代反例会重跑 Host 并结算;候选拒绝该路径, +同时保留其余对照。完整执行切换、已安装采用、其他后端资格及独立方法/成果收益 +仍开放。 + 首个任务须确有反馈或方法选择需求。已有链完整时直接复用;有缺口时由真实消费者证明并修复,不为配合示例制造失败。运行实验、训练模型和发布过程各沿原权限,本文不占用其他 lane 配额或预设 PR 数。 ## 12. 待确认的关键选择 diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index bf54f53947..2a07c52428 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -37,7 +37,10 @@ from ..control_plane.quota.turn_envelope import ( compose_turn_decision, ) -from ..control_plane.quota.settlement import SettlementIdentity, build_turn_scoped_cli_settlement_plan +from ..control_plane.quota.settlement import ( + SettlementIdentity, build_turn_scoped_cli_settlement_plan, read_heartbeat_settlement, +) +from ..control_plane.quota.settlement_cli import quota_rollout_details from ..control_plane.quota.heartbeat_receipt import ( find_heartbeat_receipt, heartbeat_receipt_settlement_replan_obligation_id, @@ -442,6 +445,20 @@ def current_turn_host_launch_decision( raise FirstPartyHostRuntimeRejected("current_turn_selection_changed") if planned_selected_work_basis is not None and plan.get("_selected_work_basis") != planned_selected_work_basis: raise FirstPartyHostRuntimeRejected("current_turn_work_changed") + if getattr(args, "resume_turn_key", None) and identity.todo_id is not None: + # A new Host attempt must qualify against the retained preparation, + # just like quota reentry. Frozen-result settlement never launches a + # Host and keeps its separate completion-lease recovery path. + current_details = quota_rollout_details(current, args, todo_id=identity.todo_id) + current_details["settlement_effect_id"] = identity.effect_id + retained = read_heartbeat_settlement( + runtime_root, goal_id=identity.goal_id, agent_id=identity.agent_id, + todo_id=identity.todo_id, turn_instance_id=identity.turn_instance_id, + registry_path=registry_path, goal_ref=goal_ref, + heartbeat_reentry_guard=current_details, + ) + if retained is None or (retained.native_child_admission or {}).get("report_permission") != "new_operation": + raise FirstPartyHostRuntimeRejected("goal_launch_not_allowed") return launch diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 6ca3fbe1aa..1b3bf8105c 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -140,7 +140,7 @@ def resolve_completion_lease() -> dict[str, Any]: } return dict(cached_completion_lease) - def rebind_resumed_task_lease(*, bind_receipt: bool = True) -> None: + def rebind_resumed_settlement_lease() -> None: nonlocal task_lease_proof, task_lease_epoch, rebound_lease_active if ( rebound_lease_active @@ -166,8 +166,7 @@ def rebind_resumed_task_lease(*, bind_receipt: bool = True) -> None: ) task_lease_proof = rebound_proof rebound_lease_active = True - if bind_receipt: - bind_heartbeat_settlement_receipt() + bind_heartbeat_settlement_receipt() transaction = ( payload.get("transaction") if isinstance(payload.get("transaction"), Mapping) @@ -380,7 +379,7 @@ def writeback( ) -> dict[str, object]: require_effect_ref(effect_ref, SettlementStepKind.DURABLE_WRITEBACK) if args.execute and completion_todo_id is None: - rebind_resumed_task_lease(bind_receipt=True) + rebind_resumed_settlement_lease() refresh = refresh_candidate(result, completion_todo_id=completion_todo_id, completion_turn_key=completion_turn_key) if refresh.get("ok") and ( @@ -521,7 +520,7 @@ def completion_writeback( prior = completion_readback() if prior is None: if args.execute: - rebind_resumed_task_lease(bind_receipt=True) + rebind_resumed_settlement_lease() # A rejected attachment must not leave the Todo completed. # The primary writer still rechecks the same capability owner # before commit; this preview authorizes no later effects. @@ -858,7 +857,10 @@ def on_managed_start_admitted() -> None: if settlement_identity.request_subject is not None: require_current_request_operation(request_locator, settlement_identity, registry_path=registry_path, runtime_root=runtime_root, workspace=project) - rebind_resumed_task_lease(bind_receipt=False) + # A Host retry is new work: retain the recorded preparation so the + # shared launch/requalification owner can reject a replaced lease. + # Only a validated settlement effect may resolve and bind a + # recovery lease for an existing frozen result. bind_heartbeat_settlement_receipt() managed_cadence = managed_cadence_start( diff --git a/loopx/control_plane/quota/settlement_cli.py b/loopx/control_plane/quota/settlement_cli.py index 39adb33660..a051d1682c 100644 --- a/loopx/control_plane/quota/settlement_cli.py +++ b/loopx/control_plane/quota/settlement_cli.py @@ -420,7 +420,7 @@ def quota_rollout_details( ) details: dict[str, object] = { "command": "quota", - "quota_command": args.quota_command, + "quota_command": getattr(args, "quota_command", ""), "ok": bool(payload.get("ok")), "should_run": bool(payload.get("should_run")), "appended": bool(payload.get("appended")), diff --git a/tests/test_codex_turn_completion.py b/tests/test_codex_turn_completion.py index ecbaf0a044..43b01dc3de 100644 --- a/tests/test_codex_turn_completion.py +++ b/tests/test_codex_turn_completion.py @@ -15,6 +15,7 @@ from loopx.cli import build_parser, main from loopx.cli_commands import turn_run_once from loopx.cli_commands.turn import handle_turn_command +from loopx.control_plane.quota.heartbeat_receipt import find_heartbeat_receipt from loopx.control_plane.todos.mutation_api import add_goal_todo from loopx.control_plane.turn_driver.codex_cli import codex_cli_result_schema from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime @@ -513,12 +514,17 @@ def test_builtin_codex_advertises_explicit_completion_without_promoting_progress @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("fail_refresh_after_rebind", [False, True]) +@pytest.mark.parametrize(("recovery_failure_stage", "fail_refresh_after_rebind"), [ + ("none", False), ("none", True), + ("task_postcondition", False), ("task_postcondition", True), + ("host_result_contract", False), ("host_result_contract_renewal", False), +]) def test_resumed_turn_rebinds_updated_lease_once_without_rereading_after_retire( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, fail_refresh_after_rebind: bool, + recovery_failure_stage: str, ) -> None: isolate_sqlite_runtime(tmp_path, monkeypatch) project, runtime, registry = _write_live_fixture( @@ -548,6 +554,9 @@ def test_resumed_turn_rebinds_updated_lease_once_without_rereading_after_retire( controller=SimpleNamespace(registry_path=registry, coordination_runtime_root=runtime) ) host_calls, lease_reads = [], [] + reject_recovery_validation = recovery_failure_stage == "task_postcondition" + reject_host_result = recovery_failure_stage in {"host_result_contract", "host_result_contract_renewal"} + same_generation = recovery_failure_stage == "host_result_contract_renewal" def resolve_lease(): lease_reads.append(lease["lease_epoch"]) @@ -558,9 +567,11 @@ def resolve_lease(): def host(request, launch_lifetime): with launch_lifetime(): host_calls.append(request["turn_key"]) - (project / "completion-artifact.txt").write_text("completed", encoding="utf-8") + (project / "completion-artifact.txt").write_text( + "tampered" if reject_recovery_validation else "completed", encoding="utf-8" + ) return { - "schema_version": "loopx_turn_result_v0", + "schema_version": "invalid-result" if reject_host_result else "loopx_turn_result_v0", "turn_key": request["turn_key"], "result_kind": "validated_completion", "completed_phases": ["host_execute", "typed_result"], @@ -610,29 +621,86 @@ def invoke(*extra: str): ) code, failed = invoke("--todo-id", todo_id) journal = _turn_journal(runtime) - assert code == 1 and journal["status"] == "in_progress", failed - assert lease_reads == [1] - - # Renew the active lease (epoch 2) before resuming from durable_writeback. - code, released = _cli( - registry, runtime, "task-lease", "release", "--goal-id", goal_id, - "--todo-id", todo_id, "--owner", agent_id, - "--idempotency-key", lease["idempotency_key"], - "--expected-version", str(lease["version"]), + assert code == 1 and journal["status"] == ( + "failed" if recovery_failure_stage != "none" else "in_progress" + ), failed + initial_lease_reads = [] if recovery_failure_stage != "none" else [1] + assert lease_reads == initial_lease_reads + original_preparation = find_heartbeat_receipt( + runtime, goal_id=goal_id, agent_id=agent_id, + turn_instance_id=journal["turn_key"], ) - assert code == 0, released - code, reclaimed = _cli( - registry, runtime, "todo", "claim", "--goal-id", goal_id, - "--todo-id", todo_id, "--agent-id", agent_id, "--claimed-by", agent_id, - "--claim-operation-id", "todo-claim:renewed-epoch-2", - ) - assert code == 0, reclaimed + assert original_preparation is not None + assert original_preparation["details"]["prepared_lease_epoch"] == 1 + + if same_generation: + code, renewed = _cli( + registry, runtime, "task-lease", "renew", "--goal-id", goal_id, + "--todo-id", todo_id, "--owner", agent_id, + "--idempotency-key", lease["idempotency_key"], + "--expected-version", str(lease["version"]), + ) + assert code == 0, renewed + else: + # Reclaim in a new generation before the frozen-result recovery. + code, released = _cli( + registry, runtime, "task-lease", "release", "--goal-id", goal_id, + "--todo-id", todo_id, "--owner", agent_id, + "--idempotency-key", lease["idempotency_key"], + "--expected-version", str(lease["version"]), + ) + assert code == 0, released + code, reclaimed = _cli( + registry, runtime, "todo", "claim", "--goal-id", goal_id, + "--todo-id", todo_id, "--agent-id", agent_id, "--claimed-by", agent_id, + "--claim-operation-id", "todo-claim:renewed-epoch-2", + ) + assert code == 0, reclaimed code, renewed_inspection = _cli(registry, runtime, *lease_argv) assert code == 0 and renewed_inspection["active"] is True lease = renewed_inspection["lease"] - assert lease["lease_epoch"] == 2 + assert lease["lease_epoch"] == (1 if same_generation else 2) monkeypatch.setattr(turn_run_once, "write_turn_validated_completion", complete) + if reject_host_result and not same_generation: + # Re-running a Host is new work, unlike settlement of a validated + # frozen result. A new lease generation cannot rewrite the old Turn's + # preparation to authorize that work. + reject_host_result = False + code, rejected = invoke( + "--resume-turn-key", journal["turn_key"], "--retry-failed-turn" + ) + retained = find_heartbeat_receipt( + runtime, goal_id=goal_id, agent_id=agent_id, + turn_instance_id=journal["turn_key"], + ) + assert (code, host_calls, lease_reads, retained) == ( + 1, [journal["turn_key"]], [], original_preparation + ), rejected + assert _effect_kinds(runtime, journal) == [] + assert _cli(registry, runtime, *lease_argv)[1]["active"] is True + return + reject_host_result = False + if reject_recovery_validation: + # A frozen result is no proof that its artifact is still valid. The + # real validator must reject it before reading or binding a completion + # lease, even though ordinary ownership recovery already succeeded. + (project / "completion-artifact.txt").write_text("tampered", encoding="utf-8") + code, rejected = invoke( + "--resume-turn-key", journal["turn_key"], "--retry-failed-turn" + ) + assert code == 1 and rejected["status"] == "failed", rejected + retained = find_heartbeat_receipt( + runtime, goal_id=goal_id, agent_id=agent_id, + turn_instance_id=journal["turn_key"], + ) + assert (lease_reads, retained) == (initial_lease_reads, original_preparation) + assert host_calls == [journal["turn_key"]] + assert _effect_kinds(runtime, journal) == [] + assert _cli(registry, runtime, *lease_argv)[1]["active"] is True + _, open_todos = _cli(registry, runtime, "todo", "list", "--goal-id", goal_id) + assert next(t for t in open_todos["todos"] if t["todo_id"] == todo_id)["status"] == "open" + (project / "completion-artifact.txt").write_text("completed", encoding="utf-8") if fail_refresh_after_rebind: original_refresh = turn_run_once.refresh_state_run @@ -646,7 +714,7 @@ def fail_non_dry_refresh(*args, **kwargs): "--resume-turn-key", journal["turn_key"], "--retry-failed-turn" ) assert code == 1, second_failed - assert lease_reads == [1, 2] + assert lease_reads == [*initial_lease_reads, lease["lease_epoch"]] assert _cli(registry, runtime, *lease_argv)[1]["active"] is False _, todos_after_second = _cli(registry, runtime, "todo", "list", "--goal-id", goal_id) assert next(t for t in todos_after_second["todos"] if t["todo_id"] == todo_id)["status"] == "done" @@ -654,7 +722,7 @@ def fail_non_dry_refresh(*args, **kwargs): code, resumed = invoke("--resume-turn-key", journal["turn_key"], "--retry-failed-turn") assert code == 0 and resumed["status"] == "committed", resumed - assert lease_reads == [1, 2] + assert lease_reads == [*initial_lease_reads, lease["lease_epoch"]] assert _cli(registry, runtime, *lease_argv)[1]["active"] is False assert _effect_kinds(runtime, journal) == [ "todo_complete", @@ -664,8 +732,8 @@ def fail_non_dry_refresh(*args, **kwargs): code, replay = invoke("--resume-turn-key", journal["turn_key"]) assert code == 0 and replay["replayed"] is True, replay - assert lease_reads == [1, 2] - assert host_calls == [journal["turn_key"]] + assert lease_reads == [*initial_lease_reads, lease["lease_epoch"]] + assert host_calls == [journal["turn_key"]] * (2 if same_generation else 1) assert _effect_kinds(runtime, journal) == [ "todo_complete", "refresh_state",