From 6a5cf1fa4289a094eafeaf74b41330d7a4e416f4 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 06:43:03 +0800 Subject: [PATCH] fix(steward): recover frozen report contract failures and rebind resumed task leases Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../native_chat/steward_managed_mode.py | 4 +- .../capabilities/native_chat/steward_turn.py | 129 +++++++++++++++++- loopx/cli_commands/turn_run_once.py | 43 +++++- .../collaboration/steward_managed_mode.ts | 32 ++++- .../quota/execution_preparation.py | 21 +++ .../control_plane/quota/heartbeat_receipt.py | 31 ++++- .../control_plane_ts/steward_reports.test.ts | 40 ++++++ ...t_steward_report_recovery_and_validator.py | 95 +++++++++++++ 8 files changed, 373 insertions(+), 22 deletions(-) diff --git a/loopx/capabilities/native_chat/steward_managed_mode.py b/loopx/capabilities/native_chat/steward_managed_mode.py index c6bab096a6..56f6af14e9 100644 --- a/loopx/capabilities/native_chat/steward_managed_mode.py +++ b/loopx/capabilities/native_chat/steward_managed_mode.py @@ -267,7 +267,8 @@ def _request(self, session_id: str, agent_id: str, operation_id: str, def admit(self, request: dict[str, Any], *, turn: dict[str, Any] | None = None, settlement_recovery: dict[str, Any] | None = None, - selection_recovery: dict[str, Any] | None = None) -> dict[str, Any]: + selection_recovery: dict[str, Any] | None = None, + frozen_contract_recovery: dict[str, Any] | None = None) -> dict[str, Any]: from .conversation_bindings import ChatConversationBindings from .steward_turn import StewardTurnHost if self.controller.closed.is_set(): @@ -310,6 +311,7 @@ def admit(self, request: dict[str, Any], *, turn: dict[str, Any] | None = None, "goal_active": bool(goal and goal.get("status") == "active" and not goal_is_stopped(goal)), "monitor_todo": monitor, **report, **creation, **({"report_settlement_recovery": settlement_recovery} if settlement_recovery is not None else {}), + **({"report_frozen_contract_recovery": frozen_contract_recovery} if frozen_contract_recovery is not None else {}), **({"report_selection_recovery": selection_recovery} if selection_recovery is not None else {}), **StewardTurnHost(self.controller).report_quota_basis(request), **StewardTurnHost(self.controller).monitor_restart_basis(request), diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 8d7939e92b..23e2f0a48e 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -82,6 +82,18 @@ def report_turn(store: Any, actor: Mapping[str, Any], window: Mapping[str, Any]) return original +def _normalize_report_kernel_candidate(candidate: Mapping[str, Any]) -> dict[str, Any]: + """Normalize a verified report candidate to canonical Turn settlement fields.""" + normalized = dict(candidate) + normalized.pop("editorial_response", None) + # This finite task's postcondition is the complete validated generation, not + # the model's choice of progress wording or profile-minimum shorthand. + normalized["result_kind"] = "validated_completion" + if normalized.get("delivery_batch_scale") == "multi_surface_or_implementation": + normalized["delivery_batch_scale"] = "multi_surface" + return normalized + + class StewardTurnHost: def __init__(self, controller: Any) -> None: self.controller = controller @@ -118,6 +130,7 @@ def _observe(self, actor: Mapping[str, Any]) -> dict[str, Any] | None: def _admit(self, request: dict[str, Any], *, turn: dict[str, Any] | None = None, observed: dict[str, Any] | None = None, prepare_context: bool = False, settlement_recovery: dict[str, Any] | None = None, + frozen_contract_recovery: dict[str, Any] | None = None, launch_recovery: dict[str, Any] | None = None, selection_recovery: dict[str, Any] | None = None) -> dict[str, Any]: if self.controller.closed.is_set(): @@ -125,7 +138,9 @@ def _admit(self, request: dict[str, Any], *, turn: dict[str, Any] | None = None, actor = request["actor"] if actor.get("schema_version") == "loopx_managed_session_actor_v0": return ManagedSessionTurn(self.controller).admit(request, turn=turn, - settlement_recovery=settlement_recovery, selection_recovery=selection_recovery) + settlement_recovery=settlement_recovery, + frozen_contract_recovery=frozen_contract_recovery, + selection_recovery=selection_recovery) if self.bindings is None: raise ValueError("the original steward binding is unavailable") observed = observed if observed is not None else self._observe(actor) @@ -175,6 +190,7 @@ def scope_id(ids): "goal": self.managed._goal(actor["goal_id"]), "goal_active": bool(goal and goal.get("status") == "active" and not goal_is_stopped(goal)), "request": request, "turn": turn, "scope": scope, "monitor_todo": monitor, **({"report_settlement_recovery": settlement_recovery} if settlement_recovery is not None else {}), + **({"report_frozen_contract_recovery": frozen_contract_recovery} if frozen_contract_recovery is not None else {}), **({"report_launch_recovery": launch_recovery} if launch_recovery is not None else {}), **({"report_selection_recovery": selection_recovery} if selection_recovery is not None else {}), **self.report_quota_basis(request), @@ -533,6 +549,81 @@ def _task_execution_lease(self, request: Mapping[str, Any], turn_id: str, *, rec self.store.append_event(actor["session_id"], turn_id, kind="native_loopx.task_lease_release_unconfirmed", payload={"todo_id": request["todo_id"], "lease_epoch": lease["lease_epoch"]}) + def _frozen_report_contract_recovery( + self, + actor: Mapping[str, Any], + window: Mapping[str, Any], + original: Mapping[str, Any], + *, + key: str, + generation: Mapping[str, Any], + root: Path, + ) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Recover only a already-frozen report whose host candidate hit host_result_contract.""" + from ...control_plane.turn_driver.host_candidate import parse_model_json + from ...control_plane.turn_driver.journal_store import load_turn_journal, turn_journal_path + + journal = load_turn_journal(turn_journal_path(root, goal_id=actor["goal_id"], turn_key=key)) + if ( + not isinstance(journal, Mapping) + or journal.get("status") != "failed" + or journal.get("result_kind") != "validation_failed" + or journal.get("validation_stage") != "host_result_contract" + or journal.get("host_attempt_count") != 1 + ): + return None + resumed = self.store.turn_for_client( + actor["session_id"], + native_steward_client_id(actor, "report." + window["window_id"] + ".admission-resume"), + ) + retry = self.store.turn_for_client( + actor["session_id"], + native_steward_client_id(actor, "report." + window["window_id"] + ".admission-retry"), + ) + seen_turn_ids: set[str] = set() + candidates: list[Mapping[str, Any]] = [] + for candidate_turn in (resumed, retry, original): + if isinstance(candidate_turn, Mapping): + tid = candidate_turn.get("turn_id") + if isinstance(tid, str) and tid not in seen_turn_ids: + seen_turn_ids.add(tid) + candidates.append(candidate_turn) + for candidate_turn in candidates: + events = self.store.events_after(actor["session_id"], candidate_turn["turn_id"], None) + native_failed = any( + row.get("kind") == "native_loopx.result" + and isinstance(row.get("payload"), Mapping) + and row["payload"].get("resume_turn_key") == key + and row["payload"].get("status") == "failed" + and row["payload"].get("result_kind") == "validation_failed" + and row["payload"].get("validation_stage") == "host_result_contract" + for row in events + ) + if not native_failed: + continue + for row in reversed(events): + if row.get("kind") != "answer.final": + continue + message = (((row.get("payload") or {}).get("response") or {}).get("message")) + parsed = parse_model_json(str(message or "")) + if not isinstance(parsed, dict): + continue + if parsed.get("result_kind") not in {"validated_progress", "validated_completion"}: + continue + editorial = parsed.get("editorial_response") + if not isinstance(editorial, Mapping) or editorial.get("request_digest") != generation.get("request_digest"): + continue + proof = { + "turn_key": key, + "result_kind": "validation_failed", + "validation_stage": "host_result_contract", + "host_attempt_count": 1, + "generation_verified": True, + "saved_candidate_verified": True, + } + return proof, _normalize_report_kernel_candidate(parsed) + return None + def recover_report_settlement(self, actor: dict[str, Any], window: dict[str, Any], todo: dict[str, Any]) -> None: """Resume only cached, validated native effects of the original report. @@ -559,7 +650,7 @@ def recover_report_settlement(self, actor: dict[str, Any], window: dict[str, Any for row in self.store.events_after(actor["session_id"], original["turn_id"], None)): return reports, owner_key = report_service(self.controller, actor) - validate_private_generation(attempt_directory(reports.root, window)) + generation = validate_private_generation(attempt_directory(reports.root, window)) if not reports.delivery_current(owner_key, window): raise ValueError("the original report authorization changed before settlement recovery") key = find_loopx_turn_key_by_settlement_identity(reports.root, goal_id=actor["goal_id"], @@ -568,9 +659,37 @@ def recover_report_settlement(self, actor: dict[str, Any], window: dict[str, Any return inspection = inspect_loopx_turn_journal(reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=key, retry_failed=True) - admitted = self._admit(request, observed=observed, settlement_recovery=inspection) + contract_recovery = ( + self._frozen_report_contract_recovery( + actor, window, original, key=key, generation=generation, root=reports.root, + ) + if (inspection.get("recovery_decision") or {}).get("reinvoke_host") is True + else None + ) + admitted = self._admit( + request, + observed=observed, + settlement_recovery=inspection, + frozen_contract_recovery=contract_recovery[0] if contract_recovery else None, + ) def no_model(*_args): raise ValueError("report settlement recovery must never invoke a model") + if contract_recovery is not None: + from ...control_plane.turn_driver.host_candidate import build_result + + normalized_candidate = contract_recovery[1] + replayed = False + + def replay_frozen_candidate(native_request: Mapping[str, Any], _launch_scope: Any) -> dict[str, Any]: + nonlocal replayed + if replayed: + raise ValueError("frozen report candidate replay may execute at most once") + replayed = True + return build_result(native_request, normalized_candidate, host_name="Codex app-server", allow_completion=True) + + host_runner = replay_frozen_candidate + else: + host_runner = no_model args = build_parser().parse_args(["turn", "run-once", "--goal-id", actor["goal_id"], "--agent-id", actor["agent_id"], "--project", admitted["workspace_path"], "--host", "generic-cli", "--resume-turn-key", key, "--retry-failed-turn", "--execute"]) @@ -580,7 +699,7 @@ def no_model(*_args): if todo["status"] == "open" else nullcontext(None)) as lease: handle_turn_command(args, registry_path=self.controller.registry_path, runtime_root_arg=str(reports.root), output_format=lambda *_a, **_kw: "json", - print_payload=lambda value, *_a: result.update(value), native_host_runner=no_model, + print_payload=lambda value, *_a: result.update(value), native_host_runner=host_runner, native_session_binding_resolver=lambda _envelope: {"schema_version": "loopx_turn_session_binding_v0", "goal_id": actor["goal_id"], "agent_id": actor["agent_id"]}, native_task_validator=self._report_validator(request), @@ -1135,7 +1254,7 @@ def launch(): # generation, not the model's choice of progress wording. # The kernel still runs its independent validator and lease # fence before closing only this window's generation Todo. - kernel_candidate["result_kind"] = "validated_completion" + kernel_candidate = _normalize_report_kernel_candidate(kernel_candidate) built = build_result(native_request, kernel_candidate, host_name="Antigravity" if isinstance(adapter, AntigravityAdapter) else "Codex app-server", **({"allow_completion": True} if (report_work or managed_turn or steward_advancement) and not governance_work else {})) diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index f43ddb5062..aa1b9a0e59 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -18,6 +18,7 @@ from ..control_plane.goals.first_party_host_admission import ( FirstPartyHostGoalAdmission, ) +from ..control_plane.quota.execution_preparation import read_active_task_lease_epoch from ..control_plane.quota.heartbeat_receipt import ( ensure_turn_heartbeat_settlement_receipt, ) @@ -117,6 +118,29 @@ def execute_turn_run_once( selected_todo = execution_context["selected_todo"] expected_todo_text_sha256 = execution_context["sha256"] task_lease_proof = execution_context.get("task_lease_proof") or None + task_lease_epoch = execution_context.get("task_lease_epoch") + rebound_lease_active = False + if ( + args.execute + and getattr(args, "resume_turn_key", None) + and native_completion_lease_resolver is not None + and selected_todo.get("todo_id") + ): + resolved_lease = dict(native_completion_lease_resolver()) + if resolved_lease.get("task_lease_idempotency_key") is not None: + rebound_proof = { + "idempotency_key": resolved_lease["task_lease_idempotency_key"], + "expected_version": resolved_lease["task_lease_expected_version"], + } + task_lease_epoch = read_active_task_lease_epoch( + runtime_root=runtime_root, + goal_id=args.goal_id, + todo_id=str(selected_todo["todo_id"]), + agent_id=args.agent_id, + requested_proof=rebound_proof, + ) + task_lease_proof = rebound_proof + rebound_lease_active = True transaction = ( payload.get("transaction") if isinstance(payload.get("transaction"), Mapping) @@ -289,7 +313,7 @@ def refresh_candidate( dry_run=dry_run, sync_global=not bool(args.no_global_sync), goal_ref=goal_ref, - task_lease_proof=task_lease_proof, + task_lease_proof=None if completion_todo_id is not None else task_lease_proof, ) # A Todo-less native governance Turn has no business artifact/argv. # Its independent validator is the canonical semantic writeback preview, @@ -312,6 +336,8 @@ def writeback( effect_ref: str, ) -> dict[str, object]: require_effect_ref(effect_ref, SettlementStepKind.DURABLE_WRITEBACK) + if args.execute and rebound_lease_active: + bind_heartbeat_settlement_receipt() refresh = refresh_candidate(result, completion_todo_id=completion_todo_id, completion_turn_key=completion_turn_key) if refresh.get("ok") and ( @@ -451,6 +477,8 @@ def completion_writeback( effect_ref: str, ) -> dict[str, object]: require_effect_ref(effect_ref, SettlementStepKind.DURABLE_WRITEBACK) + if args.execute and rebound_lease_active: + bind_heartbeat_settlement_receipt() prior = completion_readback() if prior is None: # A rejected attachment must not leave the Todo completed. @@ -742,10 +770,7 @@ def codex_settlement_admission(): settlement_admission = codex_settlement_admission - def on_managed_start_admitted() -> None: - if settlement_identity.request_subject is not None: - require_current_request_operation(request_locator, settlement_identity, - registry_path=registry_path, runtime_root=runtime_root, workspace=project) + def bind_heartbeat_settlement_receipt() -> None: replan_packet = execution_context["replan_action_packet"] capability_guard = ( replan_packet.get("capability_guard") @@ -765,9 +790,15 @@ def on_managed_start_admitted() -> None: ), goal_ref=goal_ref, task_lease_proof=task_lease_proof, - task_lease_epoch=execution_context.get("task_lease_epoch"), + task_lease_epoch=task_lease_epoch, ) + def on_managed_start_admitted() -> None: + if settlement_identity.request_subject is not None: + require_current_request_operation(request_locator, settlement_identity, + registry_path=registry_path, runtime_root=runtime_root, workspace=project) + bind_heartbeat_settlement_receipt() + managed_cadence = managed_cadence_start( runtime_root=runtime_root, goal_id=args.goal_id, diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index d56877dcaf..6a5ece9a0d 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -434,7 +434,7 @@ export function admitManagedSessionTurn(params: JsonObject): JsonObject { || todo.action_kind !== "generate_periodic_report" || !(todo.status === "open" && !recovery || settlementOnly)) { throw new EffectRuntimeRequestError("report execution requires the original Session subscription, window and task"); } - } else if (params.report_settlement_recovery != null) { + } else if (params.report_settlement_recovery != null || params.report_frozen_contract_recovery != null) { throw new EffectRuntimeRequestError("report recovery belongs only to its original calendar report"); } if (scheduledWork) { @@ -491,9 +491,34 @@ export function admitManagedSessionTurn(params: JsonObject): JsonObject { /** Both original conversation hosts use the native journal's no-model recovery. */ function reportSettlementOnly(params: JsonObject, request: JsonObject, todo: JsonObject): boolean { - if (params.report_settlement_recovery == null) return false; + if (params.report_settlement_recovery == null) { + if (params.report_frozen_contract_recovery != null) { + throw new EffectRuntimeRequestError("original report recovery requires validated remaining effects without a model"); + } + return false; + } const recovery = requireJsonObject(params.report_settlement_recovery, "native original report recovery"); const decision = requireJsonObject(recovery.recovery_decision, "native journal recovery decision"); + if (params.report_frozen_contract_recovery != null) { + const frozen = requireJsonObject(params.report_frozen_contract_recovery, "frozen report contract recovery"); + const effects = requireJsonObject(recovery.recorded_effects, "native recorded effects"); + const acceptedFrozen = recovery.schema_version === "loopx_turn_journal_inspection_v1" && recovery.ok === true + && recovery.goal_matches === true && recovery.owner_matches === true && recovery.turn_key_matches === true + && recovery.journal_consistent === true && todo.status === "open" && recovery.journal_status === "failed" + && Array.isArray(recovery.completed_phases) && recovery.completed_phases.length === 2 + && recovery.completed_phases[0] === "host_execute" && recovery.completed_phases[1] === "typed_result" + && decision.action === "continue" && decision.resume_from === "host_execute" && decision.reinvoke_host === true + && effects.host_invoked === true && effects.state_written === false + && effects.quota_spent === false && effects.scheduler_acknowledged === false + && typeof frozen.turn_key === "string" && ENVELOPED_SHA256_PATTERN.test(frozen.turn_key) + && frozen.result_kind === "validation_failed" && frozen.validation_stage === "host_result_contract" + && frozen.host_attempt_count === 1 + && frozen.generation_verified === true && frozen.saved_candidate_verified === true; + if (!acceptedFrozen || params.preparing_model_context === true) { + throw new EffectRuntimeRequestError("original report recovery requires validated remaining effects without a model"); + } + return true; + } const cachedGeneration = todo.status === "open" && recovery.journal_status === "failed" && decision.action === "continue" && decision.resume_from === "validation" && Array.isArray(recovery.completed_phases) && recovery.completed_phases.length === 2 @@ -705,7 +730,8 @@ export function admitStewardManagedTurn(params: JsonObject): JsonObject { throw new EffectRuntimeRequestError("launch recovery must retain the original native Turn key"); } } else if (request.report_window_id !== undefined || request.report_subscription_revision !== undefined - || request.resume_turn_key !== undefined || params.report_settlement_recovery != null || params.report_launch_recovery != null) { + || request.resume_turn_key !== undefined || params.report_settlement_recovery != null + || params.report_frozen_contract_recovery != null || params.report_launch_recovery != null) { throw new EffectRuntimeRequestError("report fields belong only to an original calendar report"); } let refreshScope = false; diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 4fb044f3e6..8d60cf9445 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -144,6 +144,27 @@ class RefreshTaskLeaseUnavailable(ValueError): code = "refresh_task_lease_unavailable" +def read_active_task_lease_epoch( + *, + runtime_root: Path, + goal_id: str, + todo_id: str, + agent_id: str, + requested_proof: Mapping[str, Any], +) -> int | None: + """Return the verified current lease epoch for an active resumed execution lease.""" + snapshot = _read_execution_snapshot(runtime_root, goal_id, todo_id) + if snapshot is None: + raise RefreshTaskLeaseUnavailable("The selected Todo's execution authority could not be read.") + if snapshot.get("handoff_mode") != "hard_lease": + return None + current = _project_execution_facts(_execution_facts(snapshot, goal_id, todo_id, agent_id)) + if current.get("state") != "ready" or current.get("lease_proof") != dict(requested_proof): + raise RefreshTaskLeaseUnavailable("The resumed Turn's active task lease could not be verified.") + epoch = current.get("lease_epoch") + return int(epoch) if epoch is not None else None + + def prepared_lease_receipt_fields( proof: Mapping[str, Any], *, lease_epoch: int | None = None, ) -> dict[str, Any]: diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index 3874436caa..364a5ac6d6 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -269,6 +269,15 @@ def ensure_turn_heartbeat_settlement_receipt( # identity before transport-specific start admission. Its owner may # refuse admission without writing a receipt or claiming work. before_bind() + if task_lease_proof is not None and identity.todo_id is not None: + from .execution_preparation import prepared_lease_receipt_fields + + rebound_lease_fields = prepared_lease_receipt_fields( + task_lease_proof, lease_epoch=task_lease_epoch, + ) + else: + rebound_lease_fields = None + preserved_details: dict[str, object] = {} if effective is not None: if observed is not None: effective_details = effective.get("details") @@ -277,9 +286,15 @@ def ensure_turn_heartbeat_settlement_receipt( if isinstance(effective_details, Mapping) else {} ) + preserved_details = dict(effective_details) + lease_unchanged = rebound_lease_fields is None or ( + all(effective_details.get(k) == v for k, v in rebound_lease_fields.items()) + and (task_lease_epoch is not None or "prepared_lease_epoch" not in effective_details) + ) if not semantic_replan_guard_scoped: - return effective - if "semantic_replan_obligation_id" in effective_details: + if lease_unchanged: + return effective + elif "semantic_replan_obligation_id" in effective_details: raw_existing_guard = effective_details.get( "semantic_replan_obligation_id" ) @@ -308,9 +323,11 @@ def ensure_turn_heartbeat_settlement_receipt( raise HeartbeatReceiptIdentityConflictError( "Turn heartbeat receipt belongs to another capability guard" ) - return effective + if lease_unchanged: + return effective details = { + **preserved_details, "turn_instance_id": identity.turn_instance_id, "todo_id": str(identity.todo_id or ""), "replan_obligation_id": str(identity.replan_obligation_id or ""), @@ -321,10 +338,10 @@ def ensure_turn_heartbeat_settlement_receipt( } if identity.todo_id is not None and (prepared_lease_required or task_lease_proof is not None): details["prepared_lease_required"] = True - if task_lease_proof is not None and identity.todo_id is not None: - from .execution_preparation import prepared_lease_receipt_fields - - details.update(prepared_lease_receipt_fields(task_lease_proof, lease_epoch=task_lease_epoch)) + if rebound_lease_fields is not None: + details.update(rebound_lease_fields) + if task_lease_epoch is None: + details.pop("prepared_lease_epoch", None) if semantic_replan_guard_scoped: details["semantic_replan_obligation_id"] = ( normalized_semantic_replan_obligation_id or "" diff --git a/tests/control_plane_ts/steward_reports.test.ts b/tests/control_plane_ts/steward_reports.test.ts index 83abe57a6d..2419f15714 100644 --- a/tests/control_plane_ts/steward_reports.test.ts +++ b/tests/control_plane_ts/steward_reports.test.ts @@ -106,6 +106,27 @@ test("a local calendar report retains its exact original Session, subscription, {recovery_decision: {...inspection.recovery_decision, resume_from: "host_execute"}}]) { assert.throws(() => admitManagedSessionTurn({...input, report_settlement_recovery: {...inspection, ...changed}})); } + const frozenInspection = {schema_version: "loopx_turn_journal_inspection_v1", ok: true, + goal_matches: true, owner_matches: true, turn_key_matches: true, journal_consistent: true, + journal_status: "failed", completed_phases: ["host_execute", "typed_result"], + recorded_effects: {host_invoked: true, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + recovery_decision: {action: "continue", resume_from: "host_execute", reinvoke_host: true}}; + const frozenProof = {turn_key: "sha256:" + "d".repeat(64), result_kind: "validation_failed", + validation_stage: "host_result_contract", host_attempt_count: 1, + generation_verified: true, saved_candidate_verified: true}; + assert.equal(admitManagedSessionTurn({ + ...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: frozenProof, + }).observation_only, false); + for (const invalid of [ + {...input, report_frozen_contract_recovery: frozenProof}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, saved_candidate_verified: false}}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, validation_stage: "task_postcondition"}}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, host_attempt_count: 2}}, + {...input, report_settlement_recovery: {...frozenInspection, recorded_effects: {...frozenInspection.recorded_effects, state_written: true}}, report_frozen_contract_recovery: frozenProof}, + {...input, report_todo: {...todo, status: "done"}, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: frozenProof}, + ]) { + assert.throws(() => admitManagedSessionTurn(invalid)); + } }); function original() { @@ -271,6 +292,25 @@ test("a model report requires the exact live calendar window and original task, {...recovery, report_settlement_recovery: {...recovery.report_settlement_recovery, recovery_decision: {action: "continue", reinvoke_host: true, resume_from: "quota_spend"}}}, ]) assert.throws(() => admitStewardManagedTurn(invalid)); + const frozenInspection = {schema_version: "loopx_turn_journal_inspection_v1", ok: true, + goal_matches: true, owner_matches: true, turn_key_matches: true, + journal_consistent: true, journal_status: "failed", completed_phases: ["host_execute", "typed_result"], + recorded_effects: {host_invoked: true, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + recovery_decision: {action: "continue", reinvoke_host: true, resume_from: "host_execute"}}; + const frozenProof = {turn_key: "sha256:" + "e".repeat(64), result_kind: "validation_failed", + validation_stage: "host_result_contract", host_attempt_count: 1, + generation_verified: true, saved_candidate_verified: true}; + assert.equal(admitStewardManagedTurn({ + ...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: frozenProof, + }).observation_only, false); + for (const invalid of [ + {...input, report_frozen_contract_recovery: frozenProof}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, saved_candidate_verified: false}}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, validation_stage: "task_postcondition"}}, + {...input, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: {...frozenProof, host_attempt_count: 2}}, + {...input, report_settlement_recovery: {...frozenInspection, recorded_effects: {...frozenInspection.recorded_effects, state_written: true}}, report_frozen_contract_recovery: frozenProof}, + {...input, report_todo: {...todo, status: "done"}, report_settlement_recovery: frozenInspection, report_frozen_contract_recovery: frozenProof}, + ]) assert.throws(() => admitStewardManagedTurn(invalid)); const changed = {...input, binding: {...p.binding, goal_scope: "all_registered"}, session: {...p.session, active_turn_id: "running", manager_authorization_scope_id: "old_scope", steward_context: {...p.session.steward_context, goal_scope: "all_registered"}}, diff --git a/tests/test_steward_report_recovery_and_validator.py b/tests/test_steward_report_recovery_and_validator.py index 70e74cd189..ae879b9e0a 100644 --- a/tests/test_steward_report_recovery_and_validator.py +++ b/tests/test_steward_report_recovery_and_validator.py @@ -279,3 +279,98 @@ def now(cls, tz=None): assert reports.health(owner)["publication_verified"] task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == done["loopx_request"]["todo_id"]) assert task["status"] == "done" + + +def test_steward_report_normalizes_profile_minimum_scale_shorthand(steward, tmp_path, monkeypatch): + _, runtime, _, transport, binding, _, _ = steward + sid, _, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + install_model(steward, tmp_path, sid) + helper = tmp_path / "report_model.py" + helper.write_text( + helper.read_text(encoding="utf-8").replace( + '"delivery_batch_scale": "single_surface"', + '"delivery_batch_scale": "multi_surface_or_implementation"', + ), + encoding="utf-8", + ) + + done = finish(runtime, pump_steward_monitors(runtime)[0]) + assert done["response"]["native_loopx"]["status"] == "committed", done + assert transport.steward_reports.reconcile() == 1 + assert reports.health(binding["binding_id"])["publication_verified"] + + +def test_frozen_report_recovers_host_result_contract_failure_without_model_call( + steward, tmp_path, monkeypatch, caplog, +): + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + import loopx.capabilities.native_chat.steward_turn as steward_turn_mod + from loopx.configure_goal import configure_goal + + _, runtime, _, transport, binding, _, _ = steward + point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) + + class CalendarClock(datetime): + @classmethod + def now(cls, tz=None): + return point.astimezone(tz) if tz else point.replace(tzinfo=None) + + monkeypatch.setattr(report_host, "datetime", CalendarClock) + monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + owner = binding["binding_id"] + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=1, execute=True, + )["ok"] + install_model(steward, tmp_path, sid) + assert finish(runtime, pump_steward_monitors(runtime)[0])["response"]["native_loopx"]["status"] == "committed" + assert transport.steward_reports.reconcile() == 1 + + point += timedelta(days=1) + runtime._steward_monitor_next_check.clear() + rejected = finish(runtime, pump_steward_monitors(runtime)[0]) + assert rejected["response"]["native_loopx"]["error_code"] == "turn_todo_selection_deferred" + + helper = tmp_path / "report_model.py" + helper.write_text( + helper.read_text(encoding="utf-8").replace( + '"delivery_batch_scale": "single_surface"', + '"delivery_batch_scale": "multi_surface_or_implementation"', + ), + encoding="utf-8", + ) + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=6, execute=True, + )["ok"] + runtime._steward_monitor_next_check.clear() + with monkeypatch.context() as unnormalized: + unnormalized.setattr( + steward_turn_mod, + "_normalize_report_kernel_candidate", + lambda candidate: {k: v for k, v in candidate.items() if k != "editorial_response"} | { + "result_kind": "validated_completion", + }, + ) + resumed = finish(runtime, pump_steward_monitors(runtime)[0]) + native = resumed["response"]["native_loopx"] + assert native["status"] == "failed" + assert native["result_kind"] == "validation_failed" + assert native["validation_stage"] == "host_result_contract" + window = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"])["window"] + assert (attempt_directory(reports.root, window) / "private-generation.json").is_file() + + model_calls = steward[5].read_bytes() + runtime._steward_monitor_next_check.clear() + with caplog.at_level(logging.WARNING): + assert not pump_steward_monitors(runtime) + assert "An original steward report is unavailable" not in caplog.text + assert steward[5].read_bytes() == model_calls + assert transport.steward_reports.reconcile() == 1 + assert reports.health(owner)["publication_verified"] + task = next( + row for row in StewardMonitor(runtime).todos(actor) + if row["todo_id"] == resumed["loopx_request"]["todo_id"] + ) + assert task["status"] == "done"