diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index faf881fe7e..ebb34d1565 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -216,7 +216,8 @@ def monitor_restart_request(original: Mapping[str, Any]) -> dict[str, Any] | Non """Prepare correlation only; current typed admission decides recovery.""" prior = original.get("loopx_request") or {} error = original.get("error_code") - if (original.get("status") != "failed" or error not in {"server_restarted", "runtime_error"} + if (original.get("status") != "failed" + or error not in {"server_restarted", "runtime_error", "runtime_unavailable"} or prior.get("work_kind") != "monitor" or (prior.get("monitor_restart_recovery") is not None and error != "server_restarted")): return None @@ -335,15 +336,52 @@ def report_selection_recovery(self, original: dict[str, Any], *, from ...control_plane.effect_runtime import source_number_tokens journal = load_turn_journal(turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=key)) + current_guard = None + if (journal.get("status") == "failed" + and journal.get("result_kind") == "validation_failed" + and journal.get("validation_stage") == "host_result_contract"): + current_guard = self._cleared_replan_guard(actor) + if current_guard is None: + return None # Transport canonical observations to the shared typed admission # owner. proof["probe"] = {"turn": probe_turn, "inspection": inspect_loopx_turn_journal(reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], turn_key=key), "journal": journal, - "source_number_tokens": source_number_tokens((journal.get("plan") or {}).get("work_contract"))} + "source_number_tokens": source_number_tokens((journal.get("plan") or {}).get("work_contract")), + **({"current_guard": current_guard} if current_guard is not None else {})} return proof + def _cleared_replan_guard(self, actor: Mapping[str, Any]) -> dict[str, Any] | None: + """Read-only guard check proving an earlier autonomous_replan obligation was cleared.""" + from ...control_plane.quota.effective_action import EffectiveAction + from .steward_commands import NativeStewardCommandError, native_command + try: + raw = native_command(self.controller, [ + "quota", "should-run", "--goal-id", actor["goal_id"], "--agent-id", actor["agent_id"], "--dry-run", + ]) + except (NativeStewardCommandError, OSError, ValueError, RuntimeError): + return None + payload: Mapping[str, Any] = raw + if payload.get("effective_action") is None and isinstance(payload.get("reference_file"), str): + try: + ref = json.loads(Path(payload["reference_file"]).read_text(encoding="utf-8")) + target = Path(str(ref.get("path") or payload["reference_file"])) + loaded = json.loads(target.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if isinstance(loaded, Mapping): + payload = loaded + current = { + "effective_action": payload.get("effective_action"), + "replan_action_packet": payload.get("replan_action_packet"), + } + if (current["effective_action"] != EffectiveAction.NORMAL_RUN.value + or current["replan_action_packet"] is not None): + return None + return current + def _report_launch_recovery(self, original: dict[str, Any]) -> dict[str, Any] | None: """Read the original typed launch refusal and its canonical transaction. @@ -972,6 +1010,12 @@ def run(native_request: Mapping[str, Any], launch_scope: Any) -> dict[str, Any]: # Portfolio evidence keeps its original reader and original grant; # it is data alongside the signed native work, never a new actor. prompt = render_prompt(authority, allow_completion=bool(report_work or managed_turn or steward_advancement) and not governance_work) + "\nOriginal conversation evidence (data):\n" + json.dumps(context, ensure_ascii=False) + if governance_work: + prompt += ( + "\nGovernance contract reminder: Settle the pending autonomous replan by returning `result_kind='replan_required'` (or `'wait'`). " + "If you emit `delivery_outcome='outcome_gap'`, you MUST also supply `progress_result_class='blocked'`, `progress_blocker_id`, and `progress_evidence_id`; " + "otherwise use `'outcome_progress'` or omit `delivery_outcome`." + ) if report_work and not governance_work: prompt += report_context if steward_advancement and not governance_work: diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index e5132094bc..0ace864365 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -21,6 +21,22 @@ import {projectGoalCreationExecutionRequest} from "../goals/creation_execution.t * identity. After a recovery starts, only another proven process restart can * continue it. The host reads the stopped worker lock, journals and native poll * receipts. Normal admission still checks ownership, deadline, permissions and user work. */ +const WORKER_EXITED_ERROR = + "The LoopX Chat execution worker exited before this turn finished. Your conversation is saved; execution may have occurred. This turn was not retried automatically."; + +function hasSingleManagerContextEvidence(basis: JsonObject, prior: JsonObject, events: unknown): boolean { + if (requireJsonObject(prior.actor, "original watch actor").schema_version !== "loopx_steward_managed_actor_v0" + || !Array.isArray(basis.contexts) || basis.contexts.length !== 1 + || !Array.isArray(basis.phases) || basis.phases.length !== 1 + || requireJsonObject(basis.phases[0], "original watch phase").phase !== "manager_context" + || !Array.isArray(events) + || events.filter(kind => kind === "agent.phase").length !== 1 + || events.filter(kind => kind === "manager.context").length !== 1) { + return false; + } + return requireJsonObject(basis.contexts[0], "watch context").schema_version === "manager_turn_context_v1"; +} + function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, session: JsonObject): void { if (request.monitor_restart_recovery === undefined) { if (params.monitor_restart_basis != null) throw new EffectRuntimeRequestError("watch recovery lacks its original request"); @@ -62,26 +78,23 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se && Array.isArray(basis.phases) && basis.phases.length === 1 && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1; - // A custom signal monitor may fail inside the read-only external signal - // check after manager_context completes and before any poll receipt is written. - // Allow at most one bounded recovery when zero-effect proofs hold. - let signalUnavailable = false; - if (original.error_code === "runtime_error" - && original.error === "steward monitor external signal source temporarily unavailable" - && requireJsonObject(prior.actor, "original watch actor").schema_version === "loopx_steward_managed_actor_v0" - && Array.isArray(basis.contexts) && basis.contexts.length === 1 - && Array.isArray(basis.phases) && basis.phases.length === 1 - && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" - && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1 - && Array.isArray(events) && events.filter(kind => kind === "manager.context").length === 1) { - const context = requireJsonObject(basis.contexts[0], "signal unavailable watch context"); - signalUnavailable = context.schema_version === "manager_turn_context_v1"; - } + // Post-context monitor failures (external signal outage, daemon worker exit, + // storage I/O error, or runtime unavailability) before any poll receipt or + // upstream model turn may recover at most once when strict single-context + // and zero-effect journal proofs hold; pre-context unknown worker launches remain closed. + const postContextWatchRecoverable = hasSingleManagerContextEvidence(basis, prior, events) && ( + original.error_code === "runtime_unavailable" + || original.error_code === "runtime_error" && ( + original.error === "steward monitor external signal source temporarily unavailable" + || original.error === "disk I/O error" + || original.error === WORKER_EXITED_ERROR + ) + ); if (Object.keys(recovery).join() !== "original_turn_id" || recovery.original_turn_id !== original.turn_id || typeof original.turn_id !== "string" || !/^[a-f0-9]{32}$/.test(original.turn_id) || original.session_id !== session.session_id || original.status !== "failed" - || !(original.error_code === "server_restarted" || scopeRefused || identityRefused || signalUnavailable) + || !(original.error_code === "server_restarted" || scopeRefused || identityRefused || postContextWatchRecoverable) || original.origin !== "native_loopx" || original.loopx_execution !== true || original.response !== null || original.upstream_turn_id !== null || original.delta_count !== 0 || prior.work_kind !== "monitor" || request.work_kind !== "monitor" || request.validation_command !== null @@ -196,8 +209,6 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s throw new EffectRuntimeRequestError("report selection recovery requires the original proven pre-execution replan refusal"); } if (proof.probe == null) return; - // The original host returned before calling its model when a fresh guard - // no longer owed governance. Absence of output alone cannot prove that. const probe = requireJsonObject(proof.probe, "original host admission probe"); const observed = requireJsonObject(probe.turn, "original probe Chat Turn"); const journal = requireJsonObject(probe.journal, "canonical probe journal"); @@ -207,41 +218,71 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s const plan = requireJsonObject(journal.plan, "probe plan"); const route = requireJsonObject(plan.route, "probe route"); const envelope = turnPlanWorkSource(plan, probe.source_number_tokens).work; - const action = requireJsonObject(envelope.action, "probe action"); - const selected = requireJsonObject(action.selected_todo, "probe selected task"); const identity = requireJsonObject(requireJsonObject(requireJsonObject(plan.transaction, "probe transaction") .settlement_plan, "probe settlement").identity, "probe identity"); const host = requireJsonObject(journal.host, "probe host"); - const stopped = requireJsonObject(journal.host_result, "probe host result"); + const actor = requireJsonObject(request.actor, "report actor"); const runningHere = params.turn != null && requireJsonObject(params.turn, "current Chat Turn").turn_id === observed.turn_id && ["starting", "running"].includes(String(observed.status)); - if (observed.session_id !== sessionId || typeof observed.turn_id !== "string" || !/^[a-f0-9]{32}$/.test(observed.turn_id) - || observed.turn_id === original.turn_id - || observed.origin !== "native_loopx" || observed.loopx_execution !== true - || observed.delta_count !== 0 || observed.upstream_turn_id !== null || observed.error_code != null || observed.error != null - || JSON.stringify(observed.loopx_request) !== JSON.stringify(request) - || !(runningHere || observed.status === "completed" - && JSON.stringify(requireJsonObject(observed.response, "probe response").native_loopx) === JSON.stringify(result)) - || journal.schema_version !== "loopx_turn_journal_v0" || !ENVELOPED_SHA256_PATTERN.test(String(journal.turn_key)) - || journal.status !== "stopped" || journal.goal_id !== envelope.goal_id - || journal.host_attempt_count !== 1 || journal.result_kind !== "wait" + const commonInvalid = observed.session_id !== sessionId || typeof observed.turn_id !== "string" + || !/^[a-f0-9]{32}$/.test(observed.turn_id) || observed.turn_id === original.turn_id + || observed.origin !== "native_loopx" || observed.loopx_execution !== true + || observed.error_code != null || observed.error != null + || JSON.stringify(observed.loopx_request) !== JSON.stringify(request) + || !(runningHere || observed.status === "completed" + && JSON.stringify(requireJsonObject(observed.response, "probe response").native_loopx) === JSON.stringify(result)) + || journal.schema_version !== "loopx_turn_journal_v0" || !ENVELOPED_SHA256_PATTERN.test(String(journal.turn_key)) + || journal.goal_id !== envelope.goal_id || journal.host_attempt_count !== 1 + || host.kind !== "codex-app-server" || host.executable !== "built-in" + || inspection.schema_version !== "loopx_turn_journal_inspection_v1" || inspection.ok !== true + || inspection.goal_matches !== true || inspection.owner_matches !== true || inspection.turn_key_matches !== true + || inspection.journal_consistent !== true + || recovery.action !== "return_existing" || recovery.reinvoke_host !== false + || recorded.host_invoked !== true + || ["state_written", "quota_spent", "scheduler_acknowledged"].some(key => recorded[key] !== false) + || identity.goal_id !== envelope.goal_id || identity.agent_id !== envelope.agent_id + || identity.goal_id !== actor.goal_id || identity.agent_id !== actor.agent_id + || identity.turn_instance_id !== request.turn_instance_id + ".replan"; + if (commonInvalid) { + throw new EffectRuntimeRequestError("report continuation requires its canonical host-only admission stop; retain all model or unknown work"); + } + if (journal.status === "failed" && journal.result_kind === "validation_failed" + && journal.validation_stage === "host_result_contract") { + // Only once the blocking autonomous replan obligation is cleared elsewhere + // may a report whose prior governance probe failed host contract validation + // resume cleanly via one bounded .admission-resume carrier. + const receipt = requireJsonObject(journal.receipt, "probe receipt"); + const eligibility = requireJsonObject(receipt.commit_eligibility, "probe commit eligibility"); + const guard = requireJsonObject(probe.current_guard, "current guard observation"); + if (inspection.journal_status !== "failed" + || JSON.stringify(journal.completed_phases) !== JSON.stringify(["host_execute", "typed_result"]) + || receipt.ok !== true || receipt.failed_phase !== "validation" || receipt.status !== "failed" + || ["writeback", "quota_spend", "scheduler_ack"].some(key => eligibility[key] !== false) + || route.kind !== "replan_required" + || route.effective_action !== EffectiveAction.AUTONOMOUS_REPLAN_REQUIRED + || envelope.effective_action !== EffectiveAction.AUTONOMOUS_REPLAN_REQUIRED + || identity.binding_kind !== "autonomous_replan" + || guard.effective_action !== EffectiveAction.NORMAL_RUN + || guard.replan_action_packet != null) { + throw new EffectRuntimeRequestError("report continuation after validation failure requires a zero-write governance refusal and cleared replan"); + } + return; + } + // The original host returned before calling its model when a fresh guard + // no longer owed governance. Absence of output alone cannot prove that. + const action = requireJsonObject(envelope.action, "probe action"); + const selected = requireJsonObject(action.selected_todo, "probe selected task"); + const stopped = requireJsonObject(journal.host_result, "probe host result"); + if (observed.delta_count !== 0 || observed.upstream_turn_id !== null + || journal.status !== "stopped" || journal.result_kind !== "wait" || JSON.stringify(journal.completed_phases) !== JSON.stringify(["host_execute", "typed_result", "validation"]) - || host.kind !== "codex-app-server" || host.executable !== "built-in" || stopped.schema_version !== "loopx_turn_result_v0" || stopped.turn_key !== journal.turn_key || stopped.result_kind !== "wait" || stopped.classification !== "report_replan_not_admitted" - || inspection.schema_version !== "loopx_turn_journal_inspection_v1" || inspection.ok !== true - || inspection.goal_matches !== true || inspection.owner_matches !== true || inspection.turn_key_matches !== true - || inspection.journal_consistent !== true || inspection.journal_status !== "stopped" - || recovery.action !== "return_existing" || recovery.reinvoke_host !== false - || recorded.host_invoked !== true || ["state_written", "quota_spent", "scheduler_acknowledged"].some(key => recorded[key] !== false) + || inspection.journal_status !== "stopped" || route.kind !== "ready_for_host" || route.effective_action !== EffectiveAction.NORMAL_RUN || envelope.effective_action !== EffectiveAction.NORMAL_RUN || envelope.replan_action_packet != null || action.must_attempt !== true || action.delivery_allowed !== true || selected.todo_id !== identity.todo_id - || identity.goal_id !== envelope.goal_id || identity.agent_id !== envelope.agent_id - || identity.goal_id !== requireJsonObject(request.actor, "report actor").goal_id - || identity.agent_id !== requireJsonObject(request.actor, "report actor").agent_id - || typeof identity.todo_id !== "string" || !/^todo_[A-Za-z0-9_-]{1,100}$/.test(identity.todo_id) - || identity.turn_instance_id !== request.turn_instance_id + ".replan") { + || typeof identity.todo_id !== "string" || !/^todo_[A-Za-z0-9_-]{1,100}$/.test(identity.todo_id)) { throw new EffectRuntimeRequestError("report continuation requires its canonical host-only admission stop; retain all model or unknown work"); } } diff --git a/tests/control_plane_ts/steward_managed_mode.test.ts b/tests/control_plane_ts/steward_managed_mode.test.ts index bab62ebfb9..dc77045638 100644 --- a/tests/control_plane_ts/steward_managed_mode.test.ts +++ b/tests/control_plane_ts/steward_managed_mode.test.ts @@ -361,3 +361,60 @@ test("observeStewardNativeExecutor admits qualified codex-to-antigravity fallbac session: {...qualifiedFallback.session, upstream_context_history: []}, }).execution_available, false); }); + +test("zero-effect worker exit, storage I/O, and runtime unavailability monitors admit one bounded recovery", () => { + const {params, goal} = fixture(); + const actor = bindStewardManagedActor({...params, goal}); + const prior = {operation: "native_turn", host: "codex-app-server", actor, + work_kind: "monitor", turn_instance_id: "monitor.transient", todo_id: "todo_transient_watch", validation_command: null, + monitor_deadline: "2026-11-20T07:44:39Z", monitor_lease_version: 0}; + const monitor = {todo_id: "todo_transient_watch", task_class: "continuous_monitor", + target_key: "steward-portfolio-watch", watch_only: true, claimed_by: "steward", status: "open", + next_due_at: prior.monitor_deadline}; + const workerExitedError = + "The LoopX Chat execution worker exited before this turn finished. Your conversation is saved; execution may have occurred. This turn was not retried automatically."; + for (const scenario of [ + { + error_code: "runtime_error", + error: workerExitedError, + contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + event_kinds: ["turn.queued", "agent.phase", "manager.context", "turn.failed"], + }, + { + error_code: "runtime_error", + error: "disk I/O error", + contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + event_kinds: ["turn.queued", "agent.phase", "manager.context", "turn.failed"], + }, + { + error_code: "runtime_unavailable", + error: "The Agent runtime could not be prepared.", + contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + event_kinds: ["turn.queued", "turn.failed", "agent.phase", "manager.context"], + }, + ]) { + const original = {turn_id: "4".repeat(32), session_id: params.session.session_id, + status: "failed", error_code: scenario.error_code, error: scenario.error, + origin: "native_loopx", loopx_execution: true, upstream_turn_id: null, response: null, + delta_count: 0, loopx_request: prior}; + const basis = {original, contexts: scenario.contexts, phases: [{phase: "manager_context"}], + event_kinds: scenario.event_kinds, poll_absent: true, old_worker_excluded: true, lease_available: true}; + const input = {...params, goal, goal_active: true, binding: {...params.binding, managed_actor: actor}, + request: {...prior, monitor_restart_recovery: {original_turn_id: original.turn_id}}, + monitor_todo: monitor, monitor_restart_basis: basis}; + assert.equal(admitStewardManagedTurn(input).observation_only, true); + for (const bad of [ + {contexts: []}, + {phases: []}, + {contexts: [{schema_version: "wrong"}]}, + {contexts: [{schema_version: "manager_turn_context_v1"}, {schema_version: "manager_turn_context_v1"}]}, + {poll_absent: false}, + {old_worker_excluded: false}, + {lease_available: false}, + {original: {...original, upstream_turn_id: "started-upstream"}}, + {original: {...original, loopx_request: {...prior, monitor_restart_recovery: {original_turn_id: "5".repeat(32)}}}}, + ]) { + assert.throws(() => admitStewardManagedTurn({...input, monitor_restart_basis: {...basis, ...bad}})); + } + } +}); diff --git a/tests/control_plane_ts/steward_reports.test.ts b/tests/control_plane_ts/steward_reports.test.ts index c938d3314e..84ec963790 100644 --- a/tests/control_plane_ts/steward_reports.test.ts +++ b/tests/control_plane_ts/steward_reports.test.ts @@ -510,4 +510,33 @@ test("only a proven original pre-execution replan refusal admits one correlated ({...probe, inspection: {...inspection, recorded_effects: {...inspection.recorded_effects, [key]: value}}}))), {...probe, inspection: {...inspection, recovery_decision: {...inspection.recovery_decision, reinvoke_host: true}}}, ]) assert.throws(() => admitStewardManagedTurn({...input, report_selection_recovery: {...proof, probe: changed}})); + + const validationTurnKey = "sha256:" + "d".repeat(64); + const validationJournal = {schema_version: "loopx_turn_journal_v0", goal_id: p.actor.goal_id, + turn_key: validationTurnKey, status: "failed", validation_stage: "host_result_contract", + host: {kind: "codex-app-server", executable: "built-in"}, + host_attempt_count: 1, result_kind: "validation_failed", completed_phases: ["host_execute", "typed_result"], + receipt: {ok: true, failed_phase: "validation", status: "failed", + commit_eligibility: {writeback: false, quota_spend: false, scheduler_ack: false}}, + plan: {route: {kind: "replan_required", effective_action: "autonomous_replan_required"}, + turn_envelope: {goal_id: p.actor.goal_id, agent_id: p.actor.agent_id, effective_action: "autonomous_replan_required"}, + transaction: {settlement_plan: {identity: {goal_id: p.actor.goal_id, agent_id: p.actor.agent_id, + binding_kind: "autonomous_replan", turn_instance_id: request.turn_instance_id + ".replan"}}}}}; + const validationInspection = {...inspection, journal_status: "failed"}; + const clearedGuard = {effective_action: "normal_run", replan_action_packet: null}; + const clearedProbe = {turn: {...source, turn_id: "d".repeat(32), delta_count: 18, upstream_turn_id: "model-turn"}, + journal: validationJournal, inspection: validationInspection, current_guard: clearedGuard}; + assert.equal(admitStewardManagedTurn({...input, report_selection_recovery: {...proof, probe: clearedProbe}}).observation_only, false); + for (const badProbe of [ + {...clearedProbe, current_guard: {effective_action: "autonomous_replan_required", replan_action_packet: {obligation_id: "replan-1"}}}, + {...clearedProbe, current_guard: {effective_action: "normal_run", replan_action_packet: {obligation_id: "replan-1"}}}, + {...clearedProbe, journal: {...validationJournal, validation_stage: "task_postcondition"}}, + {...clearedProbe, journal: {...validationJournal, receipt: {...validationJournal.receipt, + commit_eligibility: {writeback: true, quota_spend: false, scheduler_ack: false}}}}, + {...clearedProbe, inspection: {...validationInspection, recorded_effects: {...inspection.recorded_effects, state_written: true}}}, + {...clearedProbe, journal: {...validationJournal, plan: {...validationJournal.plan, transaction: {settlement_plan: { + identity: {...validationJournal.plan.transaction.settlement_plan.identity, turn_instance_id: "other.replan"}}}}}}, + ]) { + assert.throws(() => admitStewardManagedTurn({...input, report_selection_recovery: {...proof, probe: badProbe}})); + } }); diff --git a/tests/test_native_steward_monitor.py b/tests/test_native_steward_monitor.py index ba51e97a76..ca7463a016 100644 --- a/tests/test_native_steward_monitor.py +++ b/tests/test_native_steward_monitor.py @@ -1372,3 +1372,73 @@ def flaky_repo_observe(repo_slug, **_kwargs): recovered_done = finish(runtime, recovered_try[0]) assert recovered_done["status"] == "completed", recovered_done assert local_watch_row(runtime, actor, pr_todo_id)["next_due_at"] != third_due + + +@pytest.mark.parametrize( + ("error_code", "error_message", "fail_before_context"), + [ + ( + "runtime_error", + "The LoopX Chat execution worker exited before this turn finished. Your conversation is saved; execution may have occurred. This turn was not retried automatically.", + False, + ), + ("runtime_error", "disk I/O error", False), + ( + "runtime_unavailable", + "The Agent runtime could not be prepared. Check the runtime installation and configuration, then retry.", + True, + ), + ], +) +def test_transient_zero_effect_monitor_failures_recover_once( + steward, tmp_path, monkeypatch, error_code, error_message, fail_before_context, +): + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + + store, runtime, *_ = steward + sid, actor, monitor = prepared(steward, tmp_path, monkeypatch, "sqlite") + default_watch = monitor.current(actor) + todo_id = default_watch["todo_id"] + due = (datetime.now(timezone.utc) - timedelta(seconds=2)).isoformat() + edit_local_watch(runtime, actor, todo_id, "--next-due-at", due) + with monkeypatch.context() as paused: + paused.setattr(runtime, "_start_accepted_turn_worker", lambda **_kw: True) + runtime._steward_monitor_next_check.clear() + first = pump_steward_monitors(runtime) + assert len(first) == 1 + failed_turn = first[0] + turn_id = failed_turn["turn_id"] + if fail_before_context: + runtime._fail_turn( + sid, turn_id, error_code, error_message, + status="failed", expected_statuses={"queued", "starting", "running"}, + ) + store.append_event( + sid, turn_id, kind="agent.phase", + payload={"phase": "manager_context", "label": "Reading authorized Goal state"}, + ) + store.append_event( + sid, turn_id, kind="manager.context", + payload={"schema_version": "manager_turn_context_v1", "goals": []}, + ) + if not fail_before_context: + runtime._fail_turn( + sid, turn_id, error_code, error_message, + status="failed", expected_statuses={"queued", "starting", "running"}, + ) + request = StewardTurnHost.monitor_restart_request(store.load_turn(sid, turn_id)) + assert request is not None and request["monitor_restart_recovery"]["original_turn_id"] == turn_id + + runtime._steward_monitor_next_check.clear() + recovered = pump_steward_monitors(runtime) + assert len(recovered) == 1 + assert recovered[0]["loopx_request"]["monitor_restart_recovery"]["original_turn_id"] == turn_id + done = finish(runtime, recovered[0]) + assert done["status"] == "completed", done + assert local_watch_row(runtime, actor, todo_id)["next_due_at"] != due + + # Even if a restart carrier itself fails without polling, it must not spawn a second retry. + store.update_turn(sid, done["turn_id"], status="failed", error_code=error_code, error=error_message) + carrier = store.load_turn(sid, done["turn_id"]) + assert carrier is not None and carrier["client_turn_id"].endswith(".restart-recovery") + assert StewardTurnHost.monitor_restart_request(carrier) is None diff --git a/tests/test_native_steward_reports.py b/tests/test_native_steward_reports.py index 64e6cfb099..2ea2975782 100644 --- a/tests/test_native_steward_reports.py +++ b/tests/test_native_steward_reports.py @@ -1034,6 +1034,113 @@ def legacy_selection(host, original, **kwargs): assert (capture.read_bytes(), bot.writes) == after +def test_report_recovers_via_admission_resume_once_blocking_replan_clears_after_contract_failure(steward, tmp_path, monkeypatch): + from datetime import timezone + from loopx.capabilities.periodic_report import cadence_runtime + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.configure_goal import configure_goal + import loopx.capabilities.native_chat.steward_reports as report_host + + store, runtime, bot, transport, binding, capture, _ = steward + point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) + + class CalendarClock(datetime): + @classmethod + def now(cls, tz=None): + return point.astimezone(tz) if tz else point.replace(tzinfo=None) + + monkeypatch.setattr(report_host, "datetime", CalendarClock) + monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + owner = binding["binding_id"] + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=1, execute=True, + )["ok"] + install_model(steward, tmp_path, sid) + assert finish(runtime, pump_steward_monitors(runtime)[0])["response"]["native_loopx"]["status"] == "committed" + assert transport.steward_reports.reconcile() == 1 + + point += timedelta(days=1) + runtime._steward_monitor_next_check.clear() + rejected = finish(runtime, pump_steward_monitors(runtime)[0]) + assert rejected["response"]["native_loopx"]["error_code"] == "turn_todo_selection_deferred" + _, second_window = reports.current_window(owner) + + prompts_seen: list[str] = [] + # Return invalid host result contract (`replan_required` + `outcome_gap` without typed blocked fields). + bad_vision = { + "schema_version": "goal_vision_replan_contract_v0", + "state": "active", + "vision_patch": {"vision_summary": "Keep reporting.", "acceptance_summary": "Validate full report."}, + "path_delta": { + "outcome": "replan", + "prior_assumption": "Old assumption", + "observed_reality": "Reality", + "retained": ["Same report route"], + "changed": ["Updated baseline"], + "evidence_refs": ["fixture:evidence"], + }, + } + bad_outcome = { + "result_kind": "replan_required", + "classification": "outcome_gap_without_blocked_fields", + "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_gap", + "work_evidence": ["Observed gap"], + "validation_evidence": ["Validated check"], + "path_delta_mode": "material_replan", + "agent_vision_json": json.dumps(bad_vision), + "summary": "Missing blocked fields trigger host_result_contract validation failure.", + "next_action": "Clear replan before resuming report.", + } + helper = tmp_path / "report_model.py" + helper.write_text( + _REPORT_MODEL_SOURCE.replace("def fake_report_model(", "def report_only(") + + "\ndef fake_report_model(params):\n" + + " import json, pathlib\n" + + " prompt = '\\n'.join(row.get('text', '') for row in params.get('input', []) if isinstance(row, dict))\n" + + f" pathlib.Path({str(tmp_path / 'prompts.jsonl')!r}).open('a').write(json.dumps(prompt) + '\\n')\n" + + " request = json.JSONDecoder().raw_decode(prompt.split('Turn request:', 1)[1].lstrip())[0]\n" + + " if request.get('detail_ref', {}).get('replan_action_packet'):\n" + + " return " + repr(json.dumps(bad_outcome)) + "\n" + + " return report_only(params)\n" + ) + runtime._steward_monitor_next_check.clear() + retry_turn = finish(runtime, pump_steward_monitors(runtime)[0]) + prompts_seen = [json.loads(line) for line in (tmp_path / "prompts.jsonl").read_text().splitlines() if line.strip()] + assert any("Governance contract reminder:" in text for text in prompts_seen) + replan_events = [row["payload"] for row in store.events_after(sid, retry_turn["turn_id"], None) + if row["kind"] == "native_loopx.replan_result"] + assert len(replan_events) == 1 + assert replan_events[0]["status"] == "failed" and replan_events[0]["validation_stage"] == "host_result_contract" + + # While the autonomous replan obligation stays active, pumping must NOT spin or resume. + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + + # Once the blocking replan is cleared elsewhere, one .admission-resume carrier completes the original report cleanly. + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=6, execute=True, + )["ok"] + calls = capture.read_bytes() + runtime._steward_monitor_next_check.clear() + resumed = pump_steward_monitors(runtime) + assert len(resumed) == 1 + done = finish(runtime, resumed[0]) + assert done["response"]["native_loopx"]["status"] == "committed", done + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + assert transport.steward_reports.reconcile() == 1 + assert reports.health(owner)["publication_verified"] + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == done["loopx_request"]["todo_id"]) + assert task["status"] == "done" + tail = [json.loads(line) for line in capture.read_bytes()[len(calls):].splitlines()] + assert sum(row.get("method") == "turn/start" for row in tail) == 1 + assert transport.steward_reports.reconcile() == 0 + + def embedded_report_python(tmp_path, monkeypatch): """A real child interpreter without the embedding host's LoopX bootstrap.""" import subprocess