From c388daafd459d49fd12cfdc9f983a6f3711667fc Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 04:55:48 +0800 Subject: [PATCH] fix(steward): recover report selection across python upgrades and prevent queue kill on observation monitors Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../capabilities/native_chat/session_queue.py | 50 ++++--- .../capabilities/native_chat/steward_turn.py | 44 +++++- .../collaboration/steward_managed_mode.ts | 61 ++++++-- .../steward_managed_mode.test.ts | 40 ++++++ .../control_plane_ts/steward_reports.test.ts | 110 ++++++++++++++ tests/test_native_steward_monitor.py | 135 ++++++++++++++++++ ...t_steward_report_recovery_and_validator.py | 110 +++++++++++++- 7 files changed, 514 insertions(+), 36 deletions(-) diff --git a/loopx/capabilities/native_chat/session_queue.py b/loopx/capabilities/native_chat/session_queue.py index 217e882154..fdbb0a1fda 100644 --- a/loopx/capabilities/native_chat/session_queue.py +++ b/loopx/capabilities/native_chat/session_queue.py @@ -6,6 +6,14 @@ from ...chat_store import CHAT_SESSION_MODE_ATTACHED, TERMINAL_TURN_STATES +def _is_observation_only_monitor_turn(turn: dict[str, Any] | None) -> bool: + """Return True only for native no-model monitor turns.""" + if not isinstance(turn, dict) or turn.get("loopx_execution") is not True: + return False + request = turn.get("loopx_request") + return isinstance(request, dict) and request.get("work_kind") == "monitor" + + def drain_session_queue(controller: Any, *, session_id: str, work_dir: Path, objective: str) -> None: wakeup = threading.Event() with controller.turn_release_wakeup(wakeup): @@ -76,19 +84,20 @@ def _drain_session_queue_events(controller: Any, *, session_id: str, work_dir: P # normal preparation and execution fences below. turn = active else: - with controller.lock: - attached = controller.adapters.get(session_id) - if attached is None or not attached.healthcheck(): - try: - controller._ensure_adapter( - session, - work_dir=work_dir, - objective=objective, - ) - except Exception as exc: - controller._fail_queue_preparation(session_id, active_turn_id, exc) - else: - continue + if not _is_observation_only_monitor_turn(active): + with controller.lock: + attached = controller.adapters.get(session_id) + if attached is None or not attached.healthcheck(): + try: + controller._ensure_adapter( + session, + work_dir=work_dir, + objective=objective, + ) + except Exception as exc: + controller._fail_queue_preparation(session_id, active_turn_id, exc) + else: + continue active = controller.store.load_turn(session_id, active_turn_id) if active and active.get("status") not in TERMINAL_TURN_STATES: controller.closed.wait(0.05) @@ -132,11 +141,16 @@ def _drain_session_queue_events(controller: Any, *, session_id: str, work_dir: P current = controller.store.load_session(session_id) if current is None or current.get("active_turn_id") != turn_id: return - adapter = controller._ensure_adapter( - current, - work_dir=work_dir, - objective=objective, - ) + if _is_observation_only_monitor_turn(turn): + from .steward_monitor import StewardObservationAdapter + + adapter: Any = StewardObservationAdapter(current.get("upstream_thread_id")) + else: + adapter = controller._ensure_adapter( + current, + work_dir=work_dir, + objective=objective, + ) if (controller.store.load_session(session_id) or {}).get("active_turn_id") != turn_id: return except Exception as exc: diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index ebb34d1565..8d7939e92b 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -9,10 +9,12 @@ from collections.abc import Mapping, Sequence from contextlib import contextmanager, ExitStack, nullcontext +from datetime import datetime, timezone import hashlib import json from pathlib import Path import re +import time from typing import Any from ...control_plane.goals.activation import goal_is_stopped @@ -20,6 +22,16 @@ from ...file_lock import exclusive_file_lock from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode +MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS = 300 +_WORKER_EXITED_ERROR = ( + "The LoopX Chat execution worker exited before this turn finished. " + "Your conversation is saved; execution may have occurred. This turn was not retried automatically." +) +_POST_CONTEXT_WATCH_ERRORS = frozenset({ + "steward monitor external signal source temporarily unavailable", + "disk I/O error", + _WORKER_EXITED_ERROR, +}) _USER_ACTION_DEDUPE_KEY = re.compile(r"^[a-z0-9][a-z0-9._:@/-]{0,119}$") _REPORT_VALIDATOR_PREFIX = ( "from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; " @@ -27,6 +39,18 @@ ) +def _iso_timestamp_seconds(value: Any) -> float | None: + if not isinstance(value, str) or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + def _same_report_validator_command(frozen_cmd: Any, request_cmd: Any) -> bool: """Allow report validation commands whose only drift is the host Python path.""" return (isinstance(frozen_cmd, list) and isinstance(request_cmd, list) @@ -209,18 +233,27 @@ def monitor_restart_basis(self, request: Mapping[str, Any]) -> dict[str, Any]: "event_kinds": [row["kind"] for row in events], "poll_absent": not present, "contexts": [row["payload"] for row in events if row["kind"] == "manager.context"], "phases": [row["payload"] for row in events if row["kind"] == "agent.phase"], - "old_worker_excluded": True, "lease_available": available}} + "old_worker_excluded": True, "lease_available": available, + "observed_at": time.time()}} @staticmethod - def monitor_restart_request(original: Mapping[str, Any]) -> dict[str, Any] | None: + def monitor_restart_request(original: Mapping[str, Any], *, now: float | None = None) -> dict[str, Any] | None: """Prepare correlation only; current typed admission decides recovery.""" prior = original.get("loopx_request") or {} error = original.get("error_code") if (original.get("status") != "failed" or error not in {"server_restarted", "runtime_error", "runtime_unavailable"} - or prior.get("work_kind") != "monitor" - or (prior.get("monitor_restart_recovery") is not None and error != "server_restarted")): + or prior.get("work_kind") != "monitor"): return None + if prior.get("monitor_restart_recovery") is not None and error != "server_restarted": + post_context_error = error == "runtime_unavailable" or ( + error == "runtime_error" and original.get("error") in _POST_CONTEXT_WATCH_ERRORS + ) + completed_ts = _iso_timestamp_seconds(original.get("completed_at")) + current_ts = time.time() if now is None else now + if (not post_context_error or completed_ts is None + or current_ts - completed_ts < MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS): + return None return {**prior, "monitor_restart_recovery": {"original_turn_id": original["turn_id"]}} def monitor_restart_carrier(self, original: dict[str, Any], client_id: str) -> dict[str, Any] | None: @@ -248,7 +281,6 @@ def report_quota_basis(self, request: Mapping[str, Any]) -> dict[str, Any]: from ...control_plane.turn_driver import inspect_loopx_turn_journal from ..periodic_report.private_generation import attempt_directory from .steward_reports import report_service - import time actor = request["actor"] original = self.store.load_turn(actor["session_id"], recovery["original_turn_id"]) result = ((original or {}).get("response") or {}).get("native_loopx") or {} @@ -615,6 +647,8 @@ def _prepare_submission(self, binding: dict[str, Any] | None, *, turn_instance_i return {"turn": existing, "quota_probe": {"original": existing, "binding": binding}} quota_request = quota_observation["request"] if quota_request is not None: + if _same_report_validator_command(quota_request.get("validation_command"), request.get("validation_command")): + quota_request = {**quota_request, "validation_command": request["validation_command"]} request = quota_request client_id = native_steward_client_id(actor, request["turn_instance_id"]) existing = None diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index 0ace864365..d56877dcaf 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -18,11 +18,42 @@ import {projectTurnBoundary, turnPlanWorkSource} from "../quota/turn_envelope.ts import {projectGoalCreationExecutionRequest} from "../goals/creation_execution.ts"; /** A stopped observation may get a new Chat carrier for the same native - * identity. After a recovery starts, only another proven process restart can - * continue it. The host reads the stopped worker lock, journals and native poll - * receipts. Normal admission still checks ownership, deadline, permissions and user work. */ + * identity. After a recovery starts, only another proven process restart or a + * cooldown-bounded pre-poll transient failure can continue it. The host reads + * the stopped worker lock, journals and native poll receipts. Normal admission + * still checks ownership, deadline, permissions and user work. */ const WORKER_EXITED_ERROR = "The LoopX Chat execution worker exited before this turn finished. Your conversation is saved; execution may have occurred. This turn was not retried automatically."; +const MONITOR_RECOVERY_RETRY_COOLDOWN_MS = 300_000; +const REPORT_VALIDATOR_PREFIX = + "from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; " + + "validate_private_generation(Path("; + +function sameReportValidatorCommand(frozenCmd: unknown, requestCmd: unknown): boolean { + return Array.isArray(frozenCmd) && Array.isArray(requestCmd) + && frozenCmd.length === 3 && requestCmd.length === 3 + && frozenCmd.every(part => typeof part === "string" && part.length > 0 && !part.includes("\0")) + && requestCmd.every(part => typeof part === "string" && part.length > 0 && !part.includes("\0")) + && frozenCmd[1] === "-c" && requestCmd[1] === "-c" + && frozenCmd[2] === requestCmd[2] + && frozenCmd[2].startsWith(REPORT_VALIDATOR_PREFIX) + && frozenCmd[2].endsWith("))"); +} + +function sameReportRequestField(key: string, priorValue: unknown, requestValue: unknown): boolean { + if (key === "validation_command" && sameReportValidatorCommand(priorValue, requestValue)) { + return true; + } + return JSON.stringify(priorValue) === JSON.stringify(requestValue); +} + +function sameReportRequest(priorValue: unknown, request: JsonObject): boolean { + const prior = jsonObject(priorValue); + if (!prior) return false; + const keys = Object.keys(request); + return Object.keys(prior).length === keys.length + && keys.every(key => Object.hasOwn(prior, key) && sameReportRequestField(key, prior[key], request[key])); +} function hasSingleManagerContextEvidence(basis: JsonObject, prior: JsonObject, events: unknown): boolean { if (requireJsonObject(prior.actor, "original watch actor").schema_version !== "loopx_steward_managed_actor_v0" @@ -80,8 +111,9 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1; // Post-context monitor failures (external signal outage, daemon worker exit, // storage I/O error, or runtime unavailability) before any poll receipt or - // upstream model turn may recover at most once when strict single-context - // and zero-effect journal proofs hold; pre-context unknown worker launches remain closed. + // upstream model turn may recover once immediately and retry only after a + // bounded cooldown when strict single-context and zero-effect journal proofs + // hold; pre-context unknown worker launches remain closed. const postContextWatchRecoverable = hasSingleManagerContextEvidence(basis, prior, events) && ( original.error_code === "runtime_unavailable" || original.error_code === "runtime_error" && ( @@ -90,6 +122,12 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se || original.error === WORKER_EXITED_ERROR ) ); + const completedAtMs = typeof original.completed_at === "string" ? Date.parse(original.completed_at) : NaN; + const observedAtMs = typeof basis.observed_at === "number" && Number.isFinite(basis.observed_at) + ? basis.observed_at * 1000 : NaN; + const chainedPostContextWatchRecoverable = postContextWatchRecoverable + && Number.isFinite(completedAtMs) && Number.isFinite(observedAtMs) + && observedAtMs - completedAtMs >= MONITOR_RECOVERY_RETRY_COOLDOWN_MS; if (Object.keys(recovery).join() !== "original_turn_id" || recovery.original_turn_id !== original.turn_id || typeof original.turn_id !== "string" || !/^[a-f0-9]{32}$/.test(original.turn_id) || original.session_id !== session.session_id @@ -98,7 +136,7 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se || original.origin !== "native_loopx" || original.loopx_execution !== true || original.response !== null || original.upstream_turn_id !== null || original.delta_count !== 0 || prior.work_kind !== "monitor" || request.work_kind !== "monitor" || request.validation_command !== null - || (predecessor !== null && (original.error_code !== "server_restarted" + || (predecessor !== null && (!(original.error_code === "server_restarted" || chainedPostContextWatchRecoverable) || Object.keys(predecessor).join() !== "original_turn_id" || typeof predecessor.original_turn_id !== "string" || !/^[a-f0-9]{32}$/.test(predecessor.original_turn_id) || predecessor.original_turn_id === original.turn_id)) @@ -154,9 +192,9 @@ function admitReportQuotaRecovery(params: JsonObject, request: JsonObject, sessi || original.origin !== "native_loopx" || original.loopx_execution !== true || original.status !== "completed" || original.delta_count !== 0 || typeof original.upstream_turn_id !== "string" || !original.upstream_turn_id || Object.keys(prior).length !== keys.length - || keys.some(key => key === "turn_instance_id" + || keys.some(key => !Object.hasOwn(prior, key) || (key === "turn_instance_id" ? request[key] !== String(prior[key]) + ".quota-recovery" - : JSON.stringify(request[key]) !== JSON.stringify(prior[key])) + : !sameReportRequestField(key, prior[key], request[key]))) || result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false || result.mode !== "run_once" || result.status !== "failed" || result.result_kind !== "host_failure" || failure.schema_version !== "loopx_turn_host_failure_v0" || failure.attempt !== 1 @@ -194,8 +232,7 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s if (original.session_id !== sessionId || original.origin !== "native_loopx" || original.loopx_execution !== true || original.status !== "completed" || original.delta_count !== 0 || original.upstream_turn_id !== null - || Object.keys(prior).length !== Object.keys(request).length - || Object.keys(request).some(key => JSON.stringify(prior[key]) !== JSON.stringify(request[key])) + || !sameReportRequest(prior, request) || result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false || result.mode !== "run_once" || result.effects_scope !== "current_invocation" || result.error_code !== "turn_todo_selection_deferred" || result.resume_turn_key != null @@ -228,7 +265,7 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s || !/^[a-f0-9]{32}$/.test(observed.turn_id) || observed.turn_id === original.turn_id || observed.origin !== "native_loopx" || observed.loopx_execution !== true || observed.error_code != null || observed.error != null - || JSON.stringify(observed.loopx_request) !== JSON.stringify(request) + || !sameReportRequest(observed.loopx_request, request) || !(runningHere || observed.status === "completed" && JSON.stringify(requireJsonObject(observed.response, "probe response").native_loopx) === JSON.stringify(result)) || journal.schema_version !== "loopx_turn_journal_v0" || !ENVELOPED_SHA256_PATTERN.test(String(journal.turn_key)) @@ -638,7 +675,7 @@ export function admitStewardManagedTurn(params: JsonObject): JsonObject { || original.loopx_execution !== true || original.status !== "completed" || original.delta_count !== 0 || original.upstream_turn_id !== null || Object.keys(prior).length !== keys.length - || keys.some(key => JSON.stringify(prior[key]) !== JSON.stringify(request[key])) + || keys.some(key => !Object.hasOwn(prior, key) || !sameReportRequestField(key, prior[key], request[key])) || result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false || result.mode !== "run_once" || result.status !== "failed" || result.result_kind !== "host_failure" || result.reason !== "first-party Host runtime rejected: stale_goal_authority" diff --git a/tests/control_plane_ts/steward_managed_mode.test.ts b/tests/control_plane_ts/steward_managed_mode.test.ts index 0d118ad3fe..20ef5b8c32 100644 --- a/tests/control_plane_ts/steward_managed_mode.test.ts +++ b/tests/control_plane_ts/steward_managed_mode.test.ts @@ -434,3 +434,43 @@ test("zero-effect worker exit, storage I/O, and runtime unavailability monitors } } }); + +test("chained post-context transient watch carriers recover only after cooldown while scope and identity refusals remain one-shot", () => { + const {params, goal} = fixture(); + const actor = bindStewardManagedActor({...params, goal}); + const baseRequest = {operation: "native_turn", host: "codex-app-server", actor, + work_kind: "monitor", turn_instance_id: "monitor.chained", todo_id: "todo_chained_watch", validation_command: null, + monitor_deadline: "2026-11-20T07:45:00Z", monitor_lease_version: 0}; + const completedAt = "2026-11-20T07:45:00Z"; + const completedSec = Date.parse(completedAt) / 1000; + const monitor = {todo_id: "todo_chained_watch", task_class: "continuous_monitor", + target_key: "steward-portfolio-watch", watch_only: true, claimed_by: "steward", status: "open", + next_due_at: baseRequest.monitor_deadline}; + const carrier = {turn_id: "6".repeat(32), session_id: params.session.session_id, + status: "failed", error_code: "runtime_unavailable", + error: "The Agent runtime could not be prepared.", + origin: "native_loopx", loopx_execution: true, upstream_turn_id: null, response: null, + delta_count: 0, completed_at: completedAt, + loopx_request: {...baseRequest, monitor_restart_recovery: {original_turn_id: "5".repeat(32)}}}; + const basis = {original: carrier, contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + phases: [{phase: "manager_context"}], + event_kinds: ["turn.queued", "agent.phase", "manager.context", "turn.failed"], + poll_absent: true, old_worker_excluded: true, lease_available: true, observed_at: completedSec + 300}; + const input = {...params, goal, goal_active: true, binding: {...params.binding, managed_actor: actor}, + request: {...baseRequest, monitor_restart_recovery: {original_turn_id: carrier.turn_id}}, + monitor_todo: monitor, monitor_restart_basis: basis}; + assert.equal(admitStewardManagedTurn(input).observation_only, true); + + for (const badBasis of [ + {observed_at: completedSec + 299}, + {observed_at: undefined}, + {original: {...carrier, completed_at: undefined}}, + {original: {...carrier, error_code: "runtime_error", error: "the portfolio observation omitted authorized Goals", + contexts: [{schema_version: "manager_turn_context_v1", goals: [], warnings: ["external_authorization_changed"], + coverage: {discovered: null, verified: 0, complete: false}}]}, observed_at: completedSec + 600}, + {original: {...carrier, error_code: "runtime_error", error: "the selected App identity could not be verified"}, + contexts: [], event_kinds: ["turn.queued", "agent.phase", "turn.failed"], observed_at: completedSec + 600}, + ]) { + assert.throws(() => admitStewardManagedTurn({...input, monitor_restart_basis: {...basis, ...badBasis}})); + } +}); diff --git a/tests/control_plane_ts/steward_reports.test.ts b/tests/control_plane_ts/steward_reports.test.ts index 84ec963790..83abe57a6d 100644 --- a/tests/control_plane_ts/steward_reports.test.ts +++ b/tests/control_plane_ts/steward_reports.test.ts @@ -540,3 +540,113 @@ test("only a proven original pre-execution replan refusal admits one correlated assert.throws(() => admitStewardManagedTurn({...input, report_selection_recovery: {...proof, probe: badProbe}})); } }); + +test("report recovery paths tolerate host Python executable drift in validation_command and reject target drift", () => { + const p = original(); + const folder = "/tmp/example-goal/periodic_reports/win-1"; + const snippet = "from pathlib import Path; from loopx.capabilities.periodic_report.private_generation " + + `import validate_private_generation; validate_private_generation(Path('${folder}'))`; + const oldCmd = ["/opt/release-1/bin/python", "-c", snippet]; + const newCmd = ["/opt/release-2/bin/python3", "-c", snippet]; + const badTargetCmd = ["/opt/release-2/bin/python3", "-c", snippet.replace(folder, "/tmp/other/win-2")]; + const badFlagCmd = ["/opt/release-2/bin/python3", "-m", snippet]; + + const basePrior = {operation: "native_turn", host: "codex-app-server", actor: p.actor, + turn_instance_id: "report.cadence_original", todo_id: "todo_report", validation_command: oldCmd, + work_kind: "report", report_window_id: "cadence_original", report_subscription_revision: "sha256:original"}; + const reportMeta = { + report_window: {window_id: basePrior.report_window_id, subscription_revision: basePrior.report_subscription_revision, + goal_id: p.actor.goal_id, agent_id: p.actor.agent_id}, + report_subscription: {enabled: true, profile_preset: "portfolio-progress", + route_ref: `steward-private:${p.binding.binding_id}:${p.source}`, effective_revision: basePrior.report_subscription_revision}, + report_todo: {todo_id: basePrior.todo_id, target_key: basePrior.report_window_id, + claimed_by: "steward", action_kind: "generate_periodic_report", status: "open"}, + }; + + // 1. report_selection_recovery with cleared validation_failed probe from an older Python release. + const effects = {host_invoked: false, state_written: false, quota_spent: false, scheduler_acknowledged: false}; + const refusal = {schema_version: "loopx_turn_selection_rejection_v0", source: "quota.should-run", + requested_todo_id: basePrior.todo_id, state: "deferred", reason_code: "autonomous_replan", + recovery_action: "reenter_guard_without_selection", delivery_preemptions: ["autonomous_replan", "delivery_not_allowed"], + status_health_ok: true, contract_error_count: 0}; + const selResult = {schema_version: "loopx_turn_execution_v0", mode: "run_once", ok: false, + error_code: "turn_todo_selection_deferred", effects_scope: "current_invocation", effects, selection_rejection: refusal}; + const selOriginal = {session_id: p.session.session_id, status: "completed", origin: "native_loopx", loopx_execution: true, + delta_count: 0, upstream_turn_id: null, loopx_request: basePrior, response: {native_loopx: selResult}}; + const probeKey = "sha256:" + "e".repeat(64); + const selProbe = { + turn: {...selOriginal, turn_id: "e".repeat(32), delta_count: 8, upstream_turn_id: "probe-model-turn"}, + journal: {schema_version: "loopx_turn_journal_v0", goal_id: p.actor.goal_id, turn_key: probeKey, + status: "failed", validation_stage: "host_result_contract", + host: {kind: "codex-app-server", executable: "built-in"}, host_attempt_count: 1, + result_kind: "validation_failed", completed_phases: ["host_execute", "typed_result"], + receipt: {ok: true, failed_phase: "validation", status: "failed", + commit_eligibility: {writeback: false, quota_spend: false, scheduler_ack: false}}, + plan: {route: {kind: "replan_required", effective_action: "autonomous_replan_required"}, + turn_envelope: {goal_id: p.actor.goal_id, agent_id: p.actor.agent_id, effective_action: "autonomous_replan_required"}, + transaction: {settlement_plan: {identity: {goal_id: p.actor.goal_id, agent_id: p.actor.agent_id, + binding_kind: "autonomous_replan", turn_instance_id: basePrior.turn_instance_id + ".replan"}}}}}, + inspection: {schema_version: "loopx_turn_journal_inspection_v1", ok: true, + goal_matches: true, owner_matches: true, turn_key_matches: true, journal_consistent: true, + journal_status: "failed", recorded_effects: {host_invoked: true, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + recovery_decision: {action: "return_existing", reinvoke_host: false}}, + current_guard: {effective_action: "normal_run", replan_action_packet: null}, + }; + const selInput = {...p, ...reportMeta, request: {...basePrior, validation_command: newCmd}, + report_selection_recovery: {original: selOriginal, generation_present: false, probe: selProbe}}; + assert.equal(admitStewardManagedTurn(selInput).observation_only, false); + for (const badCmd of [badTargetCmd, badFlagCmd]) { + assert.throws(() => admitStewardManagedTurn({...selInput, request: {...selInput.request, validation_command: badCmd}})); + } + + // 2. report_quota_recovery across host Python upgrade. + const quotaResult = {schema_version: "loopx_turn_execution_v0", ok: false, mode: "run_once", + status: "failed", result_kind: "host_failure", host_failure: {schema_version: "loopx_turn_host_failure_v0", attempt: 1}, + effects: {host_invoked: true, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + receipt: {ok: true, failed_phase: "host_execute", status: "failed", completed_phases: [], + lineage: {goal_id: p.actor.goal_id, agent_id: p.actor.agent_id, todo_id: basePrior.todo_id}}}; + const quotaSource = {turn_id: "1".repeat(32), session_id: p.session.session_id, status: "completed", + origin: "native_loopx", loopx_execution: true, delta_count: 0, upstream_turn_id: "quota-native-turn", + loopx_request: basePrior, response: {native_loopx: quotaResult}}; + const quotaObs = {thread_id: "original-native-thread", turn_id: quotaSource.upstream_turn_id, + only_original_user_item: true, codex_error_info: "usageLimitExceeded", quota_available: true, quota_observed_at: 100}; + const quotaInput = {...p, ...reportMeta, session: {...p.session, upstream_thread_id: quotaObs.thread_id}, + request: {...basePrior, validation_command: newCmd, turn_instance_id: basePrior.turn_instance_id + ".quota-recovery", + report_quota_recovery: {original_turn_id: quotaSource.turn_id, native_observation: quotaObs}}, + report_quota_basis: {original: quotaSource, generation_present: false, observed_at: 101, + inspection: {schema_version: "loopx_turn_journal_inspection_v1", ok: true, + goal_matches: true, owner_matches: true, turn_key_matches: true, journal_consistent: true, + journal_status: "failed", completed_phases: [], + recorded_effects: {host_invoked: null, state_written: false, quota_spent: false, scheduler_acknowledged: false}}}}; + assert.equal(admitStewardManagedTurn(quotaInput).observation_only, false); + assert.throws(() => admitStewardManagedTurn({ + ...quotaInput, request: {...quotaInput.request, validation_command: badTargetCmd}, + })); + + // 3. report_launch_recovery across host Python upgrade. + const launchKey = "sha256:" + "f".repeat(64); + const launchResult = {schema_version: "loopx_turn_execution_v0", mode: "run_once", ok: false, + status: "failed", result_kind: "host_failure", reason: "first-party Host runtime rejected: stale_goal_authority", + host_failure: {schema_version: "loopx_turn_host_failure_v0", kind: "contract_rejected", attempt: 1, retryable: false}, + effects: {host_invoked: true, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + resume_turn_key: launchKey}; + const launchSource = {session_id: p.session.session_id, status: "completed", origin: "native_loopx", loopx_execution: true, + delta_count: 0, upstream_turn_id: null, loopx_request: basePrior, response: {native_loopx: launchResult}}; + const launchInput = {...p, ...reportMeta, + request: {...basePrior, validation_command: newCmd, resume_turn_key: launchKey}, + report_launch_recovery: { + original: launchSource, turn_key: launchKey, host_attempt_count: 1, generation_present: false, + inspection: {schema_version: "loopx_turn_journal_inspection_v1", ok: true, + goal_matches: true, owner_matches: true, turn_key_matches: true, journal_consistent: true, + journal_status: "failed", completed_phases: [], + recorded_effects: {host_invoked: null, state_written: false, quota_spent: false, scheduler_acknowledged: false}, + recovery_decision: {action: "continue", reinvoke_host: true, resume_from: "host_execute"}}, + journal: {status: "failed", result_kind: "host_failure", reason: launchResult.reason, turn_key: launchKey, + host_attempt_count: 1, completed_phases: [], host_failure: launchResult.host_failure, + receipt: {failed_phase: "host_execute"}}, + }}; + assert.equal(admitStewardManagedTurn(launchInput).observation_only, false); + assert.throws(() => admitStewardManagedTurn({ + ...launchInput, request: {...launchInput.request, validation_command: badTargetCmd}, + })); +}); diff --git a/tests/test_native_steward_monitor.py b/tests/test_native_steward_monitor.py index ca7463a016..9fa118d700 100644 --- a/tests/test_native_steward_monitor.py +++ b/tests/test_native_steward_monitor.py @@ -1442,3 +1442,138 @@ def test_transient_zero_effect_monitor_failures_recover_once( carrier = store.load_turn(sid, done["turn_id"]) assert carrier is not None and carrier["client_turn_id"].endswith(".restart-recovery") assert StewardTurnHost.monitor_restart_request(carrier) is None + + +def test_chained_transient_monitor_recovery_waits_for_cooldown_then_recovers( + steward, tmp_path, monkeypatch, +): + from loopx.capabilities.native_chat.steward_turn import MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS, StewardTurnHost + + store, runtime, *_ = steward + sid, actor, monitor = prepared(steward, tmp_path, monkeypatch, "sqlite") + default_watch = monitor.current(actor) + todo_id = default_watch["todo_id"] + due = (datetime.now(timezone.utc) - timedelta(seconds=2)).isoformat() + edit_local_watch(runtime, actor, todo_id, "--next-due-at", due) + + with monkeypatch.context() as paused: + paused.setattr(runtime, "_start_accepted_turn_worker", lambda **_kw: True) + runtime._steward_monitor_next_check.clear() + initial = pump_steward_monitors(runtime)[0] + store.append_event(sid, initial["turn_id"], kind="agent.phase", payload={"phase": "manager_context"}) + store.append_event(sid, initial["turn_id"], kind="manager.context", + payload={"schema_version": "manager_turn_context_v1", "goals": []}) + runtime._fail_turn(sid, initial["turn_id"], "runtime_error", "disk I/O error", + status="failed", expected_statuses={"queued", "starting", "running"}) + + with monkeypatch.context() as paused: + paused.setattr(runtime, "_start_accepted_turn_worker", lambda **_kw: True) + runtime._steward_monitor_next_check.clear() + carrier = pump_steward_monitors(runtime)[0] + store.append_event(sid, carrier["turn_id"], kind="agent.phase", payload={"phase": "manager_context"}) + store.append_event(sid, carrier["turn_id"], kind="manager.context", + payload={"schema_version": "manager_turn_context_v1", "goals": []}) + runtime._fail_turn(sid, carrier["turn_id"], "runtime_unavailable", + "The Agent runtime could not be prepared.", + status="failed", expected_statuses={"queued", "starting", "running"}) + + # Immediately after carrier failure (< cooldown), pumping stays idle instead of spinning. + runtime._steward_monitor_next_check.clear() + assert pump_steward_monitors(runtime) == [] + + # After cooldown elapses, a bounded .restart-recovery. carrier recovers and polls. + aged_completed = ( + datetime.now(timezone.utc) - timedelta(seconds=MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS + 5) + ).isoformat().replace("+00:00", "Z") + store.update_turn(sid, carrier["turn_id"], completed_at=aged_completed) + req = StewardTurnHost.monitor_restart_request(store.load_turn(sid, carrier["turn_id"])) + assert req is not None and req["monitor_restart_recovery"]["original_turn_id"] == carrier["turn_id"] + + runtime._steward_monitor_next_check.clear() + chained = pump_steward_monitors(runtime) + assert len(chained) == 1 + assert chained[0]["client_turn_id"].endswith(".restart-recovery." + carrier["turn_id"]) + assert chained[0]["loopx_request"]["monitor_restart_recovery"]["original_turn_id"] == carrier["turn_id"] + done = finish(runtime, chained[0]) + assert done["status"] == "completed", done + assert local_watch_row(runtime, actor, todo_id)["next_due_at"] != due + + +def test_session_queue_drain_preserves_active_and_queued_observation_only_monitors( + steward, tmp_path, monkeypatch, +): + from loopx.chat_agent import CodexChatAgentError + import json + import threading + + store, runtime, *_ = steward + sid, actor, monitor = prepared(steward, tmp_path, monkeypatch, "sqlite") + default_watch = monitor.current(actor) + todo_id = default_watch["todo_id"] + due = (datetime.now(timezone.utc) - timedelta(seconds=2)).isoformat() + edit_local_watch(runtime, actor, todo_id, "--next-due-at", due) + + # Create a crash-stranded queued observation-only monitor turn (no _acceptance, status=queued). + with monkeypatch.context() as paused: + paused.setattr(runtime, "_start_accepted_turn_worker", lambda **_kw: True) + runtime._steward_monitor_next_check.clear() + queued_watch = pump_steward_monitors(runtime)[0] + turn_id = queued_watch["turn_id"] + store.update_session(sid, active_turn_id=None) + queued_doc = store.load_turn(sid, turn_id) + assert queued_doc is not None + queued_doc.pop("_acceptance", None) + queued_doc["status"] = "queued" + queued_doc["started_at"] = None + store._turn_path(sid, turn_id).write_text(json.dumps(queued_doc)) + + adapter_calls = 0 + fail_calls = 0 + + def fail_ensure_adapter(*_args, **_kwargs): + nonlocal adapter_calls + adapter_calls += 1 + raise CodexChatAgentError("Native account API transport is unavailable.") + + orig_fail = runtime._fail_queue_preparation + + def spy_fail(*args, **kwargs): + nonlocal fail_calls + fail_calls += 1 + return orig_fail(*args, **kwargs) + + # First verify draining while monitor is already running does not invoke _ensure_adapter or fail it, + # and second verify draining a crash-stranded queued monitor uses StewardObservationAdapter cleanly. + store.update_session(sid, active_turn_id=turn_id) + store.update_turn(sid, turn_id, status="running", started_at=queued_doc["created_at"]) + monkeypatch.setattr(runtime, "_ensure_adapter", fail_ensure_adapter) + monkeypatch.setattr(runtime, "_fail_queue_preparation", spy_fail) + + def complete_soon(): + time.sleep(0.12) + store.update_turn(sid, turn_id, status="completed", response={"message": "ok", "proposals": []}) + + t = threading.Thread(target=complete_soon, daemon=True) + t.start() + runtime._drain_session_queue(session_id=sid, work_dir=tmp_path, objective="Monitor") + t.join(timeout=2) + assert adapter_calls == 0 + assert fail_calls == 0 + assert store.load_turn(sid, turn_id)["status"] == "completed" + + # Now verify crash-stranded queued monitor branch invokes _start_accepted_turn_worker with StewardObservationAdapter. + dispatched: list[object] = [] + store.update_session(sid, active_turn_id=None) + queued_doc["status"] = "queued" + queued_doc["started_at"] = None + store._turn_path(sid, turn_id).write_text(json.dumps(queued_doc)) + + def fake_run_turn(*, adapter, turn_id: str, **_kw): + dispatched.append(adapter) + store.update_turn(sid, turn_id, status="completed", response={"message": "ok", "proposals": []}) + store.update_session(sid, active_turn_id=None) + + monkeypatch.setattr(runtime, "_run_turn", fake_run_turn) + runtime._drain_session_queue(session_id=sid, work_dir=tmp_path, objective="Monitor") + assert adapter_calls == 0 and fail_calls == 0 + assert len(dispatched) == 1 and getattr(dispatched[0], "observation_only", False) is True diff --git a/tests/test_steward_report_recovery_and_validator.py b/tests/test_steward_report_recovery_and_validator.py index d602e38bb9..70e74cd189 100644 --- a/tests/test_steward_report_recovery_and_validator.py +++ b/tests/test_steward_report_recovery_and_validator.py @@ -6,7 +6,11 @@ import pytest from loopx.capabilities.native_chat.steward_monitor import pump_steward_monitors -from loopx.capabilities.native_chat.steward_turn import StewardTurnHost, _same_report_validator_command +from loopx.capabilities.native_chat.steward_turn import ( + StewardTurnHost, + _same_report_validator_command, + native_steward_client_id, +) from loopx.capabilities.periodic_report import cadence_runtime from loopx.capabilities.periodic_report.cadence_journal import read_cadence_journal from loopx.capabilities.periodic_report.incremental import read_periodic_report_publication_cursor @@ -171,3 +175,107 @@ def mid_turn_failure(adapter, _prompt, event_sink): assert transport.steward_reports.reconcile() == 1 cursor = read_periodic_report_publication_cursor(runtime_root=reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"]) assert cursor["covered_until"] == old["window"]["next_due_at"] + + +def test_selection_recovery_admission_resume_tolerates_validator_python_upgrade( + steward, tmp_path, monkeypatch, caplog, +): + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.configure_goal import configure_goal + from test_native_steward_reports import _REPORT_MODEL_SOURCE + + store, runtime, _, transport, binding, _, _ = steward + point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) + + class CalendarClock(datetime): + @classmethod + def now(cls, tz=None): + return point.astimezone(tz) if tz else point.replace(tzinfo=None) + + monkeypatch.setattr(report_host, "datetime", CalendarClock) + monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + owner = binding["binding_id"] + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=1, execute=True, + )["ok"] + install_model(steward, tmp_path, sid) + assert finish(runtime, pump_steward_monitors(runtime)[0])["response"]["native_loopx"]["status"] == "committed" + assert transport.steward_reports.reconcile() == 1 + + point += timedelta(days=1) + runtime._steward_monitor_next_check.clear() + rejected = finish(runtime, pump_steward_monitors(runtime)[0]) + assert rejected["response"]["native_loopx"]["error_code"] == "turn_todo_selection_deferred" + + bad_vision = { + "schema_version": "goal_vision_replan_contract_v0", + "state": "active", + "vision_patch": {"vision_summary": "Keep reporting.", "acceptance_summary": "Validate full report."}, + "path_delta": { + "outcome": "replan", + "prior_assumption": "Old assumption", + "observed_reality": "Reality", + "retained": ["Same report route"], + "changed": ["Updated baseline"], + "evidence_refs": ["fixture:evidence"], + }, + } + bad_outcome = { + "result_kind": "replan_required", + "classification": "outcome_gap_without_blocked_fields", + "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_gap", + "work_evidence": ["Observed gap"], + "validation_evidence": ["Validated check"], + "path_delta_mode": "material_replan", + "agent_vision_json": json.dumps(bad_vision), + "summary": "Missing blocked fields trigger host_result_contract validation failure.", + "next_action": "Clear replan before resuming report.", + } + helper = tmp_path / "report_model.py" + helper.write_text( + _REPORT_MODEL_SOURCE.replace("def fake_report_model(", "def report_only(") + + "\ndef fake_report_model(params):\n" + + " import json\n" + + " prompt = '\\n'.join(row.get('text', '') for row in params.get('input', []) if isinstance(row, dict))\n" + + " request = json.JSONDecoder().raw_decode(prompt.split('Turn request:', 1)[1].lstrip())[0]\n" + + " if request.get('detail_ref', {}).get('replan_action_packet'):\n" + + " return " + repr(json.dumps(bad_outcome)) + "\n" + + " return report_only(params)\n" + ) + runtime._steward_monitor_next_check.clear() + probe = finish(runtime, pump_steward_monitors(runtime)[0]) + + # Simulate host Python interpreter path upgrade on both the original refused turn and probe turn. + for row_id in (rejected["turn_id"], probe["turn_id"]): + row = store.load_turn(sid, row_id) + assert row is not None + req = json.loads(json.dumps(row["loopx_request"])) + req["validation_command"][0] = "/tmp/previous-release/bin/python" + row["loopx_request"] = req + store._turn_path(sid, row_id).write_text(json.dumps(row)) + + assert configure_goal( + registry_path=runtime.registry_path, goal_id=actor["goal_id"], + execution_replan_after_turns=6, execute=True, + )["ok"] + runtime._steward_monitor_next_check.clear() + with caplog.at_level(logging.WARNING): + resumed = pump_steward_monitors(runtime) + assert "An original steward report is unavailable" not in caplog.text + assert len(resumed) == 1 + expected_client_id = native_steward_client_id( + actor, + rejected["loopx_request"]["turn_instance_id"] + ".admission-resume", + ) + assert resumed[0]["client_turn_id"] == expected_client_id + done = finish(runtime, resumed[0]) + assert done["response"]["native_loopx"]["status"] == "committed", done + runtime._steward_monitor_next_check.clear() + assert not pump_steward_monitors(runtime) + assert transport.steward_reports.reconcile() == 1 + assert reports.health(owner)["publication_verified"] + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == done["loopx_request"]["todo_id"]) + assert task["status"] == "done"