diff --git a/docs/product/periodic-reports.md b/docs/product/periodic-reports.md index 9cfcdb26c0..9777864ac6 100644 --- a/docs/product/periodic-reports.md +++ b/docs/product/periodic-reports.md @@ -735,6 +735,19 @@ deadline is not a promise that the held window can run. This observation grants no cancellation, retirement or retry. A correlated quota continuation that succeeds clears the current failure projection while retaining its failed predecessor. +When a terminal Codex failure with only the original user input is verified, the +report attempt records that observation under its generation lock. The record +binds the original Chat and Codex Turns, the report window and subscription +revision, and the original native result digest. A later independent period +queries live Codex history first after the existing journal, generation and +delivery checks pass: readable counter-evidence durably invalidates that Turn's +retained proof under the generation lock and holds the original window across +subsequent history outages unless fresh same-Turn evidence explains the +contradiction, and an un-contradicted retained proof is used only when Codex +history is unavailable. Quota continuation for the original period still +requires a live observation. Missing or mismatched evidence still leaves a +window held when the original failure cannot be verified. + | Cancelled failure on the original settings page | Narrow retained-failure state | | --- | --- | | ![Cancelled generation is not completed or delivered](../assets/personal-workspace/steward-reports/failed-window-cancelled.png) | ![Retained cancelled window in a narrow viewport](../assets/personal-workspace/steward-reports/cancelled-window-narrow.png) | diff --git a/loopx/capabilities/native_chat/codex_adapter.py b/loopx/capabilities/native_chat/codex_adapter.py index 65a10d246b..f2f5ba7ccd 100644 --- a/loopx/capabilities/native_chat/codex_adapter.py +++ b/loopx/capabilities/native_chat/codex_adapter.py @@ -504,7 +504,13 @@ def _loopback_relay_transport_ready(runtime_root: Path, codex_home: Path, consum return False, now -def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dict[str, Any] | None: +def failed_request_readback( + session: CodexChatAgentSession, + turn_id: str, + *, + include_contradiction: bool = False, + contradiction_detail: bool = False, +) -> dict[str, Any] | None: """Observe a historical pre-execution failure without resuming it. Unlike a same-worker retry, a migration may inspect an older failure. @@ -516,16 +522,36 @@ def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dic """ deadline = time.monotonic() + session.response_timeout_sec + def contradicted(*, in_progress: bool = False, target_in_progress: bool = False) -> dict[str, Any] | None: + if not include_contradiction: + return None + if contradiction_detail and in_progress: + return { + "contradicted": True, + "in_progress": True, + **({"target_in_progress": True} if target_in_progress else {}), + } + return {"contradicted": True} + def terminal() -> dict[str, Any] | None: turns = list(session._control_items("thread/turns/list", { "sortDirection": "desc", "itemsView": "summary", }, deadline)) ids = [item.get("id") for item in turns] - if (not ids or any(not isinstance(value, str) or not value for value in ids) - or len(set(ids)) != len(ids) - or any(item.get("status") not in {"completed", "failed", "interrupted"} for item in turns)): + if not ids: return None + if any(not isinstance(value, str) or not value for value in ids) or len(set(ids)) != len(ids): + return contradicted() selected = next((item for item in turns if item["id"] == turn_id), None) + if any(item.get("status") not in {"completed", "failed", "interrupted"} for item in turns): + if selected and selected.get("status") == "completed": + return contradicted() + return contradicted( + in_progress=True, + target_in_progress=bool( + selected and selected.get("status") not in {"completed", "failed", "interrupted"} + ), + ) if not selected: return None if selected.get("status") == "interrupted" and selected.get("error") is None and ids[0] == turn_id: @@ -534,7 +560,7 @@ def terminal() -> dict[str, Any] | None: # only an exact user-only inventory below can prove no execution. return {"interrupted_before_execution": True} if selected.get("status") != "failed": - return None + return contradicted() if _routing_discovery_timeout(selected.get("error")): # Later work must not strand an untouched accepted request. Keep # the historical user-only rule; only the exact latest failure @@ -558,12 +584,12 @@ def terminal() -> dict[str, Any] | None: status is None or type(status) is int and 500 <= status <= 599 ): return {"codex_error_info": info} - return None + return contradicted() try: proof = terminal() - if proof is None: - return None + if proof is None or proof.get("contradicted") is True: + return proof items = list(session._control_items("thread/items/list", {"turnId": turn_id}, deadline)) routing = (proof.get("pre_execution_routing_timeout") is True or proof.get("loopback_relay_auth_failure") is True) @@ -576,14 +602,14 @@ def terminal() -> dict[str, Any] | None: if (entry.get("turnId") != turn_id or not isinstance(item, dict) or item.get("type") not in allowed or not item.get("id") or not isinstance(item["id"], str) or item["id"] in ids): - return None + return contradicted() ids.add(item["id"]) users += item["type"] == "userMessage" if users != 1 or (not routing and len(items) != 1) or terminal() != proof: - return None + return contradicted() if proof.get("interrupted_before_execution") and list( session._control_items("thread/items/list", {"turnId": turn_id}, deadline)) != items: - return None + return contradicted() proof.pop("latest_routing_failure", None) return {"thread_id": session.thread_id, "turn_id": turn_id, "only_original_user_item": len(items) == 1, @@ -875,7 +901,9 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path, work_dir: Path | None = None, model_auth_home: Path | None = None, runtime_root: Path | None = None, - consumer: str = "worker") -> dict[str, Any] | None: + consumer: str = "worker", + include_contradiction: bool = False, + contradiction_detail: bool = False) -> dict[str, Any] | None: """Read persisted native metadata/items; create no native conversation. The initialized stdio client is not a listener or a model runner. It @@ -906,10 +934,28 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path, if completed_result_review: proof = completed_result_review_readback(native, turn_id, known_turn_ids=known_turn_ids or set(), allow_untyped=allow_untyped_result_review) + elif read_only_review: + proof = interrupted_result_readback(native, turn_id) else: - reader = interrupted_result_readback if read_only_review else failed_request_readback - proof = reader(native, turn_id) + proof = failed_request_readback( + native, + turn_id, + include_contradiction=include_contradiction, + contradiction_detail=contradiction_detail, + ) if proof is not None: + if proof.get("contradicted") is True: + contradiction: dict[str, Any] = { + "contradicted": True, + "thread_id": thread_id, + "turn_id": turn_id, + } + if contradiction_detail: + if proof.get("in_progress") is True: + contradiction["in_progress"] = True + if proof.get("target_in_progress") is True: + contradiction["target_in_progress"] = True + return contradiction if proof.get("codex_error_info") == "usageLimitExceeded": # Reuse the optional provider's TS availability owner; # Absence fails closed. diff --git a/loopx/capabilities/native_chat/report_failure_proof.py b/loopx/capabilities/native_chat/report_failure_proof.py new file mode 100644 index 0000000000..3511097037 --- /dev/null +++ b/loopx/capabilities/native_chat/report_failure_proof.py @@ -0,0 +1,347 @@ +"""Scoped persistence and verification for report terminal-failure evidence.""" +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from contextlib import nullcontext +import hashlib +import json +import logging +from pathlib import Path +import subprocess +from typing import Any + +from ...file_lock import try_exclusive_file_lock +from ...registry import atomic_write_json +from ..periodic_report.private_generation import attempt_directory + +REPORT_FAILURE_PROOF = "native-terminal-failure.json" +_LOG = logging.getLogger(__name__) + + +def report_failure_binding(original: Mapping[str, Any], window: dict[str, Any]) -> dict[str, str] | None: + """Bind a terminal report host failure to one cadence window and native result.""" + request = original.get("loopx_request") or {} + actor = request.get("actor") or {} + result = (original.get("response") or {}).get("native_loopx") or {} + failure = result.get("host_failure") or {} + if (original.get("origin") != "native_loopx" or original.get("loopx_execution") is not True + or original.get("status") != "completed" or original.get("session_id") != actor.get("session_id") + or request.get("work_kind") != "report" or request.get("report_window_id") != window.get("window_id") + or request.get("report_subscription_revision") != window.get("subscription_revision") + or actor.get("goal_id") != window.get("goal_id") or actor.get("agent_id") != window.get("agent_id") + or result.get("status") != "failed" or result.get("result_kind") != "host_failure" + or failure.get("retryable") is not False): + return None + identity = { + "session_id": original.get("session_id"), + "chat_turn_id": original.get("turn_id"), + "provider_turn_id": original.get("upstream_turn_id"), + "turn_instance_id": request.get("turn_instance_id"), + "window_id": window.get("window_id"), + "subscription_revision": window.get("subscription_revision"), + } + if any(not isinstance(value, str) or not value for value in identity.values()): + return None + encoded = json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + return {**identity, "native_result_sha256": hashlib.sha256(encoded).hexdigest()} + + +def _read_report_failure_record(runtime_root: Path, window: dict[str, Any]) -> dict[str, Any] | None: + path = attempt_directory(runtime_root, window) / REPORT_FAILURE_PROOF + try: + if path.parent.is_symlink() or path.is_symlink() or not path.is_file(): + return None + proof = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return proof if isinstance(proof, dict) else None + + +def _is_valid_retained_proof(proof: dict[str, Any] | None, binding: dict[str, str]) -> bool: + return ( + isinstance(proof, dict) + and set(proof) == {"binding", "provider_thread_id", "only_original_user_item"} + and proof["binding"] == binding + and proof["only_original_user_item"] is True + and isinstance(proof["provider_thread_id"], str) + and bool(proof["provider_thread_id"]) + ) + + +def _is_contradicted_proof(proof: dict[str, Any] | None, binding: dict[str, str]) -> bool: + if not isinstance(proof, dict): + return False + thread_id = proof.get("provider_thread_id") + return ( + set(proof) == {"binding", "provider_thread_id", "only_original_user_item", "contradicted"} + and proof["binding"] == binding + and proof["only_original_user_item"] is False + and proof["contradicted"] is True + and (thread_id is None or isinstance(thread_id, str) and bool(thread_id)) + ) + + +def _failure_proof_lock(folder: Path, *, operation: str, generation_locked: bool): + if generation_locked: + return nullcontext(folder / "generation.lock") + return try_exclusive_file_lock( + folder / "generation.lock", + preserve_holder=True, + operation=operation, + ) + + +def read_retained_report_failure(runtime_root: Path, original: Mapping[str, Any], window: dict[str, Any]) -> bool: + """Scope retained provider evidence to the same failure and cadence window.""" + binding = report_failure_binding(original, window) + if binding is None: + return False + return _is_valid_retained_proof(_read_report_failure_record(runtime_root, window), binding) + + +def is_report_failure_contradicted(runtime_root: Path, original: Mapping[str, Any], window: dict[str, Any]) -> bool: + """Return True when live provider history has already contradicted this Turn's proof.""" + binding = report_failure_binding(original, window) + if binding is None: + return False + return _is_contradicted_proof(_read_report_failure_record(runtime_root, window), binding) + + +def retain_report_failure( + runtime_root: Path, store: Any, original: Mapping[str, Any], window: dict[str, Any], proof: Mapping[str, Any], + *, generation_locked: bool = False, +) -> None: + """Idempotently record a verified pre-execution failure under the generation lock.""" + binding = report_failure_binding(original, window) + thread_id = proof.get("thread_id") + if (binding is None or proof.get("turn_id") != binding["provider_turn_id"] + or proof.get("only_original_user_item") is not True + or not isinstance(thread_id, str) or not thread_id): + return + folder = attempt_directory(runtime_root, window) + if folder.is_symlink() or not folder.is_dir(): + return + path = folder / REPORT_FAILURE_PROOF + with _failure_proof_lock( + folder, operation="retain_report_failure_observation", generation_locked=generation_locked, + ) as ownership: + if ownership is None or path.exists() or path.is_symlink(): + return + # Read the original again after metadata I/O. Never overwrite a + # prior receipt, contradiction, or copy the provider's live quota observation. + current = store.load_turn(original["session_id"], original["turn_id"]) + if current is not None and report_failure_binding(current, window) == binding: + atomic_write_json(path, {"binding": binding, + "provider_thread_id": thread_id, "only_original_user_item": True}) + + +def contradict_report_failure( + runtime_root: Path, store: Any, original: Mapping[str, Any], window: dict[str, Any], + proof: Mapping[str, Any] | None = None, *, generation_locked: bool = False, +) -> None: + """Durably invalidate a report Turn's failure proof after live contradiction.""" + binding = report_failure_binding(original, window) + if binding is None: + return + if (isinstance(proof, Mapping) and proof.get("turn_id") is not None + and proof.get("turn_id") != binding["provider_turn_id"]): + return + folder = attempt_directory(runtime_root, window) + if folder.is_symlink() or not folder.is_dir(): + return + path = folder / REPORT_FAILURE_PROOF + with _failure_proof_lock( + folder, operation="contradict_report_failure_observation", generation_locked=generation_locked, + ) as ownership: + if ownership is None or path.is_symlink(): + return + existing = _read_report_failure_record(runtime_root, window) + thread_id = proof.get("thread_id") if isinstance(proof, Mapping) else None + if not isinstance(thread_id, str) or not thread_id: + existing_thread = existing.get("provider_thread_id") if existing else None + thread_id = existing_thread if isinstance(existing_thread, str) and existing_thread else None + if _is_contradicted_proof(existing, binding) and existing.get("provider_thread_id") == thread_id: + return + # Read the original again after metadata I/O before recording the + # contradiction for this exact Turn, window and native result digest. + current = store.load_turn(original["session_id"], original["turn_id"]) + if current is not None and report_failure_binding(current, window) == binding: + atomic_write_json(path, { + "binding": binding, + "provider_thread_id": thread_id, + "only_original_user_item": False, + "contradicted": True, + }) + + +def valid_report_failure_proof( + original: Mapping[str, Any], threads: Sequence[str], proof: Mapping[str, Any] | None, +) -> bool: + """Return True when a live provider observation proves a user-only host failure.""" + return (isinstance(proof, Mapping) and proof.get("contradicted") is not True + and proof.get("turn_id") == original["upstream_turn_id"] and proof.get("thread_id") in threads + and proof.get("only_original_user_item") is True and bool(proof.get("codex_error_info"))) + + +def proof_targets_original_turn( + original: Mapping[str, Any], threads: Sequence[str], proof: Mapping[str, Any] | None, +) -> bool: + """Ensure a contradicted observation refers to this exact original thread/turn.""" + if not isinstance(proof, Mapping): + return False + thread_id = proof.get("thread_id") + turn_id = proof.get("turn_id") + if proof.get("contradicted") is True: + return ((thread_id is None or thread_id in threads) + and (turn_id is None or turn_id == original.get("upstream_turn_id"))) + return thread_id in threads and turn_id == original.get("upstream_turn_id") + + +def probe_report_failure( + controller: Any, store: Any, original: Mapping[str, Any], *, + include_contradiction: bool = False, contradiction_detail: bool = False, + swallow_probe_errors: bool | None = None, +) -> tuple[list[str], dict[str, Any] | None] | None: + """Query the original Session's native threads without starting or resuming work.""" + result = (original.get("response") or {}).get("native_loopx") or {} + request = original.get("loopx_request") or {} + if (original.get("status") != "completed" or request.get("work_kind") != "report" + or result.get("result_kind") != "host_failure" or not original.get("upstream_turn_id")): + return None + from ...chat_agent import CodexChatAgentError + from .codex_adapter import CodexAppServerAdapter + from .codex_context import checked_session_home + + session = store.load_session(original["session_id"]) + if not session or not session.get("codex_home") or not session.get("upstream_thread_id"): + return None + history = session.get("upstream_context_history") or [] + if not isinstance(history, list) or any(not isinstance(item, dict) + or not isinstance(item.get("thread_id"), str) or not item["thread_id"] + for item in history): + return None + # Context replacement retains native provenance on this same Session. + # Read the original refusal there; never resume a historical thread. + threads = [session["upstream_thread_id"], *(item["thread_id"] for item in reversed(history))] + swallow = include_contradiction if swallow_probe_errors is None else swallow_probe_errors + try: + proof = CodexAppServerAdapter.failed_request_observation(codex_bin=controller.codex_bin, + codex_home=Path(session["codex_home"]), thread_ids=list(dict.fromkeys(threads)), + model_auth_home=checked_session_home(controller.codex_home, session), + turn_id=original["upstream_turn_id"], include_contradiction=include_contradiction, + contradiction_detail=contradiction_detail) + except (CodexChatAgentError, OSError, subprocess.SubprocessError): + if not swallow: + raise + proof = None + return threads, proof + + +def _try_retain_report_failure( + runtime_root: Path, store: Any, original: Mapping[str, Any], window: dict[str, Any], + proof: Mapping[str, Any], *, generation_locked: bool = False, +) -> None: + try: + retain_report_failure( + runtime_root, store, original, window, proof, generation_locked=generation_locked, + ) + except (OSError, ValueError, TypeError, KeyError): + _LOG.warning("An original report's terminal failure proof could not be retained") + + +def _try_contradict_report_failure( + runtime_root: Path, store: Any, original: Mapping[str, Any], window: dict[str, Any], + proof: Mapping[str, Any] | None = None, *, generation_locked: bool = False, +) -> None: + try: + contradict_report_failure( + runtime_root, store, original, window, proof, generation_locked=generation_locked, + ) + except (OSError, ValueError, TypeError, KeyError): + _LOG.warning("An original report's terminal failure contradiction could not be retained") + + +def handle_contradicted_report_observation( + runtime_root: Path, store: Any, original: Mapping[str, Any], window: dict[str, Any] | None, + threads: Sequence[str], proof: Mapping[str, Any], *, generation_locked: bool = False, +) -> None: + """Persist durable contradictions while holding transient in-progress threads safely.""" + if window is None or not proof_targets_original_turn(original, threads, proof): + return + if proof.get("in_progress") is True: + if proof.get("target_in_progress") is True and read_retained_report_failure(runtime_root, original, window): + _try_contradict_report_failure( + runtime_root, store, original, window, proof, generation_locked=generation_locked, + ) + return + _try_contradict_report_failure( + runtime_root, store, original, window, proof, generation_locked=generation_locked, + ) + + +def verify_report_window_failure( + controller: Any, store: Any, original: Mapping[str, Any], window: dict[str, Any], *, + probe_fn: Callable[..., tuple[list[str], dict[str, Any] | None] | None] | None = None, + generation_locked: bool = False, +) -> bool: + """Query live provider history first; fall back to un-contradicted proof only if unavailable.""" + runtime_root = controller.coordination_runtime_root + probed = ( + probe_fn(original, include_contradiction=True, contradiction_detail=True) + if probe_fn is not None + else probe_report_failure( + controller, store, original, include_contradiction=True, contradiction_detail=True, + ) + ) + if probed is None: + return False + threads, proof = probed + if proof is not None: + if proof.get("contradicted") is True or not valid_report_failure_proof(original, threads, proof): + handle_contradicted_report_observation( + runtime_root, store, original, window, threads, proof, generation_locked=generation_locked, + ) + return False + if is_report_failure_contradicted(runtime_root, original, window): + return False + _try_retain_report_failure( + runtime_root, store, original, window, proof, generation_locked=generation_locked, + ) + return True + return read_retained_report_failure(runtime_root, original, window) + + +def resolve_report_quota_observation( + controller: Any, store: Any, original: Mapping[str, Any], *, + report_window: dict[str, Any] | None = None, + probe_fn: Callable[..., tuple[list[str], dict[str, Any] | None] | None] | None = None, + fallback_observe_fn: Callable[[Mapping[str, Any]], dict[str, Any] | None] | None = None, +) -> dict[str, Any] | None: + """Inspect live quota recovery for the original Turn without rewriting prior contradictions.""" + runtime_root = controller.coordination_runtime_root + if report_window is not None and is_report_failure_contradicted(runtime_root, original, report_window): + return None + if fallback_observe_fn is not None: + proof = fallback_observe_fn(original) + else: + probed = ( + probe_fn( + original, include_contradiction=True, contradiction_detail=True, swallow_probe_errors=False, + ) + if probe_fn is not None + else probe_report_failure( + controller, store, original, + include_contradiction=True, contradiction_detail=True, swallow_probe_errors=False, + ) + ) + if probed is None: + return None + threads, proof = probed + if proof is None: + return None + if proof.get("contradicted") is True or not valid_report_failure_proof(original, threads, proof): + handle_contradicted_report_observation(runtime_root, store, original, report_window, threads, proof) + return None + if proof is not None and report_window is not None: + _try_retain_report_failure(runtime_root, store, original, report_window, proof) + return proof diff --git a/loopx/capabilities/native_chat/steward_managed_mode.py b/loopx/capabilities/native_chat/steward_managed_mode.py index 56f6af14e9..54892b23f9 100644 --- a/loopx/capabilities/native_chat/steward_managed_mode.py +++ b/loopx/capabilities/native_chat/steward_managed_mode.py @@ -451,7 +451,7 @@ def submit(self, session_id: str, *, agent_id: str, operation_id: str, retained = report_turn(self.store, request["actor"], report_window) if retained is not None and retained["turn_id"] != existing["turn_id"]: return retained, False - quota_request = host.report_quota_request(existing) + quota_request = host.report_quota_request(existing, report_window=report_window) if quota_request is not None: request = quota_request client_id += ".quota-recovery" diff --git a/loopx/capabilities/native_chat/steward_reports.py b/loopx/capabilities/native_chat/steward_reports.py index 44ee3e3d2d..7fc566447b 100644 --- a/loopx/capabilities/native_chat/steward_reports.py +++ b/loopx/capabilities/native_chat/steward_reports.py @@ -135,7 +135,9 @@ def _execution_failed(self, actor: dict[str, Any], window: dict[str, Any]) -> bo and (turn.get("status") in {"failed", "interrupted", "timed_out"} or native.get("status") == "failed")) - def _failed_without_generation(self, actor: dict[str, Any], window: dict[str, Any]) -> bool: + def _failed_without_generation( + self, actor: dict[str, Any], window: dict[str, Any], *, generation_locked: bool = False, + ) -> bool: """Read a terminal report failure under the original execution fence. The cadence owner holds the generation fence. This proves only that @@ -208,7 +210,9 @@ def _failed_without_generation(self, actor: dict[str, Any], window: dict[str, An inspection = interpret_turn_journal_projection(journal, goal_id=goal_id, agent_id=agent_id, turn_key=key) return (inspection.get("journal_consistent") is True and inspection.get("completed_phases") == [] - and StewardTurnHost(self.controller).report_failure_observation(turn) is not None) + and StewardTurnHost(self.controller).report_window_failure_verified( + turn, window, generation_locked=generation_locked, + )) def _cancel_failed_window(self, actor: dict[str, Any], window: dict[str, Any], failure: dict[str, str], *, cadence_locked: bool = False) -> None: @@ -407,7 +411,9 @@ def _frontier(self, owner_key: str, actor: dict[str, Any]) -> tuple[dict[str, An admitted = admit_cadence_window(runtime_root=self.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"], subscription=subscription, now=datetime.now(timezone.utc), subscription_resolver=lambda: self.subscription(owner_key, expected_actor=actor), - can_replace_failed_window=lambda window: self._failed_without_generation(actor, window)) + can_replace_failed_window=lambda window: self._failed_without_generation( + actor, window, generation_locked=True, + )) if admitted["status"] != "pending": return None, False window = admitted["window"] diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 900e993e7e..c2d2895040 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -22,6 +22,16 @@ from ...control_plane.goals.activation import goal_is_stopped from ...control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected from ...file_lock import exclusive_file_lock +from .report_failure_proof import ( + is_report_failure_contradicted, + probe_report_failure, + read_retained_report_failure, + report_failure_binding, + resolve_report_quota_observation, + retain_report_failure, + valid_report_failure_proof, + verify_report_window_failure, +) from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS = 300 @@ -434,42 +444,71 @@ def report_quota_basis(self, request: Mapping[str, Any]) -> dict[str, Any]: ("editorial-response.json", "private-generation.json", "private-delivery.json")), "observed_at": time.time()}} + def _report_failure_probe( + self, original: Mapping[str, Any], *, include_contradiction: bool = False, + contradiction_detail: bool = False, swallow_probe_errors: bool | None = None, + ) -> tuple[list[str], dict[str, Any] | None] | None: + return probe_report_failure( + self.controller, self.store, original, + include_contradiction=include_contradiction, + contradiction_detail=contradiction_detail, + swallow_probe_errors=swallow_probe_errors, + ) + + @staticmethod + def _valid_report_failure_proof( + original: Mapping[str, Any], threads: Sequence[str], proof: Mapping[str, Any] | None, + ) -> bool: + return valid_report_failure_proof(original, threads, proof) + def report_failure_observation(self, original: Mapping[str, Any]) -> dict[str, Any] | None: """Read the original native failure without starting or resuming work.""" - result = (original.get("response") or {}).get("native_loopx") or {} - request = original.get("loopx_request") or {} - if (original.get("status") != "completed" or request.get("work_kind") != "report" - or result.get("result_kind") != "host_failure" or not original.get("upstream_turn_id")): - return None - from .codex_adapter import CodexAppServerAdapter - from .codex_context import checked_session_home - session = self.store.load_session(original["session_id"]) - if not session or not session.get("codex_home") or not session.get("upstream_thread_id"): - return None - history = session.get("upstream_context_history") or [] - if not isinstance(history, list) or any(not isinstance(item, dict) - or not isinstance(item.get("thread_id"), str) or not item["thread_id"] - for item in history): + probed = self._report_failure_probe(original) + if probed is None: return None - # Context replacement retains native provenance on this same Session. - # Read the original refusal there; never resume a historical thread. - threads = [session["upstream_thread_id"], *(item["thread_id"] for item in reversed(history))] - proof = CodexAppServerAdapter.failed_request_observation(codex_bin=self.controller.codex_bin, - codex_home=Path(session["codex_home"]), - model_auth_home=checked_session_home(self.controller.codex_home, session), - thread_ids=list(dict.fromkeys(threads)), turn_id=original["upstream_turn_id"]) - if (not proof or proof.get("turn_id") != original["upstream_turn_id"] - or proof.get("thread_id") not in threads or proof.get("only_original_user_item") is not True - or not proof.get("codex_error_info")): - return None - return proof + threads, proof = probed + return proof if self._valid_report_failure_proof(original, threads, proof) else None + + def report_window_failure_verified( + self, original: Mapping[str, Any], window: dict[str, Any], *, generation_locked: bool = False, + ) -> bool: + """Query live provider history first; fall back to retained proof only if unavailable.""" + return verify_report_window_failure( + self.controller, self.store, original, window, + probe_fn=self._report_failure_probe, generation_locked=generation_locked, + ) - def report_quota_request(self, original: Mapping[str, Any]) -> dict[str, Any] | None: + @staticmethod + def _report_failure_binding(original: Mapping[str, Any], window: dict[str, Any]) -> dict[str, str] | None: + return report_failure_binding(original, window) + + def retained_report_failure_observation(self, original: Mapping[str, Any], window: dict[str, Any]) -> bool: + """Scope retained provider evidence to the same failure and cadence window.""" + return read_retained_report_failure(self.controller.coordination_runtime_root, original, window) + + def report_failure_contradicted(self, original: Mapping[str, Any], window: dict[str, Any]) -> bool: + """Return True when a prior live observation contradicted this Turn's failure proof.""" + return is_report_failure_contradicted(self.controller.coordination_runtime_root, original, window) + + def _retain_report_failure_observation(self, original: Mapping[str, Any], window: dict[str, Any], + proof: Mapping[str, Any], *, generation_locked: bool = False) -> None: + retain_report_failure( + self.controller.coordination_runtime_root, self.store, original, window, proof, + generation_locked=generation_locked, + ) + + def report_quota_request(self, original: Mapping[str, Any], *, + report_window: dict[str, Any] | None = None) -> dict[str, Any] | None: """Observe quota recovery, never resume or rewrite the failed native Turn.""" request = original.get("loopx_request") or {} if request.get("report_quota_recovery") is not None: return None - proof = self.report_failure_observation(original) + direct_probe = getattr(self.report_failure_observation, "__func__", None) is _DEFAULT_REPORT_FAILURE_OBSERVATION + proof = resolve_report_quota_observation( + self.controller, self.store, original, report_window=report_window, + probe_fn=self._report_failure_probe if direct_probe else None, + fallback_observe_fn=None if direct_probe else self.report_failure_observation, + ) if (not proof or proof.get("only_original_user_item") is not True or proof.get("codex_error_info") != "usageLimitExceeded" or proof.get("quota_available") is not True): return None @@ -1148,7 +1187,8 @@ def inspect_submission(quota_observation: dict[str, Any] | None = None) -> dict[ quota_observation = None if "quota_probe" in prepared: quota_observation = prepared["quota_probe"] - quota_observation["request"] = self.report_quota_request(quota_observation["original"]) + quota_observation["request"] = self.report_quota_request( + quota_observation["original"], report_window=report_window) prepared = inspect_submission(quota_observation) if "turn" in prepared: return prepared["turn"], False @@ -1895,3 +1935,6 @@ def run_primary_turn() -> None: **({"context_handoff": committed_handoff} if committed_handoff else {}), **({"steward_owner_report_queued": True} if owner_report else {}), **({"steward_user_action_receipts": user_action_receipts} if user_action_receipts else {})} + + +_DEFAULT_REPORT_FAILURE_OBSERVATION = StewardTurnHost.report_failure_observation diff --git a/tests/test_chat_agent.py b/tests/test_chat_agent.py index 7b58db3c4a..9f50854051 100644 --- a/tests/test_chat_agent.py +++ b/tests/test_chat_agent.py @@ -1049,6 +1049,109 @@ def write(packet): assert set(calls) <= {"thread/turns/list", "thread/items/list"} +@pytest.mark.parametrize("case", ["valid", "answer", "completed", "active", "policy", "empty", "pruned", "rpc-error"]) +def test_failed_request_readback_distinguishes_contradiction_from_unavailable_history(tmp_path, monkeypatch, case): + session = chat_agent.CodexChatAgentSession(process=SimpleNamespace(poll=lambda: None), + messages=queue.Queue(), thread_id="original-thread", work_dir=tmp_path, response_timeout_sec=.2) + + def write(packet): + method = packet["method"] + if case == "rpc-error": + session.messages.put({"id": packet["id"], "error": {"code": -32000, "message": "history unavailable"}}) + return + if method == "thread/turns/list": + if case == "empty": + rows = [] + elif case == "pruned": + rows = [{"id": "other-turn", "status": "completed"}] + else: + status = "completed" if case == "completed" else "failed" + info = "cyberPolicy" if case == "policy" else "other" + rows = [{"id": "failed", "status": status, "error": {"codexErrorInfo": info}}] + if case == "active": + rows.insert(0, {"id": "later", "status": "inProgress"}) + result = {"data": rows} + else: + rows = [{"turnId": "failed", "item": {"id": "input", "type": "userMessage"}}] + if case == "answer": + rows.append({"turnId": "failed", "item": {"id": "reply", "type": "agentMessage"}}) + result = {"data": rows} + session.messages.put({"id": packet["id"], "result": result}) + + monkeypatch.setattr(session, "_write", write) + default_proof = failed_request_readback(session, "failed") + marked_proof = failed_request_readback(session, "failed", include_contradiction=True) + detailed_proof = failed_request_readback( + session, "failed", include_contradiction=True, contradiction_detail=True, + ) + if case == "valid": + expected = {"thread_id": "original-thread", "turn_id": "failed", + "only_original_user_item": True, "codex_error_info": "other"} + assert default_proof == expected and marked_proof == expected and detailed_proof == expected + elif case in {"answer", "completed", "active", "policy"}: + assert default_proof is None + assert marked_proof == {"contradicted": True} + assert detailed_proof == ( + {"contradicted": True, "in_progress": True} if case == "active" else {"contradicted": True} + ) + else: + assert default_proof is None + assert marked_proof is None + assert detailed_proof is None + + +@pytest.mark.parametrize("newer_state", ["contradicted", "empty"]) +def test_failed_request_observation_stops_immediately_on_contradiction(tmp_path, monkeypatch, newer_state): + import loopx.capabilities.native_chat.codex_adapter as codex_adapter + + script = tmp_path / "fake_codex.py" + queried_threads = tmp_path / "queried.jsonl" + script.write_text(f"""import json, sys +from pathlib import Path +log = Path({str(queried_threads)!r}) +for raw in sys.stdin: + req = json.loads(raw) + method = req.get("method") + if method == "initialize": + print(json.dumps({{"id": req["id"], "result": {{}}}}), flush=True) + elif method == "thread/turns/list": + tid = req["params"]["threadId"] + with log.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({{"method": method, "threadId": tid}}) + "\\n") + if tid == "newer-thread": + rows = [] if {newer_state!r} == "empty" else [{{"id": "target-turn", "status": "failed", "error": {{"codexErrorInfo": "other"}}}}] + else: + rows = [{{"id": "target-turn", "status": "failed", "error": {{"codexErrorInfo": "other"}}}}] + print(json.dumps({{"id": req["id"], "result": {{"data": rows}}}}), flush=True) + elif method == "thread/items/list": + tid = req["params"]["threadId"] + items = [{{"turnId": "target-turn", "item": {{"id": "u1", "type": "userMessage"}}}}] + if tid == "newer-thread": + items.append({{"turnId": "target-turn", "item": {{"id": "a1", "type": "agentMessage"}}}}) + print(json.dumps({{"id": req["id"], "result": {{"data": items}}}}), flush=True) +""", encoding="utf-8") + real_popen = codex_adapter.subprocess.Popen + + def launch(command, *args, **kwargs): + if command and command[0] == "fake-codex": + command = [sys.executable, str(script)] + return real_popen(command, *args, **kwargs) + + monkeypatch.setattr(codex_adapter.subprocess, "Popen", launch) + result = codex_adapter.CodexAppServerAdapter.failed_request_observation( + codex_bin="fake-codex", codex_home=tmp_path, + thread_ids=["newer-thread", "older-thread"], turn_id="target-turn", + include_contradiction=True) + observed = [json.loads(line)["threadId"] for line in queried_threads.read_text(encoding="utf-8").splitlines()] + if newer_state == "contradicted": + assert result == {"contradicted": True, "thread_id": "newer-thread", "turn_id": "target-turn"} + assert observed == ["newer-thread"] + else: + assert result == {"thread_id": "older-thread", "turn_id": "target-turn", + "only_original_user_item": True, "codex_error_info": "other"} + assert observed == ["newer-thread", "older-thread", "older-thread"] + + @pytest.mark.parametrize("fault", [None, "older", "active", "tool", "answer", "reasoning", "compaction", "unknown", "correction", "foreign", "duplicate", "changed-items", "changed-turn", "native-error"]) def test_disconnected_latest_interruption_requires_stable_user_only_inventory(tmp_path, monkeypatch, fault): diff --git a/tests/test_native_steward_reports.py b/tests/test_native_steward_reports.py index 70af53558a..be4cb908c0 100644 --- a/tests/test_native_steward_reports.py +++ b/tests/test_native_steward_reports.py @@ -1707,23 +1707,18 @@ def post(body, origin=None): thread.join(5) -@pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("local_session", [False, True]) -@pytest.mark.parametrize("native_status", ["failed", "inProgress"]) -def test_missed_report_keeps_failed_turn_and_returns_only_the_latest_window( - steward, tmp_path, monkeypatch, provider, local_session, native_status): +def _setup_missed_report_failure(steward, tmp_path, monkeypatch, provider, local_session, native_status="failed"): from datetime import timezone from loopx.capabilities.native_chat import steward_reports as report_host from loopx.capabilities.periodic_report.cadence_journal import cadence_journal_path - from loopx.capabilities.periodic_report.private_generation import freeze_private_report from loopx.control_plane.turn_driver.journal_store import turn_journal_path - from loopx.file_lock import exclusive_file_lock - point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=5) + clock = {"now": datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=5)} class CalendarClock(datetime): @classmethod def now(cls, tz=None): + point = clock["now"] return point.astimezone(tz) if tz else point.replace(tzinfo=None) monkeypatch.setattr(report_host, "datetime", CalendarClock) @@ -1742,6 +1737,10 @@ def now(cls, tz=None): fake = Path(runtime.codex_bin) source = fake.read_text() fail_once = tmp_path / "fail-next-report" + history_unavailable = tmp_path / "report-history-unavailable" + history_rpc_error = tmp_path / "report-history-rpc-error" + contradicted_items = tmp_path / "report-contradicted-items" + other_turn_active = tmp_path / "report-other-turn-active" fail_once.touch() probe = f""" if method == "turn/start" and __import__("pathlib").Path({str(fail_once)!r}).exists(): @@ -1750,11 +1749,23 @@ def now(cls, tz=None): print(json.dumps({{"method": "turn/completed", "params": {{"threadId": "durable-thread", "turn": {{"id": "report-failed-after-launch", "status": "failed", "error": {{"codexErrorInfo": "other", "message": "Synthetic terminal failure"}}}}}}}}), flush=True) continue sys.exit(0) + if method in {{"thread/turns/list", "thread/items/list"}} and __import__("pathlib").Path({str(history_rpc_error)!r}).exists(): + print(json.dumps({{"id": request["id"], "error": {{"code": -32000, "message": "Synthetic provider history unavailable"}}}}), flush=True) + continue + if method == "thread/turns/list" and __import__("pathlib").Path({str(history_unavailable)!r}).exists(): + print(json.dumps({{"id": request["id"], "result": {{"data": []}}}}), flush=True) + continue if method == "thread/turns/list": - print(json.dumps({{"id": request["id"], "result": {{"data": [{{"id": "report-failed-after-launch", "status": {native_status!r}, "error": {{"codexErrorInfo": "other"}}}}]}}}}), flush=True) + rows = [{{"id": "report-failed-after-launch", "status": {native_status!r}, "error": {{"codexErrorInfo": "other"}}}}] + if __import__("pathlib").Path({str(other_turn_active)!r}).exists(): + rows.insert(0, {{"id": "unrelated-active-turn", "status": "inProgress"}}) + print(json.dumps({{"id": request["id"], "result": {{"data": rows}}}}), flush=True) continue if method == "thread/items/list": - print(json.dumps({{"id": request["id"], "result": {{"data": [{{"turnId": "report-failed-after-launch", "item": {{"id": "original-user", "type": "userMessage"}}}}]}}}}), flush=True) + items = [{{"turnId": "report-failed-after-launch", "item": {{"id": "original-user", "type": "userMessage"}}}}] + if __import__("pathlib").Path({str(contradicted_items)!r}).exists(): + items.append({{"turnId": "report-failed-after-launch", "item": {{"id": "extra-agent", "type": "agentMessage"}}}}) + print(json.dumps({{"id": request["id"], "result": {{"data": items}}}}), flush=True) continue """ fake.write_text(source.replace(' method = request.get("method")', ' method = request.get("method")' + probe, 1)) @@ -1767,7 +1778,8 @@ def now(cls, tz=None): cadence = cadence_journal_path(reports.root, actor["goal_id"]) previous = json.loads(cadence.read_text()) native_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"]) - old_artifacts = (store._turn_path(sid, original["turn_id"]), native_path, + old_artifacts = (store._turn_path(sid, original["turn_id"]), + store._event_path(sid, original["turn_id"]), native_path, old_folder / "source.json", old_folder / "editorial-request.json") original_bytes = [path.read_bytes() for path in old_artifacts] sent = list(bot.writes) @@ -1775,15 +1787,62 @@ def now(cls, tz=None): fail_once.unlink() Path(store.load_session(sid)["codex_home"]).mkdir(parents=True, exist_ok=True) runtime.adapters.pop(sid).close_session() + return { + "clock": clock, "sid": sid, "owner": owner, "actor": actor, "reports": reports, + "original": original, "old": old, "old_folder": old_folder, + "cadence": cadence, "previous": previous, "native_path": native_path, + "old_artifacts": old_artifacts, "original_bytes": original_bytes, + "sent": sent, "calls_before": calls_before, + "history_unavailable": history_unavailable, "history_rpc_error": history_rpc_error, + "contradicted_items": contradicted_items, "other_turn_active": other_turn_active, + } + + +def _restart_steward_runtime(store, runtime): + from loopx.chat_runtime import ChatRuntimeController + from loopx.chat_store import ChatSessionStore + + runtime.close() + restarted = ChatRuntimeController( + store=ChatSessionStore(store.root.parent), + codex_bin=runtime.codex_bin, + registry_path=runtime.registry_path, + project_contexts=runtime.project_contexts, + ) + restarted.manager_scope_resolver = runtime.manager_scope_resolver + if hasattr(runtime, "steward_report_provider"): + restarted.steward_report_provider = runtime.steward_report_provider + return restarted + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("local_session", [False, True]) +@pytest.mark.parametrize("native_status", ["failed", "inProgress"]) +def test_missed_report_keeps_failed_turn_and_returns_only_the_latest_window( + steward, tmp_path, monkeypatch, provider, local_session, native_status): + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + from loopx.capabilities.periodic_report.private_generation import freeze_private_report + from loopx.file_lock import exclusive_file_lock + + store, runtime, bot, transport, _, capture, _ = steward + ctx = _setup_missed_report_failure(steward, tmp_path, monkeypatch, provider, local_session, native_status) + clock, sid, owner, actor, reports = ctx["clock"], ctx["sid"], ctx["owner"], ctx["actor"], ctx["reports"] + original, old, old_folder = ctx["original"], ctx["old"], ctx["old_folder"] + cadence, previous, native_path = ctx["cadence"], ctx["previous"], ctx["native_path"] + old_artifacts, original_bytes = ctx["old_artifacts"], ctx["original_bytes"] + sent, calls_before = ctx["sent"], ctx["calls_before"] + history_unavailable = ctx["history_unavailable"] def wake(): runtime._steward_monitor_next_check.clear() return pump_steward_monitors(runtime) assert wake() == [] - point += timedelta(days=1) + failure_proof = old_folder / "native-terminal-failure.json" + assert failure_proof.is_file() is (native_status == "failed") + clock["now"] += timedelta(days=1) assert wake() == [] # Keep the normal recovery opportunity at the next boundary. - point += timedelta(days=2) + clock["now"] += timedelta(days=2) uncertain = old_folder / "private-delivery.json" try: uncertain.symlink_to("unreadable-delivery-target") @@ -1805,10 +1864,26 @@ def starts(): original_starts = [row for row in map(json.loads, calls_before.splitlines()) if row.get("method") == "turn/start"] assert starts() == original_starts - from loopx.capabilities.native_chat.steward_turn import StewardTurnHost - native_proof = StewardTurnHost(runtime).report_failure_observation(original) + host = StewardTurnHost(runtime) + native_proof = host.report_failure_observation(original) assert bool(native_proof) is (native_status == "failed"), native_proof assert reports._failed_without_generation(actor, old) is (native_status == "failed") + if native_status == "failed": + retained_bytes = failure_proof.read_bytes() + retained = json.loads(retained_bytes) + assert set(retained) == {"binding", "provider_thread_id", "only_original_user_item"} + assert retained["provider_thread_id"] == native_proof["thread_id"] + assert retained["binding"]["provider_turn_id"] == native_proof["turn_id"] + history_unavailable.touch() + assert host.report_failure_observation(original) is None + assert host.report_quota_request(original, report_window=old) is None + retained["binding"]["native_result_sha256"] = "0" * 64 + failure_proof.write_text(json.dumps(retained), encoding="utf-8") + try: + assert wake() == [] and starts() == original_starts + assert json.loads(cadence.read_text()) == previous + finally: + failure_proof.write_bytes(retained_bytes) pending = wake() if native_status != "failed": assert pending == [] and starts() == original_starts @@ -1821,7 +1896,7 @@ def starts(): done = finish(runtime, pending[0]) assert done["response"]["native_loopx"]["status"] == "committed", done _, latest = reports.current_window(owner) - assert latest["due_at"] == point.replace(minute=0).isoformat().replace("+00:00", "Z") + assert latest["due_at"] == clock["now"].replace(minute=0).isoformat().replace("+00:00", "Z") assert latest["subscription_revision"] == old["subscription_revision"] assert done["loopx_request"]["actor"] == original["loopx_request"]["actor"] assert done["loopx_request"]["todo_id"] != original["loopx_request"]["todo_id"] @@ -1829,6 +1904,7 @@ def starts(): assert "report_quota_recovery" not in done["loopx_request"] assert json.loads((cadence.parent / "superseded-cadence" / (old["window_id"] + ".json")).read_text()) == previous assert [path.read_bytes() for path in old_artifacts] == original_bytes + assert failure_proof.read_bytes() == retained_bytes assert reports.delivery_current(owner, old) is False with pytest.raises(ValueError, match="no longer the active window"): freeze_private_report(root=reports.root, window=old, response={}, require_current_window=True) @@ -1855,6 +1931,145 @@ def starts(): assert transport.steward_reports.reconcile() == 0 +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("local_session", [False, True]) +def test_contradicted_report_failure_proof_stays_held_after_restart_and_history_outage( + steward, tmp_path, monkeypatch, provider, local_session): + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + from loopx.capabilities.native_chat.steward_reports import ManagedSessionReports + from loopx.capabilities.native_chat.steward_turn import StewardTurnHost + + store, runtime, bot, _, _, capture, _ = steward + ctx = _setup_missed_report_failure(steward, tmp_path, monkeypatch, provider, local_session, "failed") + clock, actor, reports = ctx["clock"], ctx["actor"], ctx["reports"] + original, old, old_folder = ctx["original"], ctx["old"], ctx["old_folder"] + cadence, previous = ctx["cadence"], ctx["previous"] + old_artifacts, original_bytes = ctx["old_artifacts"], ctx["original_bytes"] + sent, calls_before = ctx["sent"], ctx["calls_before"] + history_unavailable, history_rpc_error = ctx["history_unavailable"], ctx["history_rpc_error"] + contradicted_items = ctx["contradicted_items"] + + def wake(target_runtime=runtime): + cache = getattr(target_runtime, "_steward_monitor_next_check", None) + if cache is not None: + cache.clear() + return pump_steward_monitors(target_runtime) + + def starts(): + return [row for row in map(json.loads, capture.read_text().splitlines()) if row.get("method") == "turn/start"] + + original_starts = [row for row in map(json.loads, calls_before.splitlines()) if row.get("method") == "turn/start"] + original_todos = StewardMonitor(runtime).todos(actor) + + # Phase 1: user-only failure proof is retained on the original window. + assert wake() == [] + failure_proof = old_folder / "native-terminal-failure.json" + retained = json.loads(failure_proof.read_text(encoding="utf-8")) + assert set(retained) == {"binding", "provider_thread_id", "only_original_user_item"} + assert retained["only_original_user_item"] is True + + # Phase 2: after >1 boundary, live Codex history for the same Turn reveals + # an agentMessage; the window holds and persists the contradiction under generation.lock. + clock["now"] += timedelta(days=3) + contradicted_items.touch() + assert wake() == [] + assert starts() == original_starts + assert json.loads(cadence.read_text()) == previous + assert reports.current_window(ctx["owner"])[1]["window_id"] == old["window_id"] + contradicted = json.loads(failure_proof.read_text(encoding="utf-8")) + assert contradicted["binding"] == retained["binding"] + assert contradicted["only_original_user_item"] is False + assert contradicted["contradicted"] is True + + # Phase 3: after process restart and subsequent Codex history outage (empty + # list and RPC error), the invalidated record still rejects advancing. + contradicted_items.unlink() + history_unavailable.touch() + restarted = _restart_steward_runtime(store, runtime) + try: + restarted_reports = ManagedSessionReports(restarted) if local_session else StewardReports(restarted) + restarted_host = StewardTurnHost(restarted) + assert wake(restarted) == [] + history_unavailable.unlink() + history_rpc_error.touch() + assert wake(restarted) == [] + # Even if live history later stops returning the agentMessage without + # explaining the prior contradiction, the same Turn stays held. + history_rpc_error.unlink() + assert wake(restarted) == [] + assert restarted_host.report_quota_request(original, report_window=old) is None + assert restarted_host.report_window_failure_verified(original, old) is False + assert restarted_reports.current_window(ctx["owner"])[1]["window_id"] == old["window_id"] + assert json.loads(cadence.read_text()) == previous + assert starts() == original_starts + assert StewardMonitor(restarted).todos(actor) == original_todos + assert [path.read_bytes() for path in old_artifacts] == original_bytes + assert bot.writes == sent + finally: + restarted.close() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("local_session", [False, True]) +def test_contradicted_quota_observation_and_active_thread_hold_safely( + steward, tmp_path, monkeypatch, provider, local_session): + from loopx.capabilities.native_chat.steward_reports import ManagedSessionReports + + store, runtime, _, _, _, capture, _ = steward + ctx = _setup_missed_report_failure(steward, tmp_path, monkeypatch, provider, local_session, "failed") + clock = ctx["clock"] + old, old_folder = ctx["old"], ctx["old_folder"] + cadence, previous = ctx["cadence"], ctx["previous"] + calls_before = ctx["calls_before"] + history_unavailable = ctx["history_unavailable"] + contradicted_items, other_turn_active = ctx["contradicted_items"], ctx["other_turn_active"] + + def wake(target_runtime=runtime): + cache = getattr(target_runtime, "_steward_monitor_next_check", None) + if cache is not None: + cache.clear() + return pump_steward_monitors(target_runtime) + + def starts(): + return [row for row in map(json.loads, capture.read_text().splitlines()) if row.get("method") == "turn/start"] + + original_starts = [row for row in map(json.loads, calls_before.splitlines()) if row.get("method") == "turn/start"] + + # Save user-only failure proof on initial poll. + assert wake() == [] + failure_proof = old_folder / "native-terminal-failure.json" + retained_bytes = failure_proof.read_bytes() + + # Transient unrelated inProgress turn holds immediately without invalidating the retained proof. + clock["now"] += timedelta(days=3) + other_turn_active.touch() + assert wake() == [] + assert failure_proof.read_bytes() == retained_bytes + assert json.loads(cadence.read_text()) == previous + other_turn_active.unlink() + + # Contradiction observed during quota probe (same window, before crossing >1 boundary) + # persists and survives restart + history outage. + clock["now"] -= timedelta(days=3) + contradicted_items.touch() + assert wake() == [] + contradicted = json.loads(failure_proof.read_text(encoding="utf-8")) + assert contradicted["contradicted"] is True and contradicted["only_original_user_item"] is False + + contradicted_items.unlink() + history_unavailable.touch() + clock["now"] += timedelta(days=3) + restarted = _restart_steward_runtime(store, runtime) + try: + restarted_reports = ManagedSessionReports(restarted) if local_session else StewardReports(restarted) + assert wake(restarted) == [] + assert starts() == original_starts + assert json.loads(cadence.read_text()) == previous + assert restarted_reports.current_window(ctx["owner"])[1]["window_id"] == old["window_id"] + finally: + restarted.close() + + def test_consecutive_missed_windows_keep_the_first_unreported_deadline(tmp_path): from datetime import timezone from loopx.capabilities.periodic_report.cadence_journal import admit_cadence_window, missed_cadence_bounds diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 5e8e6ca99f..b3706fbbbe 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -832,8 +832,8 @@ def test_native_submission_history_read_allows_input_and_rejects_changed_binding store.update_session(sid, status="ready", active_turn_id=None) observe_quota = host.report_quota_request - def checked_quota(turn): - result = observe_quota(turn) + def checked_quota(turn, *, report_window=None): + result = observe_quota(turn, report_window=report_window) quota_checked.set() return result @@ -1271,3 +1271,96 @@ def fake_turn(self, message, event_sink=None, **_kwargs): assert adv_done["response"].get("steward_user_action_receipts") assert guards == [("AntigravityAdapter", True), ("AntigravityAdapter", False)] assert capture.read_bytes() == before_codex + + +def test_report_failure_proof_lock_and_contradiction_lifecycle(tmp_path): + from datetime import datetime, timezone + from types import SimpleNamespace + from loopx.capabilities.native_chat.report_failure_proof import ( + REPORT_FAILURE_PROOF, + contradict_report_failure, + is_report_failure_contradicted, + read_retained_report_failure, + retain_report_failure, + ) + from loopx.capabilities.periodic_report.cadence_journal import admit_cadence_window + from loopx.capabilities.periodic_report.private_generation import attempt_directory + from loopx.file_lock import exclusive_file_lock + + subscription = { + "enabled": True, + "profile_preset": "portfolio-progress", + "effective_revision": "sha256:" + "1" * 64, + "schedule": { + "schema_version": "periodic_report_schedule_v0", + "schedule_id": "daily-report", + "timezone": "UTC", + "rrule": "FREQ=DAILY;BYHOUR=9;BYMINUTE=0", + "start_date": "2026-01-01", + }, + } + window = admit_cadence_window( + runtime_root=tmp_path, + goal_id="goal-1", + agent_id="agent-1", + subscription=subscription, + now=datetime(2026, 1, 2, 10, tzinfo=timezone.utc), + )["window"] + original = { + "origin": "native_loopx", + "loopx_execution": True, + "status": "completed", + "session_id": "sess-1", + "turn_id": "chat-turn-1", + "upstream_turn_id": "codex-turn-1", + "loopx_request": { + "actor": {"session_id": "sess-1", "goal_id": "goal-1", "agent_id": "agent-1"}, + "work_kind": "report", + "report_window_id": window["window_id"], + "report_subscription_revision": window["subscription_revision"], + "turn_instance_id": "report." + window["window_id"], + }, + "response": { + "native_loopx": { + "status": "failed", + "result_kind": "host_failure", + "host_failure": {"retryable": False, "kind": "unknown"}, + } + }, + } + store = SimpleNamespace(load_turn=lambda sid, tid: original if (sid, tid) == ("sess-1", "chat-turn-1") else None) + folder = attempt_directory(tmp_path, window) + folder.mkdir(parents=True) + proof_path = folder / REPORT_FAILURE_PROOF + valid_proof = {"thread_id": "thread-1", "turn_id": "codex-turn-1", "only_original_user_item": True} + + # Held generation.lock blocks un-locked writes; generation_locked=True writes under caller's held lock. + with exclusive_file_lock(folder / "generation.lock", operation="busy"): + retain_report_failure(tmp_path, store, original, window, valid_proof) + assert not proof_path.exists() + retain_report_failure(tmp_path, store, original, window, valid_proof, generation_locked=True) + assert read_retained_report_failure(tmp_path, original, window) is True + assert is_report_failure_contradicted(tmp_path, original, window) is False + + # Contradiction invalidates the retained proof and cannot be overwritten back by retain_report_failure. + with exclusive_file_lock(folder / "generation.lock", operation="busy"): + contradict_report_failure(tmp_path, store, original, window, {"contradicted": True, "turn_id": "codex-turn-1"}) + assert read_retained_report_failure(tmp_path, original, window) is True + contradict_report_failure( + tmp_path, store, original, window, {"contradicted": True, "turn_id": "codex-turn-1"}, + generation_locked=True, + ) + assert read_retained_report_failure(tmp_path, original, window) is False + assert is_report_failure_contradicted(tmp_path, original, window) is True + assert json.loads(proof_path.read_text(encoding="utf-8"))["provider_thread_id"] == "thread-1" + retain_report_failure(tmp_path, store, original, window, valid_proof) + assert read_retained_report_failure(tmp_path, original, window) is False + assert is_report_failure_contradicted(tmp_path, original, window) is True + + # Symlink proof paths are never read or overwritten. + proof_path.unlink() + proof_path.symlink_to("foreign-target") + contradict_report_failure(tmp_path, store, original, window, {"contradicted": True, "turn_id": "codex-turn-1"}) + assert proof_path.is_symlink() + assert read_retained_report_failure(tmp_path, original, window) is False + assert is_report_failure_contradicted(tmp_path, original, window) is False