Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/product/periodic-reports.md
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,19 @@ deadline is not a promise that the held window can run. This observation grants
no cancellation, retirement or retry. A correlated quota continuation that
succeeds clears the current failure projection while retaining its failed predecessor.

When a terminal Codex failure with only the original user input is verified, the
report attempt records that observation under its generation lock. The record
binds the original Chat and Codex Turns, the report window and subscription
revision, and the original native result digest. A later independent period
queries live Codex history first after the existing journal, generation and
delivery checks pass: readable counter-evidence durably invalidates that Turn's
retained proof under the generation lock and holds the original window across
subsequent history outages unless fresh same-Turn evidence explains the
contradiction, and an un-contradicted retained proof is used only when Codex
history is unavailable. Quota continuation for the original period still
requires a live observation. Missing or mismatched evidence still leaves a
window held when the original failure cannot be verified.

| Cancelled failure on the original settings page | Narrow retained-failure state |
| --- | --- |
| ![Cancelled generation is not completed or delivered](../assets/personal-workspace/steward-reports/failed-window-cancelled.png) | ![Retained cancelled window in a narrow viewport](../assets/personal-workspace/steward-reports/cancelled-window-narrow.png) |
Expand Down
74 changes: 60 additions & 14 deletions loopx/capabilities/native_chat/codex_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,13 @@ def _loopback_relay_transport_ready(runtime_root: Path, codex_home: Path, consum
return False, now


def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dict[str, Any] | None:
def failed_request_readback(
session: CodexChatAgentSession,
turn_id: str,
*,
include_contradiction: bool = False,
contradiction_detail: bool = False,
) -> dict[str, Any] | None:
"""Observe a historical pre-execution failure without resuming it.

Unlike a same-worker retry, a migration may inspect an older failure.
Expand All @@ -516,16 +522,36 @@ def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dic
"""
deadline = time.monotonic() + session.response_timeout_sec

def contradicted(*, in_progress: bool = False, target_in_progress: bool = False) -> dict[str, Any] | None:
if not include_contradiction:
return None
if contradiction_detail and in_progress:
return {
"contradicted": True,
"in_progress": True,
**({"target_in_progress": True} if target_in_progress else {}),
}
return {"contradicted": True}

def terminal() -> dict[str, Any] | None:
turns = list(session._control_items("thread/turns/list", {
"sortDirection": "desc", "itemsView": "summary",
}, deadline))
ids = [item.get("id") for item in turns]
if (not ids or any(not isinstance(value, str) or not value for value in ids)
or len(set(ids)) != len(ids)
or any(item.get("status") not in {"completed", "failed", "interrupted"} for item in turns)):
if not ids:
return None
if any(not isinstance(value, str) or not value for value in ids) or len(set(ids)) != len(ids):
return contradicted()
selected = next((item for item in turns if item["id"] == turn_id), None)
if any(item.get("status") not in {"completed", "failed", "interrupted"} for item in turns):
if selected and selected.get("status") == "completed":
return contradicted()
return contradicted(
in_progress=True,
target_in_progress=bool(
selected and selected.get("status") not in {"completed", "failed", "interrupted"}
),
)
if not selected:
return None
if selected.get("status") == "interrupted" and selected.get("error") is None and ids[0] == turn_id:
Expand All @@ -534,7 +560,7 @@ def terminal() -> dict[str, Any] | None:
# only an exact user-only inventory below can prove no execution.
return {"interrupted_before_execution": True}
if selected.get("status") != "failed":
return None
return contradicted()
if _routing_discovery_timeout(selected.get("error")):
# Later work must not strand an untouched accepted request. Keep
# the historical user-only rule; only the exact latest failure
Expand All @@ -558,12 +584,12 @@ def terminal() -> dict[str, Any] | None:
status is None or type(status) is int and 500 <= status <= 599
):
return {"codex_error_info": info}
return None
return contradicted()

try:
proof = terminal()
if proof is None:
return None
if proof is None or proof.get("contradicted") is True:
return proof
items = list(session._control_items("thread/items/list", {"turnId": turn_id}, deadline))
routing = (proof.get("pre_execution_routing_timeout") is True
or proof.get("loopback_relay_auth_failure") is True)
Expand All @@ -576,14 +602,14 @@ def terminal() -> dict[str, Any] | None:
if (entry.get("turnId") != turn_id or not isinstance(item, dict)
or item.get("type") not in allowed or not item.get("id")
or not isinstance(item["id"], str) or item["id"] in ids):
return None
return contradicted()
ids.add(item["id"])
users += item["type"] == "userMessage"
if users != 1 or (not routing and len(items) != 1) or terminal() != proof:
return None
return contradicted()
if proof.get("interrupted_before_execution") and list(
session._control_items("thread/items/list", {"turnId": turn_id}, deadline)) != items:
return None
return contradicted()
proof.pop("latest_routing_failure", None)
return {"thread_id": session.thread_id, "turn_id": turn_id,
"only_original_user_item": len(items) == 1,
Expand Down Expand Up @@ -875,7 +901,9 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path,
work_dir: Path | None = None,
model_auth_home: Path | None = None,
runtime_root: Path | None = None,
consumer: str = "worker") -> dict[str, Any] | None:
consumer: str = "worker",
include_contradiction: bool = False,
contradiction_detail: bool = False) -> dict[str, Any] | None:
"""Read persisted native metadata/items; create no native conversation.

The initialized stdio client is not a listener or a model runner. It
Expand Down Expand Up @@ -906,10 +934,28 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path,
if completed_result_review:
proof = completed_result_review_readback(native, turn_id, known_turn_ids=known_turn_ids or set(),
allow_untyped=allow_untyped_result_review)
elif read_only_review:
proof = interrupted_result_readback(native, turn_id)
else:
reader = interrupted_result_readback if read_only_review else failed_request_readback
proof = reader(native, turn_id)
proof = failed_request_readback(
native,
turn_id,
include_contradiction=include_contradiction,
contradiction_detail=contradiction_detail,
)
if proof is not None:
if proof.get("contradicted") is True:
contradiction: dict[str, Any] = {
"contradicted": True,
"thread_id": thread_id,
"turn_id": turn_id,
}
if contradiction_detail:
if proof.get("in_progress") is True:
contradiction["in_progress"] = True
if proof.get("target_in_progress") is True:
contradiction["target_in_progress"] = True
return contradiction
if proof.get("codex_error_info") == "usageLimitExceeded":
# Reuse the optional provider's TS availability owner;
# Absence fails closed.
Expand Down
Loading
Loading