Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 75 additions & 2 deletions loopx/capabilities/native_chat/codex_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import threading
import time
from typing import Any, Callable, Mapping, TYPE_CHECKING
from urllib.parse import urlsplit

from ...chat_agent import (
CodexChatAgentError, CodexChatAgentSession, CodexChatTimeoutError,
Expand Down Expand Up @@ -446,6 +447,63 @@ def compact_native_history(
return import_headroom


def _loopback_api_fallback_auth_failure(error: Any) -> dict[str, Any] | None:
"""Return the typed error payload only for local OpenAI relay 401 failures."""
info = _retry_error_details(error).get("retry", {}).get("codex_error_info")
if not isinstance(info, dict) or len(info) != 1:
return None
variant, detail = next(iter(info.items()))
if (variant not in {"httpConnectionFailed", "responseStreamConnectionFailed"}
or not isinstance(detail, dict) or type(detail.get("httpStatusCode")) is not int
or detail.get("httpStatusCode") != 401):
return None
message = error.get("message") if isinstance(error, dict) else None
if not isinstance(message, str) or "OpenAI API fallback request could not complete., url: " not in message:
return None
_, url_part = message.rsplit(", url: ", 1)
url_token = url_part.split()[0] if url_part.split() else ""
parsed = urlsplit(url_token)
if (parsed.scheme != "http" or parsed.hostname != "127.0.0.1"
or parsed.port is None or not 1024 <= parsed.port <= 65535
or parsed.path not in {"/v1/responses", "/steward/v1/responses"}):
return None
return info


def _loopback_relay_transport_ready(runtime_root: Path, codex_home: Path, consumer: str) -> tuple[bool, float]:
"""Verify current native-account or relay health for recovering a relay-401 Turn."""
from loopx_codex_provider_routing import openai_relay
from loopx_codex_provider_routing.native import NativeAccounts
from loopx_codex_provider_routing.native_host import HostError

owner = NativeAccounts(runtime_root)
now = float(int(time.time()))
try:
state = owner._load()
adapter = state.get("adapter")
profile_home = adapter.get("profile", {}).get("home") if isinstance(adapter, dict) else None
if not isinstance(profile_home, str) or not profile_home:
return False, now
if Path(profile_home).expanduser().resolve() != codex_home.expanduser().resolve():
return False, now
transport = owner.model_transport(codex_home=codex_home, consumer=consumer)
if transport is not None:
relay = openai_relay.route(owner)
return bool(openai_relay.healthy(relay, consumer=consumer) and owner.api_keys().candidates("")), now
status = owner.status()
if status.get("available") is not True or status.get("recovery_required") is not False or status.get("login_matches") is not True:
return False, now
if status.get("api_active") is True:
relay = openai_relay.route(owner)
return bool(openai_relay.healthy(relay, consumer=consumer) and owner.api_keys().candidates("")), now
active_slot = status.get("active_slot")
accounts = {row.get("slot"): row for row in (status.get("accounts") or []) if isinstance(row, dict)}
return (isinstance(active_slot, str) and bool(active_slot)
and accounts.get(active_slot, {}).get("availability") == "available"), now
except (HostError, ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError):
return False, now


def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dict[str, Any] | None:
"""Observe a historical pre-execution failure without resuming it.

Expand Down Expand Up @@ -483,6 +541,10 @@ def terminal() -> dict[str, Any] | None:
# can use the compaction exception to that inventory.
return {"codex_error_info": "other", "pre_execution_routing_timeout": True,
"latest_routing_failure": ids[0] == turn_id}
loopback_auth_info = _loopback_api_fallback_auth_failure(selected.get("error"))
if loopback_auth_info is not None:
return {"codex_error_info": loopback_auth_info, "loopback_relay_auth_failure": True,
"latest_routing_failure": ids[0] == turn_id}
info = _retry_error_details(selected.get("error")).get("retry", {}).get("codex_error_info")
# `other` is evidence of a failed native request, not permission to
# retry it. The typed admission owner separately decides whether
Expand All @@ -503,7 +565,8 @@ def terminal() -> dict[str, Any] | None:
if proof is None:
return None
items = list(session._control_items("thread/items/list", {"turnId": turn_id}, deadline))
routing = proof.get("pre_execution_routing_timeout") is True
routing = (proof.get("pre_execution_routing_timeout") is True
or proof.get("loopback_relay_auth_failure") is True)
allowed = ({"userMessage", "contextCompaction"}
if routing and proof["latest_routing_failure"] else {"userMessage"})
ids: set[str] = set()
Expand Down Expand Up @@ -801,7 +864,9 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path,
allow_untyped_result_review: bool = False,
known_turn_ids: set[str] | None = None,
work_dir: Path | None = None,
model_auth_home: Path | None = None) -> dict[str, Any] | None:
model_auth_home: Path | None = None,
runtime_root: Path | None = None,
consumer: str = "worker") -> dict[str, Any] | None:
"""Read persisted native metadata/items; create no native conversation.

The initialized stdio client is not a listener or a model runner. It
Expand Down Expand Up @@ -858,6 +923,14 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path,
quota_observed_at=now)
except (ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError):
proof["quota_available"] = False
elif proof.get("loopback_relay_auth_failure") is True:
proof["relay_transport_ready"] = False
if runtime_root is not None:
try:
ready, observed_at = _loopback_relay_transport_ready(runtime_root, codex_home, consumer)
proof.update(relay_transport_ready=ready, relay_observed_at=observed_at)
except (ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError):
proof["relay_transport_ready"] = False
return proof
if time.monotonic() >= deadline:
break
Expand Down
2 changes: 2 additions & 0 deletions loopx/capabilities/native_chat/external_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,8 @@ def recover_failed_request(self, request_ref: str, *,
proof = CodexAppServerAdapter.failed_request_observation(
codex_bin=self.controller.codex_bin, codex_home=Path(session["codex_home"]),
model_auth_home=model_auth_home,
runtime_root=self.controller.coordination_runtime_root,
consumer="steward" if session.get("steward_context") else "worker",
read_only_review=bool(plan.get("read_only_review")) and not quota_answer,
thread_ids=plan["native_thread_ids"], turn_id=turn["upstream_turn_id"])
if (proof is not None and proof.get("codex_error_info") == "usageLimitExceeded"
Expand Down
15 changes: 14 additions & 1 deletion loopx/control_plane/collaboration/conversation_binding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1795,7 +1795,16 @@ function planRoutingRecovery(recovery: JsonObject, row: JsonObject, current: Jso
const observed = raw == null ? null : requireJsonObject(raw, "pre-execution failure readback");
const interrupted = observed?.interrupted_before_execution === true
&& ["host_gate", "upstream_transport_failure"].includes(String(failed.error_code));
if (observed && !quota && !preparation && observed.pre_execution_routing_timeout !== true && !interrupted) return null;
const info = observed?.codex_error_info;
const infoEntries = info && typeof info === "object" && !Array.isArray(info) ? Object.entries(info) : [];
const infoDetail = infoEntries.length === 1
&& ["httpConnectionFailed", "responseStreamConnectionFailed"].includes(infoEntries[0][0])
&& infoEntries[0][1] !== null && typeof infoEntries[0][1] === "object" && !Array.isArray(infoEntries[0][1])
? infoEntries[0][1] as JsonObject : null;
const loopbackRelayAuth = observed?.loopback_relay_auth_failure === true
&& failed.error_code === "upstream_transport_failure"
&& infoDetail?.httpStatusCode === 401;
if (observed && !quota && !preparation && observed.pre_execution_routing_timeout !== true && !interrupted && !loopbackRelayAuth) return null;
const no = {operation: "admit_turn", recovery_candidate: false, recovery_eligible: false};
const reserved = previous?.failed_turn_id === failed.turn_id;
const priorAttempt = previous?.[attemptKey] ?? 0;
Expand Down Expand Up @@ -1844,6 +1853,10 @@ function planRoutingRecovery(recovery: JsonObject, row: JsonObject, current: Jso
&& observed.quota_available === true && typeof observed.quota_observed_at === "number"
&& now / 1000 >= observed.quota_observed_at && now / 1000 - observed.quota_observed_at <= 120
: interrupted ? recovery.stop_observed === false && observed.only_original_user_item === true
: loopbackRelayAuth ? observed.only_pre_execution_items === true
&& observed.relay_transport_ready === true
&& typeof observed.relay_observed_at === "number" && Number.isFinite(observed.relay_observed_at)
&& now / 1000 >= observed.relay_observed_at && now / 1000 - observed.relay_observed_at <= 120
: observed.codex_error_info === "other" && observed.only_pre_execution_items === true)};
}

Expand Down
48 changes: 48 additions & 0 deletions tests/control_plane_ts/conversation_binding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2133,3 +2133,51 @@ test("antigravity and flexible-pool steward sessions recover pre-dispatch resume
context_recovery: {...packet.context_recovery, failed_turn: {...failed, ...postDispatch}}}).recovery_candidate, false);
}
});

for (const [kind, grant] of [["project", "workspace_write"], ["steward", "portfolio_read"]] as const) {
test(`${kind} loopback relay 401 failure (${grant}) recovers only while fresh relay readiness holds`, () => {
const selected = kind === "steward"
? {...row, context_kind: "steward", grant, goal_ids: []}
: {...row, context_kind: "project", grant};
const availableProjects = [kind === "steward" ? project : {...project, grant}];
const configured = planConversationBinding({...request, binding: selected, available_projects: availableProjects}).state;
const source = {source_ref: "e".repeat(24), sender_ref: row.operator_ref, private_human_message: true};
const context = resolveBoundConversation({current: configured, binding_id: row.binding_id,
observation, available_projects: availableProjects, ...source}).context;
const session = {session_id: "original", goal_id: kind === "steward" ? "loopx-manager" : null,
agent_id: "codex", upstream_mode: "chat", status: "ready",
upstream_thread_id: "native", active_turn_id: null, manager_context_version: 23,
channel_id: `${kind === "steward" ? "manager.external.native" : "project.external"}.${row.binding_id}.${source.source_ref}`,
...(kind === "steward" ? {steward_context: context} : {project_context: context})};
const failed = {session_id: "original", turn_id: "failed", status: "failed", delta_count: 0,
response: null, error_code: "upstream_transport_failure", upstream_turn_id: "native-failed",
completed_at: "2026-01-01T00:00:00Z"};
const observedAt = Date.parse("2026-01-01T00:10:00Z") / 1000;
const observed = {thread_id: "native", turn_id: "native-failed",
codex_error_info: {httpConnectionFailed: {httpStatusCode: 401}},
loopback_relay_auth_failure: true, only_pre_execution_items: true, only_original_user_item: true,
relay_transport_ready: true, relay_observed_at: observedAt};
const packet = {request: {request_ref: "f".repeat(24), command: null, status: "accepted", delivery_verified: true,
message: "Original synthetic request", origin: "lark",
binding_id: row.binding_id, session_id: "original", turn_id: "failed", source},
binding: selected, current_session: session, context_recovery: {version: 23, failed_turn: failed,
observed_at: "2026-01-01T00:10:00Z", steering_observed: false, source_verified: true, native_observation: observed}};
const plan = planBoundConversationRequest(packet);
assert.equal(plan.recovery_eligible, true);
assert.equal(plan.routing_attempt, 1);
assert.equal(plan.client_turn_id, `external-${"f".repeat(24)}-recovery-23-routing-1`);
for (const change of [{source_verified: false}, {stop_observed: true}, {steering_observed: true},
{later_equivalent_input_present: true},
{native_observation: {...observed, relay_transport_ready: false}},
{native_observation: {...observed, relay_observed_at: observedAt - 121}},
{native_observation: {...observed, relay_observed_at: observedAt + 1}},
{native_observation: {...observed, loopback_relay_auth_failure: false}},
{native_observation: {...observed, only_pre_execution_items: false}},
{native_observation: {...observed, codex_error_info: {httpConnectionFailed: {httpStatusCode: 403}}}},
{native_observation: {...observed, codex_error_info: {responseStreamDisconnected: {httpStatusCode: 401}}}},
{failed_turn: {...failed, error_code: "host_gate"}}]) {
assert.notEqual(planBoundConversationRequest({...packet,
context_recovery: {...packet.context_recovery, ...change}}).recovery_eligible, true);
}
});
}
39 changes: 39 additions & 0 deletions tests/test_chat_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1140,6 +1140,45 @@ def write(packet):
assert proof is None


@pytest.mark.parametrize("compact", [False, True])
@pytest.mark.parametrize("fault", [None, "external-url", "status-403", "disconnect-variant", "tool", "answer", "older-compaction"])
def test_loopback_relay_auth_readback_is_strictly_bounded(tmp_path, monkeypatch, compact, fault):
session = chat_agent.CodexChatAgentSession(process=SimpleNamespace(poll=lambda: None),
messages=queue.Queue(), thread_id="relay-thread", work_dir=tmp_path, response_timeout_sec=.2)
url = "https://api.openai.com/v1/responses" if fault == "external-url" else "http://127.0.0.1:57500/v1/responses"
status = 403 if fault == "status-403" else 401
variant = "responseStreamDisconnected" if fault == "disconnect-variant" else "httpConnectionFailed"
prefix = "Error running remote compact task: " if compact else ""
message = f"{prefix}unexpected status {status}: OpenAI API fallback request could not complete., url: {url}"

def write(packet):
method = packet["method"]
if method == "thread/turns/list":
rows = [{"id": "failed", "status": "failed", "error": {
"codexErrorInfo": {variant: {"httpStatusCode": status}}, "message": message}}]
if fault == "older-compaction":
rows.insert(0, {"id": "newer", "status": "completed"})
result = {"data": rows}
else:
items = [{"turnId": "failed", "item": {"id": "user", "type": "userMessage"}}]
if compact:
items.append({"turnId": "failed", "item": {"id": "compact", "type": "contextCompaction"}})
if fault in {"tool", "answer"}:
items.append({"turnId": "failed", "item": {"id": "extra",
"type": "commandExecution" if fault == "tool" else "agentMessage"}})
result = {"data": items}
session.messages.put({"id": packet["id"], "result": result})

monkeypatch.setattr(session, "_write", write)
proof = failed_request_readback(session, "failed")
if fault is None or (fault == "older-compaction" and not compact):
assert proof == {"thread_id": "relay-thread", "turn_id": "failed",
"only_original_user_item": not compact, "only_pre_execution_items": True,
"codex_error_info": {variant: {"httpStatusCode": 401}}, "loopback_relay_auth_failure": True}
else:
assert proof is None


@pytest.mark.parametrize("message", [None, {}, [], "workspace routing discovery timed out: extra",
"workspace routing discovery failed: extra", "Please retry: workspace routing discovery failed", "unknown"])
def test_routing_diagnostic_is_an_exact_provider_diagnostic(message):
Expand Down
Loading
Loading