From 1bf67828f478bc5e52e619e49ed362c45b9d05f0 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 13:09:30 +0800 Subject: [PATCH] fix(native-chat): recover loopback relay 401 failures and isolate test auth Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../capabilities/native_chat/codex_adapter.py | 77 ++++++++++++++- .../native_chat/external_conversations.py | 2 + .../collaboration/conversation_binding.ts | 15 ++- .../conversation_binding.test.ts | 48 +++++++++ tests/test_chat_agent.py | 39 ++++++++ tests/test_lark_private_conversations.py | 99 +++++++++++++++++++ tests/test_steward_interrupted_answer.py | 4 + 7 files changed, 281 insertions(+), 3 deletions(-) diff --git a/loopx/capabilities/native_chat/codex_adapter.py b/loopx/capabilities/native_chat/codex_adapter.py index 3cf75faa2a..1a02aeae17 100644 --- a/loopx/capabilities/native_chat/codex_adapter.py +++ b/loopx/capabilities/native_chat/codex_adapter.py @@ -18,6 +18,7 @@ import threading import time from typing import Any, Callable, Mapping, TYPE_CHECKING +from urllib.parse import urlsplit from ...chat_agent import ( CodexChatAgentError, CodexChatAgentSession, CodexChatTimeoutError, @@ -446,6 +447,63 @@ def compact_native_history( return import_headroom +def _loopback_api_fallback_auth_failure(error: Any) -> dict[str, Any] | None: + """Return the typed error payload only for local OpenAI relay 401 failures.""" + info = _retry_error_details(error).get("retry", {}).get("codex_error_info") + if not isinstance(info, dict) or len(info) != 1: + return None + variant, detail = next(iter(info.items())) + if (variant not in {"httpConnectionFailed", "responseStreamConnectionFailed"} + or not isinstance(detail, dict) or type(detail.get("httpStatusCode")) is not int + or detail.get("httpStatusCode") != 401): + return None + message = error.get("message") if isinstance(error, dict) else None + if not isinstance(message, str) or "OpenAI API fallback request could not complete., url: " not in message: + return None + _, url_part = message.rsplit(", url: ", 1) + url_token = url_part.split()[0] if url_part.split() else "" + parsed = urlsplit(url_token) + if (parsed.scheme != "http" or parsed.hostname != "127.0.0.1" + or parsed.port is None or not 1024 <= parsed.port <= 65535 + or parsed.path not in {"/v1/responses", "/steward/v1/responses"}): + return None + return info + + +def _loopback_relay_transport_ready(runtime_root: Path, codex_home: Path, consumer: str) -> tuple[bool, float]: + """Verify current native-account or relay health for recovering a relay-401 Turn.""" + from loopx_codex_provider_routing import openai_relay + from loopx_codex_provider_routing.native import NativeAccounts + from loopx_codex_provider_routing.native_host import HostError + + owner = NativeAccounts(runtime_root) + now = float(int(time.time())) + try: + state = owner._load() + adapter = state.get("adapter") + profile_home = adapter.get("profile", {}).get("home") if isinstance(adapter, dict) else None + if not isinstance(profile_home, str) or not profile_home: + return False, now + if Path(profile_home).expanduser().resolve() != codex_home.expanduser().resolve(): + return False, now + transport = owner.model_transport(codex_home=codex_home, consumer=consumer) + if transport is not None: + relay = openai_relay.route(owner) + return bool(openai_relay.healthy(relay, consumer=consumer) and owner.api_keys().candidates("")), now + status = owner.status() + if status.get("available") is not True or status.get("recovery_required") is not False or status.get("login_matches") is not True: + return False, now + if status.get("api_active") is True: + relay = openai_relay.route(owner) + return bool(openai_relay.healthy(relay, consumer=consumer) and owner.api_keys().candidates("")), now + active_slot = status.get("active_slot") + accounts = {row.get("slot"): row for row in (status.get("accounts") or []) if isinstance(row, dict)} + return (isinstance(active_slot, str) and bool(active_slot) + and accounts.get(active_slot, {}).get("availability") == "available"), now + except (HostError, ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError): + return False, now + + def failed_request_readback(session: CodexChatAgentSession, turn_id: str) -> dict[str, Any] | None: """Observe a historical pre-execution failure without resuming it. @@ -483,6 +541,10 @@ def terminal() -> dict[str, Any] | None: # can use the compaction exception to that inventory. return {"codex_error_info": "other", "pre_execution_routing_timeout": True, "latest_routing_failure": ids[0] == turn_id} + loopback_auth_info = _loopback_api_fallback_auth_failure(selected.get("error")) + if loopback_auth_info is not None: + return {"codex_error_info": loopback_auth_info, "loopback_relay_auth_failure": True, + "latest_routing_failure": ids[0] == turn_id} info = _retry_error_details(selected.get("error")).get("retry", {}).get("codex_error_info") # `other` is evidence of a failed native request, not permission to # retry it. The typed admission owner separately decides whether @@ -503,7 +565,8 @@ def terminal() -> dict[str, Any] | None: if proof is None: return None items = list(session._control_items("thread/items/list", {"turnId": turn_id}, deadline)) - routing = proof.get("pre_execution_routing_timeout") is True + routing = (proof.get("pre_execution_routing_timeout") is True + or proof.get("loopback_relay_auth_failure") is True) allowed = ({"userMessage", "contextCompaction"} if routing and proof["latest_routing_failure"] else {"userMessage"}) ids: set[str] = set() @@ -801,7 +864,9 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path, allow_untyped_result_review: bool = False, known_turn_ids: set[str] | None = None, work_dir: Path | None = None, - model_auth_home: Path | None = None) -> dict[str, Any] | None: + model_auth_home: Path | None = None, + runtime_root: Path | None = None, + consumer: str = "worker") -> dict[str, Any] | None: """Read persisted native metadata/items; create no native conversation. The initialized stdio client is not a listener or a model runner. It @@ -858,6 +923,14 @@ def failed_request_observation(*, codex_bin: str, codex_home: Path, quota_observed_at=now) except (ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError): proof["quota_available"] = False + elif proof.get("loopback_relay_auth_failure") is True: + proof["relay_transport_ready"] = False + if runtime_root is not None: + try: + ready, observed_at = _loopback_relay_transport_ready(runtime_root, codex_home, consumer) + proof.update(relay_transport_ready=ready, relay_observed_at=observed_at) + except (ImportError, KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError): + proof["relay_transport_ready"] = False return proof if time.monotonic() >= deadline: break diff --git a/loopx/capabilities/native_chat/external_conversations.py b/loopx/capabilities/native_chat/external_conversations.py index 31e6cfd608..ba050d2a6f 100644 --- a/loopx/capabilities/native_chat/external_conversations.py +++ b/loopx/capabilities/native_chat/external_conversations.py @@ -585,6 +585,8 @@ def recover_failed_request(self, request_ref: str, *, proof = CodexAppServerAdapter.failed_request_observation( codex_bin=self.controller.codex_bin, codex_home=Path(session["codex_home"]), model_auth_home=model_auth_home, + runtime_root=self.controller.coordination_runtime_root, + consumer="steward" if session.get("steward_context") else "worker", read_only_review=bool(plan.get("read_only_review")) and not quota_answer, thread_ids=plan["native_thread_ids"], turn_id=turn["upstream_turn_id"]) if (proof is not None and proof.get("codex_error_info") == "usageLimitExceeded" diff --git a/loopx/control_plane/collaboration/conversation_binding.ts b/loopx/control_plane/collaboration/conversation_binding.ts index cecb9273ec..9af05bca17 100644 --- a/loopx/control_plane/collaboration/conversation_binding.ts +++ b/loopx/control_plane/collaboration/conversation_binding.ts @@ -1795,7 +1795,16 @@ function planRoutingRecovery(recovery: JsonObject, row: JsonObject, current: Jso const observed = raw == null ? null : requireJsonObject(raw, "pre-execution failure readback"); const interrupted = observed?.interrupted_before_execution === true && ["host_gate", "upstream_transport_failure"].includes(String(failed.error_code)); - if (observed && !quota && !preparation && observed.pre_execution_routing_timeout !== true && !interrupted) return null; + const info = observed?.codex_error_info; + const infoEntries = info && typeof info === "object" && !Array.isArray(info) ? Object.entries(info) : []; + const infoDetail = infoEntries.length === 1 + && ["httpConnectionFailed", "responseStreamConnectionFailed"].includes(infoEntries[0][0]) + && infoEntries[0][1] !== null && typeof infoEntries[0][1] === "object" && !Array.isArray(infoEntries[0][1]) + ? infoEntries[0][1] as JsonObject : null; + const loopbackRelayAuth = observed?.loopback_relay_auth_failure === true + && failed.error_code === "upstream_transport_failure" + && infoDetail?.httpStatusCode === 401; + if (observed && !quota && !preparation && observed.pre_execution_routing_timeout !== true && !interrupted && !loopbackRelayAuth) return null; const no = {operation: "admit_turn", recovery_candidate: false, recovery_eligible: false}; const reserved = previous?.failed_turn_id === failed.turn_id; const priorAttempt = previous?.[attemptKey] ?? 0; @@ -1844,6 +1853,10 @@ function planRoutingRecovery(recovery: JsonObject, row: JsonObject, current: Jso && observed.quota_available === true && typeof observed.quota_observed_at === "number" && now / 1000 >= observed.quota_observed_at && now / 1000 - observed.quota_observed_at <= 120 : interrupted ? recovery.stop_observed === false && observed.only_original_user_item === true + : loopbackRelayAuth ? observed.only_pre_execution_items === true + && observed.relay_transport_ready === true + && typeof observed.relay_observed_at === "number" && Number.isFinite(observed.relay_observed_at) + && now / 1000 >= observed.relay_observed_at && now / 1000 - observed.relay_observed_at <= 120 : observed.codex_error_info === "other" && observed.only_pre_execution_items === true)}; } diff --git a/tests/control_plane_ts/conversation_binding.test.ts b/tests/control_plane_ts/conversation_binding.test.ts index 72d6881225..54fa12b6a9 100644 --- a/tests/control_plane_ts/conversation_binding.test.ts +++ b/tests/control_plane_ts/conversation_binding.test.ts @@ -2133,3 +2133,51 @@ test("antigravity and flexible-pool steward sessions recover pre-dispatch resume context_recovery: {...packet.context_recovery, failed_turn: {...failed, ...postDispatch}}}).recovery_candidate, false); } }); + +for (const [kind, grant] of [["project", "workspace_write"], ["steward", "portfolio_read"]] as const) { + test(`${kind} loopback relay 401 failure (${grant}) recovers only while fresh relay readiness holds`, () => { + const selected = kind === "steward" + ? {...row, context_kind: "steward", grant, goal_ids: []} + : {...row, context_kind: "project", grant}; + const availableProjects = [kind === "steward" ? project : {...project, grant}]; + const configured = planConversationBinding({...request, binding: selected, available_projects: availableProjects}).state; + const source = {source_ref: "e".repeat(24), sender_ref: row.operator_ref, private_human_message: true}; + const context = resolveBoundConversation({current: configured, binding_id: row.binding_id, + observation, available_projects: availableProjects, ...source}).context; + const session = {session_id: "original", goal_id: kind === "steward" ? "loopx-manager" : null, + agent_id: "codex", upstream_mode: "chat", status: "ready", + upstream_thread_id: "native", active_turn_id: null, manager_context_version: 23, + channel_id: `${kind === "steward" ? "manager.external.native" : "project.external"}.${row.binding_id}.${source.source_ref}`, + ...(kind === "steward" ? {steward_context: context} : {project_context: context})}; + const failed = {session_id: "original", turn_id: "failed", status: "failed", delta_count: 0, + response: null, error_code: "upstream_transport_failure", upstream_turn_id: "native-failed", + completed_at: "2026-01-01T00:00:00Z"}; + const observedAt = Date.parse("2026-01-01T00:10:00Z") / 1000; + const observed = {thread_id: "native", turn_id: "native-failed", + codex_error_info: {httpConnectionFailed: {httpStatusCode: 401}}, + loopback_relay_auth_failure: true, only_pre_execution_items: true, only_original_user_item: true, + relay_transport_ready: true, relay_observed_at: observedAt}; + const packet = {request: {request_ref: "f".repeat(24), command: null, status: "accepted", delivery_verified: true, + message: "Original synthetic request", origin: "lark", + binding_id: row.binding_id, session_id: "original", turn_id: "failed", source}, + binding: selected, current_session: session, context_recovery: {version: 23, failed_turn: failed, + observed_at: "2026-01-01T00:10:00Z", steering_observed: false, source_verified: true, native_observation: observed}}; + const plan = planBoundConversationRequest(packet); + assert.equal(plan.recovery_eligible, true); + assert.equal(plan.routing_attempt, 1); + assert.equal(plan.client_turn_id, `external-${"f".repeat(24)}-recovery-23-routing-1`); + for (const change of [{source_verified: false}, {stop_observed: true}, {steering_observed: true}, + {later_equivalent_input_present: true}, + {native_observation: {...observed, relay_transport_ready: false}}, + {native_observation: {...observed, relay_observed_at: observedAt - 121}}, + {native_observation: {...observed, relay_observed_at: observedAt + 1}}, + {native_observation: {...observed, loopback_relay_auth_failure: false}}, + {native_observation: {...observed, only_pre_execution_items: false}}, + {native_observation: {...observed, codex_error_info: {httpConnectionFailed: {httpStatusCode: 403}}}}, + {native_observation: {...observed, codex_error_info: {responseStreamDisconnected: {httpStatusCode: 401}}}}, + {failed_turn: {...failed, error_code: "host_gate"}}]) { + assert.notEqual(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, ...change}}).recovery_eligible, true); + } + }); +} diff --git a/tests/test_chat_agent.py b/tests/test_chat_agent.py index ad81fd36f9..a620eb8e9b 100644 --- a/tests/test_chat_agent.py +++ b/tests/test_chat_agent.py @@ -1140,6 +1140,45 @@ def write(packet): assert proof is None +@pytest.mark.parametrize("compact", [False, True]) +@pytest.mark.parametrize("fault", [None, "external-url", "status-403", "disconnect-variant", "tool", "answer", "older-compaction"]) +def test_loopback_relay_auth_readback_is_strictly_bounded(tmp_path, monkeypatch, compact, fault): + session = chat_agent.CodexChatAgentSession(process=SimpleNamespace(poll=lambda: None), + messages=queue.Queue(), thread_id="relay-thread", work_dir=tmp_path, response_timeout_sec=.2) + url = "https://api.openai.com/v1/responses" if fault == "external-url" else "http://127.0.0.1:57500/v1/responses" + status = 403 if fault == "status-403" else 401 + variant = "responseStreamDisconnected" if fault == "disconnect-variant" else "httpConnectionFailed" + prefix = "Error running remote compact task: " if compact else "" + message = f"{prefix}unexpected status {status}: OpenAI API fallback request could not complete., url: {url}" + + def write(packet): + method = packet["method"] + if method == "thread/turns/list": + rows = [{"id": "failed", "status": "failed", "error": { + "codexErrorInfo": {variant: {"httpStatusCode": status}}, "message": message}}] + if fault == "older-compaction": + rows.insert(0, {"id": "newer", "status": "completed"}) + result = {"data": rows} + else: + items = [{"turnId": "failed", "item": {"id": "user", "type": "userMessage"}}] + if compact: + items.append({"turnId": "failed", "item": {"id": "compact", "type": "contextCompaction"}}) + if fault in {"tool", "answer"}: + items.append({"turnId": "failed", "item": {"id": "extra", + "type": "commandExecution" if fault == "tool" else "agentMessage"}}) + result = {"data": items} + session.messages.put({"id": packet["id"], "result": result}) + + monkeypatch.setattr(session, "_write", write) + proof = failed_request_readback(session, "failed") + if fault is None or (fault == "older-compaction" and not compact): + assert proof == {"thread_id": "relay-thread", "turn_id": "failed", + "only_original_user_item": not compact, "only_pre_execution_items": True, + "codex_error_info": {variant: {"httpStatusCode": 401}}, "loopback_relay_auth_failure": True} + else: + assert proof is None + + @pytest.mark.parametrize("message", [None, {}, [], "workspace routing discovery timed out: extra", "workspace routing discovery failed: extra", "Please retry: workspace routing discovery failed", "unknown"]) def test_routing_diagnostic_is_an_exact_provider_diagnostic(message): diff --git a/tests/test_lark_private_conversations.py b/tests/test_lark_private_conversations.py index 20980db6e4..d7279ddd8f 100644 --- a/tests/test_lark_private_conversations.py +++ b/tests/test_lark_private_conversations.py @@ -9136,3 +9136,102 @@ def reconcile_request(self, path): release.set() service.close() service._delivery_thread.join(5) + + +@pytest.mark.parametrize("relay_ready_initially", [False, True]) +def test_project_workspace_write_recovers_loopback_relay_auth_once_ready( + ordinary, monkeypatch, tmp_path, relay_ready_initially, # noqa: F811 +): + from datetime import timedelta, timezone + from pathlib import Path + from loopx.capabilities.native_chat import codex_adapter as adapter_module + from loopx.capabilities.native_chat import external_conversations + + ordinary[2].workspace_grant = "workspace_write" + store, runtime, provider, transport = connect(ordinary) + try: + capture, fake = ordinary[-3:-1] + binding = next(row for row in transport.bindings.read()["bindings"] if row["transport_ref"] == "notes-app") + assert binding["grant"] == "workspace_write" + source = fake.read_text().replace( + '"codexErrorInfo": "cyberPolicy"', + '"codexErrorInfo": {"httpConnectionFailed": {"httpStatusCode": 401}}', + ).replace( + '"private-fixture-upstream-detail"', + '"unexpected status 401 Unauthorized: OpenAI API fallback request could not complete., url: http://127.0.0.1:57500/v1/responses"', + ).replace( + ' print(json.dumps({"method": "item/agentMessage/delta", "params": {"threadId": "durable-thread", "turnId": active_turn, "delta": "Partial answer."}}), flush=True)\n', + "", + ) + fake.write_text(source) + event = provider.event("notes-app", "relay-401", "typed terminal failure") + transport.admit("notes-app", event) + ref = identity_ref(binding["provider_ref"], event["message_id"]) + original = transport.core.read_request(ref) + failed = runtime.wait_for_turn(session_id=original["session_id"], turn_id=original["turn_id"], timeout_sec=10) + assert failed["status"] == "failed" and failed["error_code"] == "upstream_transport_failure" and failed["delta_count"] == 0 + transport.reconcile() + delivery_path = transport.root / f"{ref}.json" + old_notice = json.loads(delivery_path.read_text())["deliveries"]["terminal"] + runtime.adapters.pop(original["session_id"]).close_session() + Path(store.load_session(original["session_id"])["codex_home"]).mkdir(parents=True, exist_ok=True) + inventory = tmp_path / "relay-inventory.json" + inventory.write_text(json.dumps({ + "turns": [{"id": failed["upstream_turn_id"], "status": "failed", "error": { + "codexErrorInfo": {"httpConnectionFailed": {"httpStatusCode": 401}}, + "message": "unexpected status 401 Unauthorized: OpenAI API fallback request could not complete., url: http://127.0.0.1:57500/v1/responses", + }}], + "items": [{"turnId": failed["upstream_turn_id"], "item": {"type": "userMessage", "id": "user-1"}}], + })) + begin = source.index(' elif method == "thread/turns/list":') + end = source.index(' else:\n result = {}', begin) + reader = f''' elif method in {{"thread/turns/list", "thread/items/list"}}: + with open({str(inventory)!r}) as observed: + snapshot = json.load(observed) + result = {{"data": snapshot["turns" if method == "thread/turns/list" else "items"]}} + elif method == "thread/backgroundTerminals/list": + result = {{"data": []}} +''' + fake.write_text((source[:begin] + reader + source[end:]).replace( + 'if "typed terminal failure" in json.dumps(request.get("params") or {}):', + "if False:", + )) + clock = [datetime.now(timezone.utc) + timedelta(seconds=61)] + + class AfterBackoff(datetime): + @classmethod + def now(cls, tz=None): + return clock[0] + + ready_state = [relay_ready_initially] + monkeypatch.setattr(external_conversations, "datetime", AfterBackoff) + monkeypatch.setattr( + adapter_module, + "_loopback_relay_transport_ready", + lambda *_a, **_kw: (ready_state[0], int(clock[0].timestamp())), + ) + from loopx.extensions.lark.private_progress import presentation_phase + + worker = LarkPrivateConversations(controller=runtime, runtime_root=store.root.parent, + runner=provider, cli_bin="lark-cli") + worker.reconcile() + row = worker.core.read_request(ref) + if not relay_ready_initially: + assert row["turn_id"] == original["turn_id"] and not row.get("context_recovery") + ready_state[0] = True + clock[0] += timedelta(minutes=1) + worker = LarkPrivateConversations(controller=runtime, runtime_root=store.root.parent, + runner=provider, cli_bin="lark-cli") + worker.reconcile() + row = worker.core.read_request(ref) + assert row["turn_id"] != original["turn_id"] + assert row["context_recovery"]["routing_attempt"] == 1 + assert runtime.wait_for_turn(session_id=row["session_id"], turn_id=row["turn_id"], timeout_sec=10)["status"] == "completed" + worker.reconcile() + delivered = json.loads(delivery_path.read_text()) + assert delivered["deliveries"]["terminal"] == old_notice + assert delivered["deliveries"][presentation_phase(row, "terminal")]["verified"] + starts = [json.loads(line) for line in capture.read_text().splitlines() if json.loads(line).get("method") == "turn/start"] + assert len(starts) == 2 + finally: + runtime.close() diff --git a/tests/test_steward_interrupted_answer.py b/tests/test_steward_interrupted_answer.py index 43472f82c9..50ab5d9492 100644 --- a/tests/test_steward_interrupted_answer.py +++ b/tests/test_steward_interrupted_answer.py @@ -1221,6 +1221,10 @@ def test_original_worker_answers_interrupted_steward_question_once(tmp_path, lan monkeypatch.setenv("LOOPX_MANAGER_MODEL", "gemini-4-argon-high") monkeypatch.setenv("LOOPX_MANAGER_REASONING_EFFORT", "high") + codex_home = tmp_path / "codex-home" + codex_home.mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("LOOPX_CHAT_CODEX_HOME", str(codex_home)) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) fixture = steward.__wrapped__(tmp_path) store, runtime, provider, transport, binding, capture, workspace = next(fixture) try: