diff --git a/loopx/capabilities/native_chat/antigravity_adapter.py b/loopx/capabilities/native_chat/antigravity_adapter.py index 4311d965ca..f3b58d1697 100644 --- a/loopx/capabilities/native_chat/antigravity_adapter.py +++ b/loopx/capabilities/native_chat/antigravity_adapter.py @@ -264,8 +264,13 @@ def discover_antigravity_language_server( execution caller without that deadline retains the existing phase limits. """ source_env = os.environ if env is None else env - explicit_url = str(source_env.get("ANTIGRAVITY_LS_URL") or "").strip() explicit_token = str(source_env.get("ANTIGRAVITY_CSRF_TOKEN") or "").strip() + raw_url = str( + source_env.get("ANTIGRAVITY_LS_URL") + or (source_env.get("ANTIGRAVITY_LS_ADDRESS") if explicit_token else "") + or "" + ).strip() + explicit_url = f"http://{raw_url}" if raw_url and "://" not in raw_url else raw_url if explicit_url or explicit_token: if not explicit_url or not explicit_token: return None @@ -1409,6 +1414,12 @@ def observe(method: str, args: dict[str, Any], # A running status alone is not progress. Observe only this input's # steps, including streamed text and command output; old history or # a stuck RUNNING flag cannot keep an invocation alive indefinitely. + non_cleared_errors = [ + index for index, step in current_steps + if step.get("type") == "CORTEX_STEP_TYPE_ERROR_MESSAGE" + and step.get("status") != "CORTEX_STEP_STATUS_CLEARED" + ] + first_error_index = non_cleared_errors[0] if non_cleared_errors else None progress = [(index, step.get("status"), (step.get("plannerResponse") or {}).get("modifiedResponse") or (step.get("plannerResponse") or {}).get("response"), @@ -1416,7 +1427,17 @@ def observe(method: str, args: dict[str, Any], (step.get("runCommand") or {}).get("combinedOutput"), (step.get("runCommand") or {}).get("exitCode")) for index, step in current_steps - if step.get("type") != "CORTEX_STEP_TYPE_ERROR_MESSAGE"] + if step.get("type") != "CORTEX_STEP_TYPE_ERROR_MESSAGE" + and not ( + first_error_index is not None + and index > first_error_index + and step.get("type") == "CORTEX_STEP_TYPE_PLANNER_RESPONSE" + and not ( + str((step.get("plannerResponse") or {}).get("modifiedResponse") + or (step.get("plannerResponse") or {}).get("response") or "").strip() + or (step.get("plannerResponse") or {}).get("toolCalls") + ) + )] now = time.monotonic() if progress and progress != previous_progress: previous_progress = progress diff --git a/loopx/capabilities/native_chat/external_conversations.py b/loopx/capabilities/native_chat/external_conversations.py index ba050d2a6f..b77bcebb86 100644 --- a/loopx/capabilities/native_chat/external_conversations.py +++ b/loopx/capabilities/native_chat/external_conversations.py @@ -730,10 +730,27 @@ def _preparation_scope_available(self, session: dict[str, Any], *, runtime_prepa # Preparation never starts a model request. Check the same # optional transport owner before reserving a bounded retry. if session.get("agent_id") == "antigravity": - from .antigravity_adapter import discover_antigravity_language_server + from .antigravity_adapter import _rpc_post, discover_antigravity_language_server - if discover_antigravity_language_server(deadline=time.monotonic() + 3.0) is None: + discovered = discover_antigravity_language_server(deadline=time.monotonic() + 3.0) + if discovered is None: return False + cascade_id = str(session.get("upstream_thread_id") or "").strip() + if cascade_id: + from ...chat_agent import CodexChatAgentError + + try: + trajectory = _rpc_post( + str(discovered["url"]), + str(discovered["csrf_token"]), + "GetCascadeTrajectory", + {"cascadeId": cascade_id}, + timeout=2.5, + ) + except CodexChatAgentError: + return False + if not isinstance(trajectory, dict) or trajectory.get("status") != "CASCADE_RUN_STATUS_IDLE": + return False else: from .codex_auth import native_api_transport native_api_transport(self.controller.coordination_runtime_root, diff --git a/loopx/control_plane/collaboration/conversation_binding.ts b/loopx/control_plane/collaboration/conversation_binding.ts index 9af05bca17..30927c00ef 100644 --- a/loopx/control_plane/collaboration/conversation_binding.ts +++ b/loopx/control_plane/collaboration/conversation_binding.ts @@ -1648,23 +1648,33 @@ function legacyImagePreparation(recovery: JsonObject, failed: JsonObject): boole */ function planRuntimePreparationRecovery(recovery: JsonObject, row: JsonObject, current: JsonObject, failed: JsonObject, version: number): JsonObject | null { - const expectedError = new Map([ - ["model_auth_unavailable", "Native account API transport is unavailable."], - ["resume_failed", "The previous Agent conversation could not be restored."], - ["execution_worker_exited_before_dispatch", "The LoopX Chat worker exited before the original request was dispatched."], + const expectedErrors = new Map([ + ["model_auth_unavailable", ["Native account API transport is unavailable."]], + ["resume_failed", [ + "The previous Agent conversation could not be restored.", + ...(current.agent_id === "antigravity" + ? ["The original Antigravity conversation is still active. Stop it in Antigravity before retrying."] + : []), + ]], + ["execution_worker_exited_before_dispatch", ["The LoopX Chat worker exited before the original request was dispatched."]], ]).get(String(failed.error_code)); - if (expectedError === undefined) return null; + if (expectedErrors === undefined) return null; const no = {operation: "admit_turn", recovery_candidate: false, recovery_eligible: false}; const lostWorker = failed.error_code === "execution_worker_exited_before_dispatch"; + const stewardManagerContext = !lostWorker && row.origin === "lark" && current.steward_context != null + && failed.execution_dispatch == null && failed.loopx_request == null && !failed.loopx_execution + && (failed.attachments === undefined || Array.isArray(failed.attachments) && failed.attachments.length === 0) + && typeof failed.started_at === "string" && Number.isFinite(Date.parse(failed.started_at)) + && typeof failed.first_event_at === "string" && Number.isFinite(Date.parse(failed.first_event_at)); const beforeDispatch = lostWorker ? row.origin === "lark" && current.steward_context != null && failed.execution_dispatch === "pending" && failed.loopx_request == null && !failed.loopx_execution && (failed.attachments === undefined || Array.isArray(failed.attachments) && failed.attachments.length === 0) && typeof failed.started_at === "string" && Number.isFinite(Date.parse(failed.started_at)) - : failed.started_at === null && failed.first_event_at === null; + : (failed.started_at === null && failed.first_event_at === null) || stewardManagerContext; if (!["codex", "antigravity"].includes(String(current.agent_id)) || failed.session_id !== current.session_id || failed.turn_id !== row.turn_id - || failed.status !== "failed" || failed.error !== expectedError || !beforeDispatch + || failed.status !== "failed" || !expectedErrors.includes(String(failed.error)) || !beforeDispatch || failed.response !== null || failed.upstream_turn_id !== null || failed.delta_count !== 0 || failed.message !== row.message || failed.origin !== row.origin || recovery.attachments_present !== false || recovery.steering_observed !== false @@ -1673,13 +1683,16 @@ function planRuntimePreparationRecovery(recovery: JsonObject, row: JsonObject, c : requireJsonObject(recovery.preparation_observation, "runtime preparation readback"); if (observed?.kind !== "core_runtime_preparation_failure" || observed.session_id !== failed.session_id || observed.turn_id !== failed.turn_id || !Array.isArray(observed.events) - || observed.events.length < 2 || (!lostWorker && observed.events.length !== 2)) return no; + || observed.events.length < 2 + || (!lostWorker && !stewardManagerContext && observed.events.length !== 2) + || (stewardManagerContext && observed.events.length <= 2)) return no; const events = observed.events.map(value => requireJsonObject(value, "runtime preparation event")); const failure = events.at(-1)!; if (!events.every(item => typeof item.event_id === "string" && item.event_id.length > 0) || new Set(events.map(item => item.event_id)).size !== events.length || events[0].kind !== "turn.queued" || events[0].delivery_mode !== "session_queue" || failure.kind !== "turn.failed" - || failure.error_code !== failed.error_code || failure.message !== expectedError + || failure.error_code !== failed.error_code || failure.message !== failed.error + || (stewardManagerContext && !events.slice(1, -1).some(item => item.kind === "manager.context")) || events.slice(1, -1).some(item => item.kind !== "manager.context" && !(item.kind === "agent.phase" && item.phase === "manager_context"))) return no; const original = `external-${ref(row.request_ref, "original request")}`; diff --git a/tests/control_plane_ts/conversation_binding.test.ts b/tests/control_plane_ts/conversation_binding.test.ts index 54fa12b6a9..7478c5bd60 100644 --- a/tests/control_plane_ts/conversation_binding.test.ts +++ b/tests/control_plane_ts/conversation_binding.test.ts @@ -2181,3 +2181,56 @@ for (const [kind, grant] of [["project", "workspace_write"], ["steward", "portfo } }); } + +test("antigravity steward pre-dispatch active-cascade resume_failed after manager context recovers when cascade is idle", () => { + const antigravityBinding = {...row, context_kind: "steward", grant: "portfolio_read", goal_ids: [], executor_endpoint_id: "antigravity"}; + const configured = planConversationBinding({...request, binding: antigravityBinding}).state; + const source = {source_ref: "e".repeat(24), sender_ref: row.operator_ref, private_human_message: true}; + const context = resolveBoundConversation({current: configured, binding_id: row.binding_id, + observation, available_projects: [project], ...source}).context; + const session = {session_id: "original", goal_id: "loopx-manager", agent_id: "antigravity", + executor_endpoint_id: "antigravity", upstream_mode: "default", status: "ready", + upstream_thread_id: "antigravity-native", active_turn_id: null, + channel_id: `manager.external.native.${row.binding_id}.${source.source_ref}`, steward_context: context}; + const ref = "d".repeat(24); + const activeError = "The original Antigravity conversation is still active. Stop it in Antigravity before retrying."; + const failed = {session_id: "original", turn_id: "prep-failed", client_turn_id: `external-${ref}`, + message: "Daily report and monitor structure", origin: "lark", status: "failed", delta_count: 0, response: null, + upstream_turn_id: null, started_at: "2026-01-01T00:00:00Z", first_event_at: "2026-01-01T00:00:01Z", + error_code: "resume_failed", error: activeError, completed_at: "2026-01-01T00:00:30Z"}; + const proof = {kind: "core_runtime_preparation_failure", session_id: "original", turn_id: failed.turn_id, + events: [ + {event_id: "1", kind: "turn.queued", delivery_mode: "session_queue"}, + {event_id: "2", kind: "agent.phase", phase: "manager_context"}, + {event_id: "3", kind: "manager.context", schema_version: "manager_turn_context_v1"}, + {event_id: "4", kind: "turn.failed", error_code: failed.error_code, message: activeError}, + ]}; + const packet = { + request: {request_ref: ref, command: null, status: "accepted", delivery_verified: true, + message: failed.message, origin: "lark", binding_id: row.binding_id, session_id: "original", + turn_id: failed.turn_id, source}, + binding: antigravityBinding, + current_session: session, + context_recovery: {version: 23, failed_turn: failed, observed_at: "2026-01-01T00:02:30Z", + source_verified: true, steering_observed: false, stop_observed: false, attachments_present: false, + native_result_present: false, queued_input_present: false, later_equivalent_input_present: false, + preparation_scope_available: true, preparation_observation: proof}, + }; + const plan = planBoundConversationRequest(packet); + assert.equal(plan.recovery_candidate, true); + assert.equal(plan.recovery_eligible, true); + assert.equal(plan.preparation_attempt, 1); + assert.equal(plan.client_turn_id, `external-${ref}-recovery-23-preparation-1`); + assert.equal(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, preparation_scope_available: false}}).recovery_eligible, false); + assert.equal(planBoundConversationRequest({...packet, + binding: {...antigravityBinding, executor_endpoint_id: "codex"}, + current_session: {...session, agent_id: "codex", executor_endpoint_id: "codex", upstream_mode: "chat"}}).recovery_candidate, false); + for (const mutation of [{execution_dispatch: "committed"}, {upstream_turn_id: "native-1"}, {delta_count: 1}]) { + assert.equal(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, failed_turn: {...failed, ...mutation}}}).recovery_candidate, false); + } + assert.equal(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, + preparation_observation: {...proof, events: [proof.events[0], proof.events[1], proof.events[3]]}}}).recovery_candidate, false); +}); diff --git a/tests/test_antigravity_steward_adapter.py b/tests/test_antigravity_steward_adapter.py index 8e56ccad56..e1fa93bb46 100644 --- a/tests/test_antigravity_steward_adapter.py +++ b/tests/test_antigravity_steward_adapter.py @@ -1813,3 +1813,50 @@ def test_antigravity_endpoint_model_capacity_resolves_short_aliases_and_rejects_ model_id="carbon", reasoning_effort="xhigh", ) is None assert len(language_server["calls"]) == calls_before + + +def test_discover_antigravity_ls_address_with_explicit_token(language_server, monkeypatch) -> None: + from loopx.capabilities.native_chat.antigravity_adapter import discover_antigravity_language_server + + monkeypatch.delenv("ANTIGRAVITY_LS_URL", raising=False) + monkeypatch.setenv("ANTIGRAVITY_LS_ADDRESS", language_server["url"].removeprefix("http://")) + found = discover_antigravity_language_server() + assert found is not None + assert found["url"] == language_server["url"] + assert found["source"] == "environment" + + +def test_repeated_native_error_plus_thinking_only_planner_retry_times_out_and_cancels( + tmp_path, language_server, monkeypatch +) -> None: + from loopx.chat_agent import CodexChatAgentError + + user = {"type": "CORTEX_STEP_TYPE_USER_INPUT", "status": "CORTEX_STEP_STATUS_DONE", + "metadata": {"sourceTrajectoryStepInfo": {"stepIndex": 0}}} + frames = [] + steps = [user] + step_idx = 1 + for at in (1.0, 120.0, 240.0, 305.0): + steps = [ + *steps, + { + "type": "CORTEX_STEP_TYPE_ERROR_MESSAGE", + "status": "CORTEX_STEP_STATUS_DONE", + "metadata": {"sourceTrajectoryStepInfo": {"stepIndex": step_idx}}, + "errorMessage": {"error": {"shortError": "The stream was interrupted."}}, + }, + { + "type": "CORTEX_STEP_TYPE_PLANNER_RESPONSE", + "status": "CORTEX_STEP_STATUS_DONE", + "metadata": {"sourceTrajectoryStepInfo": {"stepIndex": step_idx + 1}}, + "plannerResponse": {"thinking": f"retrying thinking attempt {step_idx}"}, + }, + ] + step_idx += 2 + frames.append({"at": at, "steps": steps, "status": "CASCADE_RUN_STATUS_RUNNING"}) + _native_clock(monkeypatch, language_server, frames) + with pytest.raises(CodexChatAgentError) as raised: + adapter_for(tmp_path).start_turn("original task") + assert raised.value.error_code == "response_timeout" + assert language_server["now"] == 305.0 + assert language_server["cancelled"].is_set() diff --git a/tests/test_chat_runtime_preparation_recovery.py b/tests/test_chat_runtime_preparation_recovery.py index f413df1220..d715cf59f3 100644 --- a/tests/test_chat_runtime_preparation_recovery.py +++ b/tests/test_chat_runtime_preparation_recovery.py @@ -3,10 +3,12 @@ import pytest +from test_antigravity_steward_adapter import language_server # noqa: F401 from test_chat_ordinary_project import ordinary # noqa: F401 from test_chat_queue_preparation import session_runtime from test_lark_private_conversations import connect from test_manager_preparation_recovery import advance_recovery_clock +from test_native_steward_private import steward, finish # noqa: F401 from loopx.chat import CHAT_RESPONSE_EXAMPLE_SUFFIX from loopx.chat_agent import CodexChatAgentError from loopx.chat_store import _read_json @@ -162,3 +164,69 @@ def committed(**kwargs): assert sum(json.loads(line)["method"] == "turn/start" for line in before.decode().splitlines()) == 1 finally: runtime.close() + + +def test_antigravity_steward_active_cascade_resume_failed_waits_for_idle_then_recovers( + steward, language_server, monkeypatch, # noqa: F811 +): + from loopx.capabilities.steward_executor import ( + STEWARD_EXECUTOR_SCHEMA, + effective_steward_executor_defaults, + ) + + store, runtime, provider, transport, binding, _capture, _workspace = steward + defaults = effective_steward_executor_defaults({ + "schema_version": "loopx_machine_configuration_v0", + "namespaces": {"steward_executor": { + "schema_version": STEWARD_EXECUTOR_SCHEMA, + "selection_policy": "preferred", + "executor_endpoint": "antigravity", + "eligible_endpoints": [], + "executor_model": "gemini-4-argon-high", + "executor_reasoning_effort": "high", + }}, + }) + monkeypatch.setattr(runtime, "steward_executor_defaults", lambda: defaults) + transport.bindings.configure( + transport_ref="steward-app", + project_ref=binding["project_ref"], + executor_endpoint_id="antigravity", + context_kind="steward", + ) + + assert transport.admit("steward-app", provider.event("steward-app", "warmup", "Warm up cascade"))["status"] == "durably_accepted" + warmup = transport.core.pending()[0] + assert finish(runtime, warmup)["status"] == "completed" + assert transport.reconcile() >= 1 + + # Simulate an active orphaned cascade while Steward's second request enters _run_turn. + language_server["mode"] = "running" + language_server["active"] = True + assert transport.admit("steward-app", provider.event("steward-app", "active-msg", "Daily report and monitors"))["status"] == "durably_accepted" + row = next(item for item in transport.core.pending() if item["request_ref"] != warmup["request_ref"]) + failed = finish(runtime, row) + assert failed["status"] == "failed" and failed["error_code"] == "resume_failed" + assert failed["upstream_turn_id"] is None and failed["delta_count"] == 0 + kinds = [ev["kind"] for ev in store.events_after(row["session_id"], failed["turn_id"], None)] + assert kinds[0] == "turn.queued" and "manager.context" in kinds[1:-1] and kinds[-1] == "turn.failed" + assert transport.reconcile() >= 1 + assert transport.core.read_request(row["request_ref"])["delivery_verified"] is True + + advance_recovery_clock(monkeypatch) + # While the cascade remains RUNNING, auto-recovery stays candidate-only and defers admission. + assert transport.core.context_recovery_candidate(row["request_ref"]) is True + transport.reconcile() + assert transport.core.read_request(row["request_ref"])["turn_id"] == failed["turn_id"] + + # Once the cascade becomes IDLE, auto-recovery admits the single continuation and delivers. + language_server["active"] = False + language_server["mode"] = "complete" + transport.reconcile_request(transport.root / f"{row['request_ref']}.json") + current = transport.core.read_request(row["request_ref"]) + assert current["turn_id"] != failed["turn_id"] + assert current["context_recovery"]["preparation_attempt"] == 1 + assert finish(runtime, current)["status"] == "completed" + transport.reconcile() + record = _read_json(transport.root / f"{row['request_ref']}.json") + assert any(key.startswith("terminal.recovery.") and value.get("verified") + for key, value in record["deliveries"].items())