Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions loopx/capabilities/native_chat/antigravity_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,13 @@ def discover_antigravity_language_server(
execution caller without that deadline retains the existing phase limits.
"""
source_env = os.environ if env is None else env
explicit_url = str(source_env.get("ANTIGRAVITY_LS_URL") or "").strip()
explicit_token = str(source_env.get("ANTIGRAVITY_CSRF_TOKEN") or "").strip()
raw_url = str(
source_env.get("ANTIGRAVITY_LS_URL")
or (source_env.get("ANTIGRAVITY_LS_ADDRESS") if explicit_token else "")
or ""
).strip()
explicit_url = f"http://{raw_url}" if raw_url and "://" not in raw_url else raw_url
if explicit_url or explicit_token:
if not explicit_url or not explicit_token:
return None
Expand Down Expand Up @@ -1409,14 +1414,30 @@ def observe(method: str, args: dict[str, Any],
# A running status alone is not progress. Observe only this input's
# steps, including streamed text and command output; old history or
# a stuck RUNNING flag cannot keep an invocation alive indefinitely.
non_cleared_errors = [
index for index, step in current_steps
if step.get("type") == "CORTEX_STEP_TYPE_ERROR_MESSAGE"
and step.get("status") != "CORTEX_STEP_STATUS_CLEARED"
]
first_error_index = non_cleared_errors[0] if non_cleared_errors else None
progress = [(index, step.get("status"),
(step.get("plannerResponse") or {}).get("modifiedResponse")
or (step.get("plannerResponse") or {}).get("response"),
(step.get("plannerResponse") or {}).get("thinking"),
(step.get("runCommand") or {}).get("combinedOutput"),
(step.get("runCommand") or {}).get("exitCode"))
for index, step in current_steps
if step.get("type") != "CORTEX_STEP_TYPE_ERROR_MESSAGE"]
if step.get("type") != "CORTEX_STEP_TYPE_ERROR_MESSAGE"
and not (
first_error_index is not None
and index > first_error_index
and step.get("type") == "CORTEX_STEP_TYPE_PLANNER_RESPONSE"
and not (
str((step.get("plannerResponse") or {}).get("modifiedResponse")
or (step.get("plannerResponse") or {}).get("response") or "").strip()
or (step.get("plannerResponse") or {}).get("toolCalls")
)
)]
now = time.monotonic()
if progress and progress != previous_progress:
previous_progress = progress
Expand Down
21 changes: 19 additions & 2 deletions loopx/capabilities/native_chat/external_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -730,10 +730,27 @@ def _preparation_scope_available(self, session: dict[str, Any], *, runtime_prepa
# Preparation never starts a model request. Check the same
# optional transport owner before reserving a bounded retry.
if session.get("agent_id") == "antigravity":
from .antigravity_adapter import discover_antigravity_language_server
from .antigravity_adapter import _rpc_post, discover_antigravity_language_server

if discover_antigravity_language_server(deadline=time.monotonic() + 3.0) is None:
discovered = discover_antigravity_language_server(deadline=time.monotonic() + 3.0)
if discovered is None:
return False
cascade_id = str(session.get("upstream_thread_id") or "").strip()
if cascade_id:
from ...chat_agent import CodexChatAgentError

try:
trajectory = _rpc_post(
str(discovered["url"]),
str(discovered["csrf_token"]),
"GetCascadeTrajectory",
{"cascadeId": cascade_id},
timeout=2.5,
)
except CodexChatAgentError:
return False
if not isinstance(trajectory, dict) or trajectory.get("status") != "CASCADE_RUN_STATUS_IDLE":
return False
else:
from .codex_auth import native_api_transport
native_api_transport(self.controller.coordination_runtime_root,
Expand Down
31 changes: 22 additions & 9 deletions loopx/control_plane/collaboration/conversation_binding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1648,23 +1648,33 @@ function legacyImagePreparation(recovery: JsonObject, failed: JsonObject): boole
*/
function planRuntimePreparationRecovery(recovery: JsonObject, row: JsonObject, current: JsonObject,
failed: JsonObject, version: number): JsonObject | null {
const expectedError = new Map([
["model_auth_unavailable", "Native account API transport is unavailable."],
["resume_failed", "The previous Agent conversation could not be restored."],
["execution_worker_exited_before_dispatch", "The LoopX Chat worker exited before the original request was dispatched."],
const expectedErrors = new Map<string, string[]>([
["model_auth_unavailable", ["Native account API transport is unavailable."]],
["resume_failed", [
"The previous Agent conversation could not be restored.",
...(current.agent_id === "antigravity"
? ["The original Antigravity conversation is still active. Stop it in Antigravity before retrying."]
: []),
]],
["execution_worker_exited_before_dispatch", ["The LoopX Chat worker exited before the original request was dispatched."]],
]).get(String(failed.error_code));
if (expectedError === undefined) return null;
if (expectedErrors === undefined) return null;
const no = {operation: "admit_turn", recovery_candidate: false, recovery_eligible: false};
const lostWorker = failed.error_code === "execution_worker_exited_before_dispatch";
const stewardManagerContext = !lostWorker && row.origin === "lark" && current.steward_context != null
&& failed.execution_dispatch == null && failed.loopx_request == null && !failed.loopx_execution
&& (failed.attachments === undefined || Array.isArray(failed.attachments) && failed.attachments.length === 0)
&& typeof failed.started_at === "string" && Number.isFinite(Date.parse(failed.started_at))
&& typeof failed.first_event_at === "string" && Number.isFinite(Date.parse(failed.first_event_at));
const beforeDispatch = lostWorker
? row.origin === "lark" && current.steward_context != null && failed.execution_dispatch === "pending"
&& failed.loopx_request == null && !failed.loopx_execution
&& (failed.attachments === undefined || Array.isArray(failed.attachments) && failed.attachments.length === 0)
&& typeof failed.started_at === "string" && Number.isFinite(Date.parse(failed.started_at))
: failed.started_at === null && failed.first_event_at === null;
: (failed.started_at === null && failed.first_event_at === null) || stewardManagerContext;
if (!["codex", "antigravity"].includes(String(current.agent_id))
|| failed.session_id !== current.session_id || failed.turn_id !== row.turn_id
|| failed.status !== "failed" || failed.error !== expectedError || !beforeDispatch
|| failed.status !== "failed" || !expectedErrors.includes(String(failed.error)) || !beforeDispatch
|| failed.response !== null || failed.upstream_turn_id !== null
|| failed.delta_count !== 0 || failed.message !== row.message || failed.origin !== row.origin
|| recovery.attachments_present !== false || recovery.steering_observed !== false
Expand All @@ -1673,13 +1683,16 @@ function planRuntimePreparationRecovery(recovery: JsonObject, row: JsonObject, c
: requireJsonObject(recovery.preparation_observation, "runtime preparation readback");
if (observed?.kind !== "core_runtime_preparation_failure" || observed.session_id !== failed.session_id
|| observed.turn_id !== failed.turn_id || !Array.isArray(observed.events)
|| observed.events.length < 2 || (!lostWorker && observed.events.length !== 2)) return no;
|| observed.events.length < 2
|| (!lostWorker && !stewardManagerContext && observed.events.length !== 2)
|| (stewardManagerContext && observed.events.length <= 2)) return no;
const events = observed.events.map(value => requireJsonObject(value, "runtime preparation event"));
const failure = events.at(-1)!;
if (!events.every(item => typeof item.event_id === "string" && item.event_id.length > 0)
|| new Set(events.map(item => item.event_id)).size !== events.length || events[0].kind !== "turn.queued"
|| events[0].delivery_mode !== "session_queue" || failure.kind !== "turn.failed"
|| failure.error_code !== failed.error_code || failure.message !== expectedError
|| failure.error_code !== failed.error_code || failure.message !== failed.error
|| (stewardManagerContext && !events.slice(1, -1).some(item => item.kind === "manager.context"))
|| events.slice(1, -1).some(item => item.kind !== "manager.context"
&& !(item.kind === "agent.phase" && item.phase === "manager_context"))) return no;
const original = `external-${ref(row.request_ref, "original request")}`;
Expand Down
53 changes: 53 additions & 0 deletions tests/control_plane_ts/conversation_binding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2181,3 +2181,56 @@ for (const [kind, grant] of [["project", "workspace_write"], ["steward", "portfo
}
});
}

test("antigravity steward pre-dispatch active-cascade resume_failed after manager context recovers when cascade is idle", () => {
const antigravityBinding = {...row, context_kind: "steward", grant: "portfolio_read", goal_ids: [], executor_endpoint_id: "antigravity"};
const configured = planConversationBinding({...request, binding: antigravityBinding}).state;
const source = {source_ref: "e".repeat(24), sender_ref: row.operator_ref, private_human_message: true};
const context = resolveBoundConversation({current: configured, binding_id: row.binding_id,
observation, available_projects: [project], ...source}).context;
const session = {session_id: "original", goal_id: "loopx-manager", agent_id: "antigravity",
executor_endpoint_id: "antigravity", upstream_mode: "default", status: "ready",
upstream_thread_id: "antigravity-native", active_turn_id: null,
channel_id: `manager.external.native.${row.binding_id}.${source.source_ref}`, steward_context: context};
const ref = "d".repeat(24);
const activeError = "The original Antigravity conversation is still active. Stop it in Antigravity before retrying.";
const failed = {session_id: "original", turn_id: "prep-failed", client_turn_id: `external-${ref}`,
message: "Daily report and monitor structure", origin: "lark", status: "failed", delta_count: 0, response: null,
upstream_turn_id: null, started_at: "2026-01-01T00:00:00Z", first_event_at: "2026-01-01T00:00:01Z",
error_code: "resume_failed", error: activeError, completed_at: "2026-01-01T00:00:30Z"};
const proof = {kind: "core_runtime_preparation_failure", session_id: "original", turn_id: failed.turn_id,
events: [
{event_id: "1", kind: "turn.queued", delivery_mode: "session_queue"},
{event_id: "2", kind: "agent.phase", phase: "manager_context"},
{event_id: "3", kind: "manager.context", schema_version: "manager_turn_context_v1"},
{event_id: "4", kind: "turn.failed", error_code: failed.error_code, message: activeError},
]};
const packet = {
request: {request_ref: ref, command: null, status: "accepted", delivery_verified: true,
message: failed.message, origin: "lark", binding_id: row.binding_id, session_id: "original",
turn_id: failed.turn_id, source},
binding: antigravityBinding,
current_session: session,
context_recovery: {version: 23, failed_turn: failed, observed_at: "2026-01-01T00:02:30Z",
source_verified: true, steering_observed: false, stop_observed: false, attachments_present: false,
native_result_present: false, queued_input_present: false, later_equivalent_input_present: false,
preparation_scope_available: true, preparation_observation: proof},
};
const plan = planBoundConversationRequest(packet);
assert.equal(plan.recovery_candidate, true);
assert.equal(plan.recovery_eligible, true);
assert.equal(plan.preparation_attempt, 1);
assert.equal(plan.client_turn_id, `external-${ref}-recovery-23-preparation-1`);
assert.equal(planBoundConversationRequest({...packet,
context_recovery: {...packet.context_recovery, preparation_scope_available: false}}).recovery_eligible, false);
assert.equal(planBoundConversationRequest({...packet,
binding: {...antigravityBinding, executor_endpoint_id: "codex"},
current_session: {...session, agent_id: "codex", executor_endpoint_id: "codex", upstream_mode: "chat"}}).recovery_candidate, false);
for (const mutation of [{execution_dispatch: "committed"}, {upstream_turn_id: "native-1"}, {delta_count: 1}]) {
assert.equal(planBoundConversationRequest({...packet,
context_recovery: {...packet.context_recovery, failed_turn: {...failed, ...mutation}}}).recovery_candidate, false);
}
assert.equal(planBoundConversationRequest({...packet,
context_recovery: {...packet.context_recovery,
preparation_observation: {...proof, events: [proof.events[0], proof.events[1], proof.events[3]]}}}).recovery_candidate, false);
});
47 changes: 47 additions & 0 deletions tests/test_antigravity_steward_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -1813,3 +1813,50 @@ def test_antigravity_endpoint_model_capacity_resolves_short_aliases_and_rejects_
model_id="carbon", reasoning_effort="xhigh",
) is None
assert len(language_server["calls"]) == calls_before


def test_discover_antigravity_ls_address_with_explicit_token(language_server, monkeypatch) -> None:
from loopx.capabilities.native_chat.antigravity_adapter import discover_antigravity_language_server

monkeypatch.delenv("ANTIGRAVITY_LS_URL", raising=False)
monkeypatch.setenv("ANTIGRAVITY_LS_ADDRESS", language_server["url"].removeprefix("http://"))
found = discover_antigravity_language_server()
assert found is not None
assert found["url"] == language_server["url"]
assert found["source"] == "environment"


def test_repeated_native_error_plus_thinking_only_planner_retry_times_out_and_cancels(
tmp_path, language_server, monkeypatch
) -> None:
from loopx.chat_agent import CodexChatAgentError

user = {"type": "CORTEX_STEP_TYPE_USER_INPUT", "status": "CORTEX_STEP_STATUS_DONE",
"metadata": {"sourceTrajectoryStepInfo": {"stepIndex": 0}}}
frames = []
steps = [user]
step_idx = 1
for at in (1.0, 120.0, 240.0, 305.0):
steps = [
*steps,
{
"type": "CORTEX_STEP_TYPE_ERROR_MESSAGE",
"status": "CORTEX_STEP_STATUS_DONE",
"metadata": {"sourceTrajectoryStepInfo": {"stepIndex": step_idx}},
"errorMessage": {"error": {"shortError": "The stream was interrupted."}},
},
{
"type": "CORTEX_STEP_TYPE_PLANNER_RESPONSE",
"status": "CORTEX_STEP_STATUS_DONE",
"metadata": {"sourceTrajectoryStepInfo": {"stepIndex": step_idx + 1}},
"plannerResponse": {"thinking": f"retrying thinking attempt {step_idx}"},
},
]
step_idx += 2
frames.append({"at": at, "steps": steps, "status": "CASCADE_RUN_STATUS_RUNNING"})
_native_clock(monkeypatch, language_server, frames)
with pytest.raises(CodexChatAgentError) as raised:
adapter_for(tmp_path).start_turn("original task")
assert raised.value.error_code == "response_timeout"
assert language_server["now"] == 305.0
assert language_server["cancelled"].is_set()
68 changes: 68 additions & 0 deletions tests/test_chat_runtime_preparation_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@

import pytest

from test_antigravity_steward_adapter import language_server # noqa: F401
from test_chat_ordinary_project import ordinary # noqa: F401
from test_chat_queue_preparation import session_runtime
from test_lark_private_conversations import connect
from test_manager_preparation_recovery import advance_recovery_clock
from test_native_steward_private import steward, finish # noqa: F401
from loopx.chat import CHAT_RESPONSE_EXAMPLE_SUFFIX
from loopx.chat_agent import CodexChatAgentError
from loopx.chat_store import _read_json
Expand Down Expand Up @@ -162,3 +164,69 @@ def committed(**kwargs):
assert sum(json.loads(line)["method"] == "turn/start" for line in before.decode().splitlines()) == 1
finally:
runtime.close()


def test_antigravity_steward_active_cascade_resume_failed_waits_for_idle_then_recovers(
steward, language_server, monkeypatch, # noqa: F811
):
from loopx.capabilities.steward_executor import (
STEWARD_EXECUTOR_SCHEMA,
effective_steward_executor_defaults,
)

store, runtime, provider, transport, binding, _capture, _workspace = steward
defaults = effective_steward_executor_defaults({
"schema_version": "loopx_machine_configuration_v0",
"namespaces": {"steward_executor": {
"schema_version": STEWARD_EXECUTOR_SCHEMA,
"selection_policy": "preferred",
"executor_endpoint": "antigravity",
"eligible_endpoints": [],
"executor_model": "gemini-4-argon-high",
"executor_reasoning_effort": "high",
}},
})
monkeypatch.setattr(runtime, "steward_executor_defaults", lambda: defaults)
transport.bindings.configure(
transport_ref="steward-app",
project_ref=binding["project_ref"],
executor_endpoint_id="antigravity",
context_kind="steward",
)

assert transport.admit("steward-app", provider.event("steward-app", "warmup", "Warm up cascade"))["status"] == "durably_accepted"
warmup = transport.core.pending()[0]
assert finish(runtime, warmup)["status"] == "completed"
assert transport.reconcile() >= 1

# Simulate an active orphaned cascade while Steward's second request enters _run_turn.
language_server["mode"] = "running"
language_server["active"] = True
assert transport.admit("steward-app", provider.event("steward-app", "active-msg", "Daily report and monitors"))["status"] == "durably_accepted"
row = next(item for item in transport.core.pending() if item["request_ref"] != warmup["request_ref"])
failed = finish(runtime, row)
assert failed["status"] == "failed" and failed["error_code"] == "resume_failed"
assert failed["upstream_turn_id"] is None and failed["delta_count"] == 0
kinds = [ev["kind"] for ev in store.events_after(row["session_id"], failed["turn_id"], None)]
assert kinds[0] == "turn.queued" and "manager.context" in kinds[1:-1] and kinds[-1] == "turn.failed"
assert transport.reconcile() >= 1
assert transport.core.read_request(row["request_ref"])["delivery_verified"] is True

advance_recovery_clock(monkeypatch)
# While the cascade remains RUNNING, auto-recovery stays candidate-only and defers admission.
assert transport.core.context_recovery_candidate(row["request_ref"]) is True
transport.reconcile()
assert transport.core.read_request(row["request_ref"])["turn_id"] == failed["turn_id"]

# Once the cascade becomes IDLE, auto-recovery admits the single continuation and delivers.
language_server["active"] = False
language_server["mode"] = "complete"
transport.reconcile_request(transport.root / f"{row['request_ref']}.json")
current = transport.core.read_request(row["request_ref"])
assert current["turn_id"] != failed["turn_id"]
assert current["context_recovery"]["preparation_attempt"] == 1
assert finish(runtime, current)["status"] == "completed"
transport.reconcile()
record = _read_json(transport.root / f"{row['request_ref']}.json")
assert any(key.startswith("terminal.recovery.") and value.get("verified")
for key, value in record["deliveries"].items())
Loading