Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 29 additions & 6 deletions loopx/chat_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
"Your conversation is saved; this turn was not retried automatically. "
"You can continue with a follow-up in this session."
)
WORKER_EXITED_CODE = "execution_worker_exited"
WORKER_EXITED_MESSAGE = (
"The LoopX Chat execution worker exited before this turn finished. "
"Your conversation is saved; execution may have occurred. "
Expand Down Expand Up @@ -757,12 +758,17 @@ def recover_exited_turn(self, session_id: str, turn_id: str | None = None) -> No
# Stop deliberately retires the execution worker before its
# caller commits `interrupted`; a free fence is expected here.
return
if managed_turn_recovery_allowed(
session, turn, managed=bool(session and session.get("session_mode") == CHAT_SESSION_MODE_MANAGED),
owner_dead=True,
):
managed = bool(session and session.get("session_mode") == CHAT_SESSION_MODE_MANAGED)
if (managed and turn and turn.get("status") == "completing"
and not turn.get("completion_id")):
# All response effects preceded the durable reservation. Finish its
# idempotent transcript/events under the original execution fence;
# the running service can recover it without a restart or replay.
self.store.finalize_managed_turn_completion(session_id, turn_id)
return
if managed_turn_recovery_allowed(session, turn, managed=managed, owner_dead=True):
code, message = (("server_restarted", SERVER_RESTARTED_MESSAGE)
if state == LOCK_HOLDER_DEAD else ("runtime_error", WORKER_EXITED_MESSAGE))
if state == LOCK_HOLDER_DEAD else (WORKER_EXITED_CODE, WORKER_EXITED_MESSAGE))
self._fail_turn(
session_id, turn_id, code, message,
status="failed", expected_statuses={"starting", "running", "interrupting"},
Expand All @@ -775,7 +781,8 @@ def recover_exited_turn(self, session_id: str, turn_id: str | None = None) -> No
self.store.complete_recovered_terminalization(
session_id, turn_id, session, turn,
recovery_messages={"server_restarted": SERVER_RESTARTED_MESSAGE,
"runtime_error": WORKER_EXITED_MESSAGE},
WORKER_EXITED_CODE: WORKER_EXITED_MESSAGE,
"runtime_error": WORKER_EXITED_MESSAGE}, # Legacy exact record.
)

def _run_turn(
Expand All @@ -801,6 +808,7 @@ def _run_turn(
# Turn was never dispatched, so the single-flight guard must be given
# back here rather than left holding the key, which would make every
# later dispatch of this same Turn a silent no-op.
phase = "start"
try:
with try_exclusive_file_lock(
self.store.turn_execution_lock_path(session_id, turn_id),
Expand All @@ -824,6 +832,7 @@ def _run_turn(
# Reuse one code revision across this worker's typed decisions.
# This caches no Session, Turn or authority facts: each checkpoint
# still reads current state, including after a long provider wait.
phase = "execute_or_finalize"
with effect_runtime_request_scope():
retry_binding: dict[str, str] = {}
while self._run_started_turn(
Expand All @@ -832,6 +841,12 @@ def _run_turn(
loopx_execution=loopx_execution, retry_binding=retry_binding,
):
pass
except OSError as exc:
# The queue will reconcile this Turn. Keep a correlated diagnostic
# without logging exception text (which may contain private paths).
logging.getLogger(__name__).warning(
"Chat worker I/O failure: turn=%s phase=%s errno=%s", turn_id, phase, exc.errno)
raise
finally:
self._release_done_event(key, done_event)

Expand Down Expand Up @@ -904,6 +919,7 @@ def event_sink(kind: str, payload: dict[str, Any]) -> None:
if kind == "native_goal.status":
self.store.update_session(session_id, native_goal=payload)

completion_reserved = False
try:
if execution_ended():
return
Expand Down Expand Up @@ -1120,6 +1136,7 @@ def record_owner_resume() -> None:
)
if completed_turn is None:
return
completion_reserved = True
if adapter.upstream_thread_id != str((self.store.load_session(session_id) or {}).get("upstream_thread_id") or ""):
self.store.update_session(session_id, upstream_thread_id=adapter.upstream_thread_id)
if not native_loopx and scope.get("bound_steward") is True:
Expand Down Expand Up @@ -1163,6 +1180,12 @@ def record_owner_resume() -> None:
last_error_code="transport_disconnected",
)
except Exception as exc: # noqa: BLE001 - preserve compact runtime failure.
if completion_reserved and isinstance(exc, OSError):
# The answer already exists. Its lifecycle owner can finish
# publication without turning this into an execution failure.
logging.getLogger(__name__).warning(
"Chat worker I/O failure: turn=%s phase=publish_saved_answer errno=%s", turn_id, exc.errno)
return
event_buffer.close()
self._fail_turn(session_id, turn_id, "runtime_error", str(exc), status="failed")
finally:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ export function planBoundConversationRequest(params: JsonObject): JsonObject {
|| turn?.status === "failed"
&& ["usage_limit_exceeded", "response_timeout", "upstream_transport_failure", "context_window_exceeded",
"antigravity_rpc_error", "antigravity_transport_error", "antigravity_step_error",
"manager_authorization_unavailable", "runtime_error", "server_restarted"].includes(String(turn.error_code)))
"manager_authorization_unavailable", "runtime_error", "execution_worker_exited", "server_restarted"].includes(String(turn.error_code)))
&& !(current?.session_id === row.session_id && current?.status === "closed")
&& observation.stop_observed !== true};
}
Expand Down
4 changes: 4 additions & 0 deletions loopx/extensions/lark/manager_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def manager_failure_reply(
"host_gate": "执行器未能完成本次调用,请检查管家执行器的具体错误",
"upstream_invalid_request": "上游拒绝了请求参数,请检查所选模型与 Codex CLI、账户的兼容性",
"server_restarted": "管家服务在处理过程中重启",
"execution_worker_exited": "本轮执行任务提前退出",
"runtime_unavailable": "本地运行环境初始化失败,请检查 LoopX 安装与运行配置,修复后重试",
"resume_failed": "原 Agent 会话恢复失败,请检查执行器状态,修复后重试",
"model_auth_unavailable": "账户模型连接暂时不可用",
Expand Down Expand Up @@ -101,6 +102,9 @@ def manager_failure_reply(
if code == "server_restarted":
return code, ("处理被管家服务重启中断,原请求和会话已保留。系统会先核对原执行记录,"
"符合条件的由原队列恢复结果回复;执行效果不明的操作不会直接重做。")
if code == "execution_worker_exited":
return code, ("本轮执行任务在完整答复保存前退出,原请求和会话已保留。"
"已有操作可能已生效,系统不会自动重跑这条请求。可用 /status 查看状态。")
# Old hosts can terminate with `other` after typed disconnect retries.
# Describe that observed history without changing the terminal category,
# inferring an authorization failure, or granting permission to replay.
Expand Down
26 changes: 11 additions & 15 deletions loopx/extensions/lark/private_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -1866,24 +1866,19 @@ def inbox() -> Path:
# A name supplies presentation metadata only; outbound
# provenance and Core recovery/admission keep their own
# fresh checks, including on the completed-stop path.
config = self._inbox(record, observation=selected["observation"] if selected else None,
reuse_display_name=control_receipt_first and selected is None)
config = self._inbox(record, observation=selected["observation"] if selected else None, reuse_display_name=control_receipt_first and selected is None)
return config

try:
if record["status"] in {"command_queued", "command_completed", "commission_running"}:
native = self.core.read_request(record["request_ref"])
if native["status"] == "command_queued":
selected = self.bindings.resolve(binding_id=record["binding_id"],
include_observation=True, **record["source"])
selected = self.bindings.resolve(binding_id=record["binding_id"], include_observation=True, **record["source"])
self._feedback(path, record, inbox=inbox)
self._deliver(path, record, "admission", native["response"], inbox=inbox)
return 0
record.update(status=native["status"], response=native.get("response"),
commission_resources=native.get("commission_resources"),
status_snapshot=native.get("status_snapshot"))
control_receipt_first = (native["status"] == "command_completed"
and native.get("command") in CHAT_CONTROL_COMMANDS.values())
record.update(status=native["status"], response=native.get("response"), commission_resources=native.get("commission_resources"), status_snapshot=native.get("status_snapshot"))
control_receipt_first = (native["status"] == "command_completed" and native.get("command") in CHAT_CONTROL_COMMANDS.values())
# A completed group stop has no Core work left. Its existing
# Inbox name is presentation only; sending still runs both
# fresh binding/source observations in _source_verified. Do
Expand All @@ -1893,8 +1888,7 @@ def inbox() -> Path:
and record["source"].get("group_human_message") is True
and not record["deliveries"].get("terminal", {}).get("verified")
and _same_fields(native, record, "request_ref", "binding_id", "source")):
selected = self.bindings.resolve(binding_id=record["binding_id"],
include_observation=True, **record["source"])
selected = self.bindings.resolve(binding_id=record["binding_id"], include_observation=True, **record["source"])
if record["status"] == "accepted":
native = self.core.read_request(record["request_ref"])
if native.get("agent_target"):
Expand Down Expand Up @@ -1925,6 +1919,10 @@ def inbox() -> Path:
turn = self.core.controller.store.load_turn(record["session_id"], record["turn_id"])
if turn and turn["status"] in {"starting", "running"}:
self._progress(path, record, inbox=inbox)
if turn and turn["status"] == "completing":
# Finish an orphaned completing turn's saved answer under the original request without replaying.
self.core.controller.recover_exited_turn(record["session_id"], record["turn_id"])
turn = self.core.controller.store.load_turn(record["session_id"], record["turn_id"])
if not turn or turn["status"] not in TERMINAL_TURN_STATES:
return 0
if turn["status"] == "failed":
Expand Down Expand Up @@ -1977,11 +1975,9 @@ def inbox() -> Path:
if not self._deliver(path, record, "commission_result", result_text, inbox=inbox):
return 0
config = inbox()
acknowledge_lark_event_inbox(project=self.runtime_root, config_path=config,
message_ids=[record["event"]["message_id"]], execute=True)
acknowledge_lark_event_inbox(project=self.runtime_root, config_path=config, message_ids=[record["event"]["message_id"]], execute=True)
if native_path.exists():
self.core.record_delivery(record["request_ref"], session_id=record.get("session_id"),
turn_id=record.get("turn_id"), message_text=response)
self.core.record_delivery(record["request_ref"], session_id=record.get("session_id"), turn_id=record.get("turn_id"), message_text=response)
elif record["status"] != "rejected":
raise ValueError("delivery request journal unavailable")
record["status"] = "delivered"
Expand Down
2 changes: 1 addition & 1 deletion tests/control_plane_ts/conversation_binding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1968,7 +1968,7 @@ test("quota backlog projection retires explicit lifecycle stops, not unavailable
});

for (const errorCode of ["response_timeout", "upstream_transport_failure", "context_window_exceeded",
"antigravity_rpc_error", "antigravity_transport_error", "antigravity_step_error", "runtime_error", "server_restarted"]) {
"antigravity_rpc_error", "antigravity_transport_error", "antigravity_step_error", "runtime_error", "execution_worker_exited", "server_restarted"]) {
test(`delivered ${errorCode} notices retain the exact failed request, including other executors`, () => {
const request = {request_ref: "f".repeat(24), command: null, status: "accepted",
session_id: "original", turn_id: "timed-out", delivery_verified: true};
Expand Down
50 changes: 50 additions & 0 deletions tests/test_chat_queue_preparation.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,56 @@ def write(packet):
assert store.messages(sid)[0]["attachments"] == attachments


def test_live_queue_recovers_reserved_answer_then_runs_followup(tmp_path, monkeypatch, caplog):
import errno
import queue
from types import SimpleNamespace
from loopx.chat_agent import CodexChatAgentSession

store, sid, runtime = session_runtime(tmp_path)
original, _ = store.create_queued_turn(sid, client_turn_id="first", message="First request.")
followup, _ = store.create_queued_turn(sid, client_turn_id="second", message="Follow-up request.")
native = CodexChatAgentSession(process=SimpleNamespace(poll=lambda: None),
messages=queue.Queue(), thread_id="public-thread", work_dir=tmp_path)
sent = []

def write(packet):
assert packet["method"] == "turn/start"
sent.append(packet["params"]["input"][0]["text"])
text = f"Answer {len(sent)}."
native.messages.put({"id": packet["id"], "result": {"turn": {"id": f"native-{len(sent)}"}}})
native.messages.put({"method": "item/completed", "params": {"item": {
"type": "agentMessage", "phase": "final_answer", "text": text}}})
native.messages.put({"method": "turn/completed", "params": {"turn": {"status": "completed"}}})

publish = store.append_completed_response_events
failures = []

def fail_first_publication(session_id, turn_id, **kwargs):
if turn_id == original["turn_id"] and not failures:
failures.append(turn_id)
raise OSError(errno.EIO, "PRIVATE_ERROR_DETAIL")
return publish(session_id, turn_id, **kwargs)

monkeypatch.setattr(native, "_write", write)
monkeypatch.setattr(runtime, "_ensure_adapter", lambda *a, **kw: CodexAppServerAdapter(native))
monkeypatch.setattr(store, "append_completed_response_events", fail_first_publication)
runtime.resume_session_queue(session_id=sid, work_dir=tmp_path, objective="Answer requests")
try:
assert runtime.wait_for_turn(session_id=sid, turn_id=followup["turn_id"], timeout_sec=10)["status"] == "completed"
first = store.load_turn(sid, original["turn_id"])
assert first["status"] == "completed" and first["response"]["message"] == "Answer 1."
assert failures == [original["turn_id"]]
assert len(sent) == 2 and "First request." in sent[0] and "Follow-up request." in sent[1]
assert [m["text"] for m in store.messages(sid) if m["role"] == "agent"] == ["Answer 1.", "Answer 2."]
events = store.events_after(sid, original["turn_id"], None)
assert [e["kind"] for e in events if e["kind"] in {"turn.completed", "turn.failed"}] == ["turn.completed"]
assert f"turn={original['turn_id']} phase=publish_saved_answer errno={errno.EIO}" in caplog.text
assert "PRIVATE_ERROR_DETAIL" not in caplog.text
finally:
runtime.close()


@pytest.mark.parametrize("failure", ["before_message", "after_message", "before_event"])
@pytest.mark.parametrize("display", ["Execution request", "Original visible request 原文"])
def test_queue_replay_leaves_prepared_managed_input_with_its_owner(tmp_path, monkeypatch, failure, display):
Expand Down
Loading
Loading