From 623f22a28e0984bc1582e33c9764feb30dea736a Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 08:45:31 +0800 Subject: [PATCH] fix(steward): recover pre-dispatch runtime failures across endpoints and stop receipt-unknown disk churn Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../native_chat/external_conversations.py | 142 +++++------------- .../native_chat/file_discovery.py | 91 +++++++++++ .../collaboration/conversation_binding.ts | 15 +- .../extensions/lark/private_conversations.py | 136 ++++++----------- .../lark/private_delivery_discovery.py | 104 ++++++++++++- .../conversation_binding.test.ts | 53 +++++++ tests/test_manager_preparation_recovery.py | 75 +++++++++ 7 files changed, 413 insertions(+), 203 deletions(-) diff --git a/loopx/capabilities/native_chat/external_conversations.py b/loopx/capabilities/native_chat/external_conversations.py index 17c47fa28b..31e6cfd608 100644 --- a/loopx/capabilities/native_chat/external_conversations.py +++ b/loopx/capabilities/native_chat/external_conversations.py @@ -10,13 +10,15 @@ from copy import deepcopy from functools import cache from pathlib import Path -import hashlib import json import re import threading +import time from datetime import datetime as datetime, timezone as timezone from typing import Any +from ...chat_attachments import normalize_chat_image_attachments + from ...chat_ingress import ( STEWARD_BINDING_ADOPTION_PENDING as STEWARD_BINDING_ADOPTION_PENDING, ensure_external_admission_scope, @@ -28,7 +30,6 @@ _read_json as _read_json, RESUMABLE_SESSION_STATES, ) -from ...chat_attachments import normalize_chat_image_attachments from ...control_plane import effect_runtime from ...control_plane.quota.automation_cadence_readback import ( automation_cadence_readback, @@ -40,6 +41,11 @@ from .file_discovery import ( FileChatDiscovery, QuotaBacklogObservationCache, + _DeferredBacklogRequest, + _RecoveryDiscoveryReads, + _preparation_image_identity, + _project_source_request_digest, + _project_user_message_hint, _same_store_method, external_source_route_path, file_change_fingerprint, @@ -58,94 +64,6 @@ _QUOTA_BACKLOG_BATCH_BYTES = 1024 * 1024 -class _DeferredBacklogRequest(dict[str, Any]): - """Bounded backlog request view carrying its observed file fingerprint.""" - - __slots__ = ("req_fp", "needs_full_read") - - def __init__( - self, - data: Mapping[str, Any], - *, - req_fp: tuple[int, ...] | None, - needs_full_read: bool, - ) -> None: - super().__init__(data) - self.req_fp = req_fp - self.needs_full_read = needs_full_read - - -def _preparation_image_identity(value: Any) -> dict[str, Any] | None: - """Fingerprint already normalized Core images without exporting their bytes.""" - if not isinstance(value, list) or not value: - return None - try: - images = normalize_chat_image_attachments(value) - if images != value: - return None - encoded = json.dumps(images, sort_keys=True, separators=(",", ":")).encode() - return {"count": len(images), "sha256": hashlib.sha256(encoded).hexdigest()} - except (ValueError, TypeError, OverflowError): - return None - - -def _project_source_request_digest(_path: Path, item: dict[str, Any]) -> dict[str, Any]: - canonical = json.dumps(item, sort_keys=True, ensure_ascii=False, separators=(",", ":")) - return { - "request_ref": item.get("request_ref"), - "binding_id": item.get("binding_id"), - "source": item.get("source"), - "sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), - } - - -def _project_user_message_hint(message: Mapping[str, Any]) -> dict[str, Any]: - return { - key: message[key] - for key in ("role", "turn_id", "text", "origin", "created_at") - if key in message - } - - -class _RecoveryDiscoveryReads: - """Lazy facts for one queue-discovery pass, never recovery admission. - - Historical requests share Session history. Retain queue/control/history - hints, never provider identity or a recovery decision. A new - pass builds new observations; normal admission never receives this object. - """ - - def __init__(self, core: ChatExternalConversations) -> None: - self.core = core - self.queues: dict[str, bool] = {} - self.histories: dict[str, list[dict[str, Any]]] = {} - self.message_indexes: dict[str, dict[str, list[dict[str, Any]]]] = {} - self.controls: list[dict[str, Any]] | None = None - - def queued(self, session_id: str) -> bool: - if session_id not in self.queues: - self.queues[session_id] = bool(self.core.controller.store.queued_turns(session_id)) - return self.queues[session_id] - - def user_messages(self, session_id: str) -> list[dict[str, Any]]: - if session_id not in self.histories: - self.histories[session_id] = list(self.core._cached_user_messages(session_id)) - return self.histories[session_id] - - def source_controls(self) -> list[dict[str, Any]]: - if self.controls is None: - self.controls = self.core.source_controls() - return self.controls - - def matching_user_messages(self, session_id: str, text: str) -> list[dict[str, Any]]: - if session_id not in self.message_indexes: - index: dict[str, list[dict[str, Any]]] = {} - for row in self.user_messages(session_id): - index.setdefault(row.get("text", ""), []).append(row) - self.message_indexes[session_id] = index - return self.message_indexes[session_id].get(text, []) - - class ChatExternalConversations(ExternalStewardCommissionMixin): def __init__(self, controller: Any) -> None: self.controller = controller @@ -192,6 +110,7 @@ def _context_recovery_plan(self, row: dict[str, Any], *, discovery_only: bool = False, preparation_scope_available: bool = False, metadata_capacity: dict[str, Any] | None = None, + failure_notice_receipt_unknown: bool = False, discovery_reads: _RecoveryDiscoveryReads | None = None) -> dict[str, Any]: from ...chat_manager import MANAGER_CONTEXT_VERSION @@ -320,6 +239,7 @@ def _context_recovery_plan(self, row: dict[str, Any], *, "preparation_observation": preparation, "later_equivalent_input_present": later_equivalent, "preparation_scope_available": preparation_scope_available, + "failure_notice_receipt_unknown": bool(failure_notice_receipt_unknown), "later_source_input_present": any(item.get("binding_id") == row["binding_id"] and item.get("source") == row["source"] and item.get("command") is None and datetime.fromisoformat(item["created_at"]) > datetime.fromisoformat(row["created_at"]) @@ -530,13 +450,15 @@ def quota_backlog_pending(self, row: dict[str, Any], *, """Retain quota and declared unfinished work without admitting recovery.""" return self.quota_backlog_observer(source_controls=source_controls)([row])[0] - def context_recovery_observer(self) -> Callable[[str], bool]: + def context_recovery_observer(self) -> Callable[..., bool]: """Reuse scan-local hints; reread each request, Turn and binding.""" reads = _RecoveryDiscoveryReads(self) - return lambda ref: self.context_recovery_candidate(ref, discovery_reads=reads) + return lambda ref, *, failure_notice_receipt_unknown=False: self.context_recovery_candidate( + ref, discovery_reads=reads, failure_notice_receipt_unknown=failure_notice_receipt_unknown) def context_recovery_candidate(self, request_ref: str, *, - discovery_reads: _RecoveryDiscoveryReads | None = None) -> bool: + discovery_reads: _RecoveryDiscoveryReads | None = None, + failure_notice_receipt_unknown: bool = False) -> bool: """Discovery only; every authority and native fact is re-read at admission.""" try: row = self.read_request(request_ref) @@ -551,6 +473,7 @@ def context_recovery_candidate(self, request_ref: str, *, return True if not row.get("delivery_verified"): return self._context_recovery_plan(row, discovery_only=True, + failure_notice_receipt_unknown=failure_notice_receipt_unknown, discovery_reads=discovery_reads).get("recovery_candidate") is True turn = self.controller.store.load_turn(row["session_id"], row["turn_id"]) if not turn or turn.get("status") not in {"failed", "completed"}: @@ -563,12 +486,14 @@ def context_recovery_candidate(self, request_ref: str, *, and result_review_evidence.collect_related_result_review(self, row, turn, discovery_reads=discovery_reads) is None): return False # A lexical probe hint, never permission to adopt or send. return self._context_recovery_plan(row, discovery_only=True, + failure_notice_receipt_unknown=failure_notice_receipt_unknown, discovery_reads=discovery_reads).get("recovery_candidate") is True except (KeyError, ValueError, RuntimeError, OSError): return False def recover_failed_request(self, request_ref: str, *, - source_observation: Callable[[dict[str, Any]], dict[str, Any] | None]) -> dict[str, Any]: + source_observation: Callable[[dict[str, Any]], dict[str, Any] | None], + failure_notice_receipt_unknown: bool = False) -> dict[str, Any]: """Reserve/admit one same-source continuation through the existing queue. The failed Turn and its original notice remain immutable. The new Turn @@ -582,7 +507,7 @@ def recover_failed_request(self, request_ref: str, *, path = self.root / f"{request_ref}.json" with exclusive_file_lock(path, operation="admit_external_chat_request"): row = self.read_request(request_ref) - plan = self._context_recovery_plan(row) + plan = self._context_recovery_plan(row, failure_notice_receipt_unknown=failure_notice_receipt_unknown) if not plan.get("recovery_candidate"): return row session = self.controller.store.load_session(row["session_id"]) @@ -707,7 +632,7 @@ def recover_failed_request(self, request_ref: str, *, return row plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None, selected_binding=selected["binding"], preparation_scope_available=scope_available, - metadata_capacity=capacity) + metadata_capacity=capacity, failure_notice_receipt_unknown=failure_notice_receipt_unknown) if plan.get("retain_metadata_result") and facts and not self.controller.store.native_result(row["session_id"], turn["turn_id"]): try: self.controller.store.retain_native_result(row["session_id"], turn["turn_id"], @@ -718,7 +643,8 @@ def recover_failed_request(self, request_ref: str, *, return row # Preserve the delivered answer and unspent review. proof = self._original_native_result(row, session, turn, context, include_raw_response=True) plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None, - selected_binding=selected["binding"], metadata_capacity=capacity) + selected_binding=selected["binding"], metadata_capacity=capacity, + failure_notice_receipt_unknown=failure_notice_receipt_unknown) if not plan.get("recovery_eligible"): if plan.get("metadata_quota_wait") is not None and row.get("metadata_quota_wait") != plan["metadata_quota_wait"]: row["metadata_quota_wait"] = plan["metadata_quota_wait"] @@ -731,13 +657,13 @@ def recover_failed_request(self, request_ref: str, *, recovery = {"version": MANAGER_CONTEXT_VERSION, "failed_turn_id": turn["turn_id"], "native_observation": proof, "source_observation": source, "client_turn_id": plan["client_turn_id"], - "failure_notice_delivery_verified": row["delivery_verified"]} + "failure_notice_delivery_verified": row.get("delivery_verified") is True} if plan.get("read_only_review"): recovery.update(read_only_review=True, presentation_key=f"{MANAGER_CONTEXT_VERSION}.review.1") if plan.get("unfinished_work") or plan.get("legacy_result_review"): recovery.pop("failure_notice_delivery_verified", None) recovery.update(original_turn=deepcopy(turn), binding_snapshot=selected["binding"], - prior_result_delivery_verified=row["delivery_verified"]) + prior_result_delivery_verified=row.get("delivery_verified") is True) if plan.get("unfinished_work"): recovery["unfinished_work"] = deepcopy(plan["unfinished_work"]) else: @@ -759,7 +685,7 @@ def recover_failed_request(self, request_ref: str, *, if plan.get("work_attempt"): recovery.pop("failure_notice_delivery_verified", None) recovery.update(work_attempt=1, unfinished_work=plan["unfinished_work"], - prior_result_delivery_verified=row["delivery_verified"], + prior_result_delivery_verified=row.get("delivery_verified") is True, presentation_key=f"{MANAGER_CONTEXT_VERSION}.work.1", not_before=plan["not_before"]) if plan.get("completed_review_turn_id"): recovery.update(completed_review_turn_id=plan["completed_review_turn_id"], @@ -801,11 +727,17 @@ def _preparation_scope_available(self, session: dict[str, Any], *, runtime_prepa if runtime_preparation: # Preparation never starts a model request. Check the same # optional transport owner before reserving a bounded retry. - from .codex_auth import native_api_transport - native_api_transport(self.controller.coordination_runtime_root, - checked_session_home(self.controller.codex_home, session), - consumer="worker" if session.get("project_context") is not None else "steward", - model=session.get("requested_agent_model_id") or session.get("agent_model_id")) + if session.get("agent_id") == "antigravity": + from .antigravity_adapter import discover_antigravity_language_server + + if discover_antigravity_language_server(deadline=time.monotonic() + 3.0) is None: + return False + else: + from .codex_auth import native_api_transport + native_api_transport(self.controller.coordination_runtime_root, + checked_session_home(self.controller.codex_home, session), + consumer="worker" if session.get("project_context") is not None else "steward", + model=session.get("requested_agent_model_id") or session.get("agent_model_id")) if session.get("project_context") is not None: # Resolve the existing project grant again; a project assistant # has no manager portfolio scope to borrow. diff --git a/loopx/capabilities/native_chat/file_discovery.py b/loopx/capabilities/native_chat/file_discovery.py index 39bc49ae3f..84e0e79ffc 100644 --- a/loopx/capabilities/native_chat/file_discovery.py +++ b/loopx/capabilities/native_chat/file_discovery.py @@ -6,6 +6,7 @@ from datetime import datetime import base64 import hashlib +import json from pathlib import Path import re import sys @@ -563,3 +564,93 @@ def capture_reply_image_plan(workspace: Path, turn_id: str, targets: list[str]) except (OSError, ValueError): result.append(None) return result + + +class _DeferredBacklogRequest(dict[str, Any]): + """Bounded backlog request view carrying its observed file fingerprint.""" + + __slots__ = ("req_fp", "needs_full_read") + + def __init__( + self, + data: Mapping[str, Any], + *, + req_fp: tuple[int, ...] | None, + needs_full_read: bool, + ) -> None: + super().__init__(data) + self.req_fp = req_fp + self.needs_full_read = needs_full_read + + +def _preparation_image_identity(value: Any) -> dict[str, Any] | None: + """Fingerprint already normalized Core images without exporting their bytes.""" + from ...chat_attachments import normalize_chat_image_attachments + + if not isinstance(value, list) or not value: + return None + try: + images = normalize_chat_image_attachments(value) + if images != value: + return None + encoded = json.dumps(images, sort_keys=True, separators=(",", ":")).encode() + return {"count": len(images), "sha256": hashlib.sha256(encoded).hexdigest()} + except (ValueError, TypeError, OverflowError): + return None + + +def _project_source_request_digest(_path: Path, item: dict[str, Any]) -> dict[str, Any]: + canonical = json.dumps(item, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return { + "request_ref": item.get("request_ref"), + "binding_id": item.get("binding_id"), + "source": item.get("source"), + "sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + } + + +def _project_user_message_hint(message: Mapping[str, Any]) -> dict[str, Any]: + return { + key: message[key] + for key in ("role", "turn_id", "text", "origin", "created_at") + if key in message + } + + +class _RecoveryDiscoveryReads: + """Lazy facts for one queue-discovery pass, never recovery admission. + + Historical requests share Session history. Retain queue/control/history + hints, never provider identity or a recovery decision. A new + pass builds new observations; normal admission never receives this object. + """ + + def __init__(self, core: Any) -> None: + self.core = core + self.queues: dict[str, bool] = {} + self.histories: dict[str, list[dict[str, Any]]] = {} + self.message_indexes: dict[str, dict[str, list[dict[str, Any]]]] = {} + self.controls: list[dict[str, Any]] | None = None + + def queued(self, session_id: str) -> bool: + if session_id not in self.queues: + self.queues[session_id] = bool(self.core.controller.store.queued_turns(session_id)) + return self.queues[session_id] + + def user_messages(self, session_id: str) -> list[dict[str, Any]]: + if session_id not in self.histories: + self.histories[session_id] = list(self.core._cached_user_messages(session_id)) + return self.histories[session_id] + + def source_controls(self) -> list[dict[str, Any]]: + if self.controls is None: + self.controls = self.core.source_controls() + return self.controls + + def matching_user_messages(self, session_id: str, text: str) -> list[dict[str, Any]]: + if session_id not in self.message_indexes: + index: dict[str, list[dict[str, Any]]] = {} + for row in self.user_messages(session_id): + index.setdefault(row.get("text", ""), []).append(row) + self.message_indexes[session_id] = index + return self.message_indexes[session_id].get(text, []) diff --git a/loopx/control_plane/collaboration/conversation_binding.ts b/loopx/control_plane/collaboration/conversation_binding.ts index ef1912a6e9..cecb9273ec 100644 --- a/loopx/control_plane/collaboration/conversation_binding.ts +++ b/loopx/control_plane/collaboration/conversation_binding.ts @@ -765,11 +765,15 @@ function planContextRecovery(params: JsonObject, row: JsonObject, current: JsonO const no = {operation: "admit_turn", recovery_candidate: false, recovery_eligible: false}; const pendingTurn = recovery.failed_turn as JsonObject | undefined; const pendingResponse = pendingTurn?.response as JsonObject | undefined; - // A committed empty answer has no delivered failure to acknowledge first. - // It may only adopt its verified native result, never dispatch recovery work. + // A committed empty answer or a pre-dispatch preparation failure whose + // notice receipt is ambiguous has no delivered failure to acknowledge first. const pendingEmpty = current?.agent_id === "antigravity" && pendingTurn?.status === "completed" && pendingResponse?.message === "" && pureResultText(pendingResponse, true); - if (!current || row.command !== null || row.status !== "accepted" || row.delivery_verified !== true && !pendingEmpty + const pendingPreparationNotice = recovery.failure_notice_receipt_unknown === true + && pendingTurn?.status === "failed" && pendingTurn?.upstream_turn_id === null + && pendingTurn?.delta_count === 0 && pendingTurn?.response === null; + if (!current || row.command !== null || row.status !== "accepted" + || (row.delivery_verified !== true && !pendingEmpty && !pendingPreparationNotice) || row.agent_target != null || current.session_id !== row.session_id || !["codex", "antigravity"].includes(String(current.agent_id)) || current.upstream_mode !== (current.agent_id === "antigravity" ? "default" : "chat") || current.status === "closed" @@ -799,6 +803,8 @@ function planContextRecovery(params: JsonObject, row: JsonObject, current: JsonO if (runtimePreparation !== null) return runtimePreparation; const preparation = steward ? planPreparationRecovery(recovery, row, current, failed, version) : null; if (preparation !== null) return preparation; + // Post-dispatch and read-only recovery families always require a verified prior delivery. + if (row.delivery_verified !== true && !pendingEmpty) return no; // Once preparation has dispatched, a different terminal category must not // switch recovery families and start a fresh business retry budget. if ((row.context_recovery != null @@ -1656,7 +1662,8 @@ function planRuntimePreparationRecovery(recovery: JsonObject, row: JsonObject, c && (failed.attachments === undefined || Array.isArray(failed.attachments) && failed.attachments.length === 0) && typeof failed.started_at === "string" && Number.isFinite(Date.parse(failed.started_at)) : failed.started_at === null && failed.first_event_at === null; - if (current.agent_id !== "codex" || failed.session_id !== current.session_id || failed.turn_id !== row.turn_id + if (!["codex", "antigravity"].includes(String(current.agent_id)) + || failed.session_id !== current.session_id || failed.turn_id !== row.turn_id || failed.status !== "failed" || failed.error !== expectedError || !beforeDispatch || failed.response !== null || failed.upstream_turn_id !== null || failed.delta_count !== 0 || failed.message !== row.message || failed.origin !== row.origin diff --git a/loopx/extensions/lark/private_conversations.py b/loopx/extensions/lark/private_conversations.py index f7bf694579..15f8945434 100644 --- a/loopx/extensions/lark/private_conversations.py +++ b/loopx/extensions/lark/private_conversations.py @@ -27,19 +27,18 @@ STEWARD_BINDING_ADOPTION_PENDING, ) from ...capabilities.native_chat.file_discovery import ( - REPLY_IMAGE_NAME_PATTERN, FileChatDiscovery, optional_file_fingerprint, reply_image_workspace, + FileChatDiscovery, optional_file_fingerprint, reply_image_workspace, ) from ...chat_attachments import CHAT_IMAGE_MAX_COUNT from ...chat_store import TERMINAL_TURN_STATES, _atomic_write_json, _read_json from ...control_plane.collaboration import conversation_reply_context -from ...control_plane.content_digest import BARE_SHA256_PATTERN from ...control_plane.effect_runtime import effect_runtime_request_scope from ...file_lock import exclusive_file_lock from .conversation_identity import identity_ref, lark_private_source from .event_inbox import acknowledge_lark_event_inbox, ingest_lark_event_inbox from .event_collector_runtime import enrich_lark_event_reply_context from .goal_channel_transport import call, json_payload, lark_args, lark_provider_mention_identities, record_event_timing -from .identity_shapes import LARK_APP_ID_PATTERN as APP_ID_PATTERN, LARK_MESSAGE_ID_PATTERN +from .identity_shapes import LARK_APP_ID_PATTERN as APP_ID_PATTERN from .inbox_reply import ( SourceAuthorityVerifier, _message, reply_lark_event_inbox, update_lark_inbox_reply, verify_lark_inbox_reply, ) @@ -48,8 +47,11 @@ mark_lark_event_inbox_processing, mark_lark_event_inbox_received) from .outbound import normalize_lark_outbound_text, present_lark_conversation_text as _presentation_text from .private_images import private_message_caption, private_message_images -from .private_delivery_discovery import (PrivateDeliveryDiscovery, answer_image_source_matches, answer_media_dispatch, - historical_image_reply_matches, prepared_steward_binding_waits, private_answer_images, resumable_answer_image_session) +from .private_delivery_discovery import ( + PrivateDeliveryDiscovery, answer_image_retry_waits, answer_image_source_matches, answer_media_dispatch, + failure_notice_receipt_unknown, historical_image_plan, historical_image_reply_matches, historical_image_source, + is_answer_image_followup, prepared_steward_binding_waits, private_answer_images, resumable_answer_image_session, +) from .private_ingress import PrivateSourceUnavailable, private_source_text, recovery_source_matches, steward_source_preflight from .private_ingress import project_private_reply_history as project_private_reply_history from . import private_steward_triage as triage @@ -94,21 +96,6 @@ def _lark_payload(runner: Any, cli_bin: str, profile: str, tail: list[str]) -> M return payload -def _is_answer_image_followup(record: Mapping[str, Any]) -> bool: - if record.get("status") != "accepted" or not isinstance(record.get("answer_images"), dict): - return False - terminal = record.get("deliveries", {}).get(presentation_phase(record, "terminal"), {}) - return terminal.get("verified") is True - - -def _answer_image_retry_waits(record: Mapping[str, Any], now: float) -> bool: - if not _is_answer_image_followup(record): - return False - state = record["answer_images"] - retry_at = state.get("next_check_at") - return state.get("status") != "delivered" and type(retry_at) in (int, float) and retry_at > now - - class _ControlReadFallback(Exception): """A valid source needs ordinary rendering, before any Core write.""" @@ -1213,8 +1200,9 @@ def before_send(_intent: str) -> dict[str, bool]: before_send=before_send) elif phase_state.get("started"): # A crash between provider write and its receipt is ambiguous. Keep - # the original source recoverable; never resend blindly. - save_phase(blocker="provider_delivery_receipt_unknown") + # the original source recoverable; never resend blindly or churn disk. + if phase_state.get("blocker") != "provider_delivery_receipt_unknown": + save_phase(blocker="provider_delivery_receipt_unknown") return False else: def attempt(value: Any) -> None: @@ -1324,7 +1312,7 @@ def authorized() -> bool: if state.get("status") == "delivered" and record["deliveries"].get(media_phase, {}).get("images_verified") is True: return True now = datetime.now(timezone.utc).timestamp() - if _answer_image_retry_waits(record, now): + if answer_image_retry_waits(record, now): return False state.update(status="pending", next_check_at=now + ANSWER_IMAGE_RETRY_SECONDS) _atomic_write_json(path, record) @@ -1370,63 +1358,6 @@ def plan_scope() -> tuple[Path, str] | None: inbox=inbox, authorization_check=authorized, finalize_source=False) return False - def _historical_image_source(self, record: Mapping[str, Any], plan: Mapping[str, Any]) -> dict: - event = record["event"] - terminal = record["deliveries"]["terminal"] - if (record["status"] != "delivered" or record.get("context_recovery") - or event.get("chat_type") != "p2p" or event.get("message_type") != "text" - or record["source"].get("private_human_message") is not True - or record["binding_id"] != plan["binding_id"] or record["turn_id"] != plan["turn_id"] - or terminal.get("verified") is not True or terminal.get("image_keys") - or terminal.get("attempt", {}).get("message_ref") != plan["message_id"]): - raise ValueError("original text-only reply unavailable") - native = self.core.read_request(record["request_ref"]) - if (native.get("status") != "accepted" or native.get("origin") != "lark" - or native.get("command") is not None or native.get("agent_target") - or native.get("delivery_verified") is not True - or not _same_fields(native, record, "request_ref", "binding_id", "source", "session_id", "turn_id")): - raise ValueError("original request no longer matches") - turn = self.core.controller.store.load_turn(record["session_id"], record["turn_id"]) or {} - response = turn.get("response") or {} - if (not isinstance(answer := response.get("message"), str) - or hashlib.sha256(answer.encode()).hexdigest() != plan["answer_sha256"] - or terminal.get("text") != _presentation_text(answer) - or (session := resumable_answer_image_session(self.core, native, turn)) is None): - raise ValueError("original completed answer or its authority changed") - image_targets = private_answer_images(answer)[1] - if "captured_images" in plan: - if image_targets or response.get("answer_image_plan") or reply_image_workspace(session, turn) is None: - raise ValueError("original reply is not a qualified legacy project answer") - elif not image_targets: - raise ValueError("original remote image reply unavailable") - return session - - def _historical_image_plan(self, record: Mapping[str, Any], plan: Mapping[str, Any]) -> dict: - from ..public_source_reader import public_url - - captured = "captured_images" in plan - image_field, reference_field = ("captured_images", "name") if captured else ("images", "url") - if set(plan) != {"binding_id", "turn_id", "message_id", "answer_sha256", image_field}: - raise ValueError("invalid original image repair fields") - if (not isinstance(plan["message_id"], str) or not LARK_MESSAGE_ID_PATTERN.fullmatch(plan["message_id"]) - or not isinstance(plan["answer_sha256"], str) - or not BARE_SHA256_PATTERN.fullmatch(plan["answer_sha256"]) - or not isinstance(plan[image_field], list) or not 1 <= len(plan[image_field]) <= CHAT_IMAGE_MAX_COUNT): - raise ValueError("invalid original image repair target") - images = [] - for image in plan[image_field]: - if (not isinstance(image, dict) or set(image) != {reference_field, "sha256"} - or not isinstance(reference := image[reference_field], str) - or not isinstance(image["sha256"], str) or not BARE_SHA256_PATTERN.fullmatch(image["sha256"])): - raise ValueError("each original image requires a reference and its SHA-256") - if captured and not REPLY_IMAGE_NAME_PATTERN.fullmatch(reference): - raise ValueError("invalid original image snapshot name") - images.append({reference_field: reference if captured else public_url(reference), "sha256": image["sha256"]}) - if len({image[reference_field] for image in images}) != len(images): - raise ValueError("duplicate original image reference") - self._historical_image_source(record, plan) - return {**plan, image_field: images} - def _load_historical_image_plan(self, path: Path, record: dict[str, Any]) -> dict | None: if record.get("historical_answer_images") is not None: return record["historical_answer_images"] @@ -1444,7 +1375,7 @@ def _load_historical_image_plan(self, path: Path, record: dict[str, Any]) -> dic plan = json.loads(raw) if not isinstance(plan, dict) or path.stem != record.get("request_ref"): raise ValueError("invalid original image repair descriptor") - plan = self._historical_image_plan(record, plan) + plan = historical_image_plan(self.core, record, plan) state = record["historical_answer_images"] = {"plan": plan, "status": "pending"} _atomic_write_json(path, record) return state @@ -1463,7 +1394,7 @@ def _reconcile_historical_answer_images(self, path: Path) -> int: _atomic_write_json(path, record) plan = state["plan"] try: - session = self._historical_image_source(record, plan) + session = historical_image_source(self.core, record, plan) selected = self.bindings.resolve(binding_id=plan["binding_id"], session=session, session_context=session.get("project_context") or session.get("steward_context"), include_observation=True, **record["source"]) @@ -1504,7 +1435,7 @@ def original_pair_matches() -> bool: def authorized() -> bool: try: return (selected["binding"] in self.bindings.read()["bindings"] - and bool(self._historical_image_source(record, plan))) + and bool(historical_image_source(self.core, record, plan))) except (KeyError, OSError, ValueError): return False @@ -1552,7 +1483,8 @@ def _planned_historical_repairs(self) -> frozenset[str]: def pending_body_paths(self) -> list[Path]: """Find unsent bodies without waiting for historical recovery probes.""" paths = [path for path, row, _ in self._body_discovery.rows() - if row["status"] != "delivered" and triage.pending(row) and not _is_answer_image_followup(row)] + if row["status"] != "delivered" and triage.pending(row) + and not is_answer_image_followup(row) and not failure_notice_receipt_unknown(row)] return [*paths, *self._pending_web_projection_paths()] def _noop_backoff_cache(self) -> dict[Path, tuple[tuple[tuple[int, ...], ...], int]]: @@ -1606,8 +1538,9 @@ def pending_delivery_paths(self) -> list[Path]: planned_repairs = self._planned_historical_repairs() noop_backoff = self._noop_backoff_cache() for path, row, repairs in self._delivery_discovery.rows(): - if row["status"] != "delivered": - if triage.pending(row) and not _answer_image_retry_waits(row, now): + receipt_unknown = failure_notice_receipt_unknown(row) + if row["status"] != "delivered" and not receipt_unknown: + if triage.pending(row) and not answer_image_retry_waits(row, now): paths.append(path) elif row.get("historical_images_due_at") is not None and row["historical_images_due_at"] <= now: paths.append(path) @@ -1630,7 +1563,11 @@ def pending_delivery_paths(self) -> list[Path]: now_mono = time.monotonic() stagger = zlib.crc32(path.name.encode("utf-8")) % 60 self._context_recovery_probe_at[path] = now_mono + 60 + stagger - if repairs or recovery_candidate(row["request_ref"]): + candidate = ( + recovery_candidate(row["request_ref"], failure_notice_receipt_unknown=True) + if receipt_unknown else recovery_candidate(row["request_ref"]) + ) + if repairs or candidate: self._context_recovery_probe_at[path] = now_mono + 60 paths.append(path) return [*paths, *self._web_projection_paths()] @@ -1642,7 +1579,8 @@ def is_recovery_path(self, path: Path) -> bool: row = _read_json(path) card = row.get("triage") or {} return ( - row.get("status") == "delivered" or _is_answer_image_followup(row) + row.get("status") == "delivered" or is_answer_image_followup(row) + or failure_notice_receipt_unknown(row) or bool(card.get("send_started_at") or card.get("message_id")) ) @@ -1745,7 +1683,7 @@ def reconcile_request(self, path: Path) -> int: if historical_images is None: self._historical_image_probe_at[path] = time.monotonic() + 60 return self._reconcile_historical_answer_images(path) - if _answer_image_retry_waits(row, datetime.now(timezone.utc).timestamp()): + if answer_image_retry_waits(row, datetime.now(timezone.utc).timestamp()): return 0 record_event_timing(row["profile"], row["event"]["message_id"], "reconciliation_started") if row["status"] == "delivered" and row.get("session_id") and row.get("turn_id"): @@ -1760,13 +1698,21 @@ def reconcile_request(self, path: Path) -> int: self._repair_recovery_presentation(path) if not self.core.context_recovery_candidate(row["request_ref"]): return 0 + receipt_unknown = failure_notice_receipt_unknown(row) if (row.get("session_id") and row.get("turn_id") and (row["status"] == "delivered" - or row["status"] == "accepted" and self.core.context_recovery_candidate(row["request_ref"]))): + or (row["status"] == "accepted" and ( + self.core.context_recovery_candidate(row["request_ref"], failure_notice_receipt_unknown=True) + if receipt_unknown else self.core.context_recovery_candidate(row["request_ref"]))))): # Legacy terminal delivery may be only a verified failure notice. # Core decides whether a migration continuation is safe; preserve # the old Turn and every frozen/ambiguous provider attempt. - recovered = self.core.recover_failed_request(row["request_ref"], - source_observation=lambda selected: self._recovery_source_observation(row, selected)) + if receipt_unknown: + recovered = self.core.recover_failed_request(row["request_ref"], + source_observation=lambda selected: self._recovery_source_observation(row, selected), + failure_notice_receipt_unknown=True) + else: + recovered = self.core.recover_failed_request(row["request_ref"], + source_observation=lambda selected: self._recovery_source_observation(row, selected)) if (row["status"] == "delivered" and recovered.get("native_result_recovery") and recovered["native_result_recovery"]["turn_id"] == recovered.get("turn_id")): delivered_native = self._reconcile_native_result(path, recovered) @@ -1785,8 +1731,14 @@ def reconcile_request(self, path: Path) -> int: row.update(status="accepted", turn_id=recovered["turn_id"], context_recovery=recovered["context_recovery"]) _atomic_write_json(path, row) + elif receipt_unknown: + self._record_context_recovery_noop_backoff(path, row) + return 0 elif row["status"] == "delivered": self._record_context_recovery_noop_backoff(path, row) + elif receipt_unknown: + self._record_context_recovery_noop_backoff(path, row) + return 0 if row["status"] == "captured": # Intake may still own this capture. Wait on its existing fence and # read its result before retrying admission from a stale snapshot. diff --git a/loopx/extensions/lark/private_delivery_discovery.py b/loopx/extensions/lark/private_delivery_discovery.py index 5a42030c70..0e94714222 100644 --- a/loopx/extensions/lark/private_delivery_discovery.py +++ b/loopx/extensions/lark/private_delivery_discovery.py @@ -1,6 +1,7 @@ """Read-only source and scheduling checks for private-message delivery.""" from __future__ import annotations +import hashlib import json import re from collections.abc import Callable, Iterator, Mapping @@ -8,12 +9,23 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from ...chat_attachments import CHAT_IMAGE_MAX_COUNT from ...chat_store import RESUMABLE_SESSION_STATES, ChatSessionStore, _read_json -from ...capabilities.native_chat.file_discovery import FileChatDiscovery, external_source_route_path, file_change_fingerprint +from ...control_plane.content_digest import BARE_SHA256_PATTERN +from ...capabilities.native_chat.file_discovery import ( + REPLY_IMAGE_NAME_PATTERN, + FileChatDiscovery, + external_source_route_path, + file_change_fingerprint, + reply_image_workspace, +) from ...file_lock import try_exclusive_file_lock +from .identity_shapes import LARK_MESSAGE_ID_PATTERN +from .outbound import present_lark_conversation_text as _presentation_text from .presentation.markdown_post import lark_markdown_readback_matches from .private_ingress import private_source_text -from .private_progress import legacy_recovery_repairs +from .private_progress import legacy_recovery_repairs, presentation_phase +from .private_web_projection import _same_fields if TYPE_CHECKING: from ...capabilities.native_chat.external_conversations import ChatExternalConversations @@ -251,3 +263,91 @@ def _observe(self, path: Path) -> tuple[dict[str, Any], bool]: if before is not None and file_change_fingerprint(path) == before: self._completed[path] = (before, view, repairs) return view, repairs + + +def failure_notice_receipt_unknown(record: Mapping[str, Any]) -> bool: + """Return True when the current terminal failure notice lost its provider receipt.""" + if record.get("status") != "accepted" or "triage" in record: + return False + phase = presentation_phase(record, "terminal") + terminal = (record.get("deliveries") or {}).get(phase) or {} + return ( + terminal.get("started") is True + and not terminal.get("attempt") + and terminal.get("verified") is not True + and terminal.get("blocker") == "provider_delivery_receipt_unknown" + ) + + +def is_answer_image_followup(record: Mapping[str, Any]) -> bool: + if record.get("status") != "accepted" or not isinstance(record.get("answer_images"), dict): + return False + terminal = record.get("deliveries", {}).get(presentation_phase(record, "terminal"), {}) + return terminal.get("verified") is True + + +def answer_image_retry_waits(record: Mapping[str, Any], now: float) -> bool: + if not is_answer_image_followup(record): + return False + state = record["answer_images"] + retry_at = state.get("next_check_at") + return state.get("status") != "delivered" and type(retry_at) in (int, float) and retry_at > now + + +def historical_image_source(core: ChatExternalConversations, record: Mapping[str, Any], plan: Mapping[str, Any]) -> dict: + event = record["event"] + terminal = record["deliveries"]["terminal"] + if (record["status"] != "delivered" or record.get("context_recovery") + or event.get("chat_type") != "p2p" or event.get("message_type") != "text" + or record["source"].get("private_human_message") is not True + or record["binding_id"] != plan["binding_id"] or record["turn_id"] != plan["turn_id"] + or terminal.get("verified") is not True or terminal.get("image_keys") + or terminal.get("attempt", {}).get("message_ref") != plan["message_id"]): + raise ValueError("original text-only reply unavailable") + native = core.read_request(record["request_ref"]) + if (native.get("status") != "accepted" or native.get("origin") != "lark" + or native.get("command") is not None or native.get("agent_target") + or native.get("delivery_verified") is not True + or not _same_fields(native, record, "request_ref", "binding_id", "source", "session_id", "turn_id")): + raise ValueError("original request no longer matches") + turn = core.controller.store.load_turn(record["session_id"], record["turn_id"]) or {} + response = turn.get("response") or {} + if (not isinstance(answer := response.get("message"), str) + or hashlib.sha256(answer.encode()).hexdigest() != plan["answer_sha256"] + or terminal.get("text") != _presentation_text(answer) + or (session := resumable_answer_image_session(core, native, turn)) is None): + raise ValueError("original completed answer or its authority changed") + image_targets = private_answer_images(answer)[1] + if "captured_images" in plan: + if image_targets or response.get("answer_image_plan") or reply_image_workspace(session, turn) is None: + raise ValueError("original reply is not a qualified legacy project answer") + elif not image_targets: + raise ValueError("original remote image reply unavailable") + return session + + +def historical_image_plan(core: ChatExternalConversations, record: Mapping[str, Any], plan: Mapping[str, Any]) -> dict: + from ..public_source_reader import public_url + + captured = "captured_images" in plan + image_field, reference_field = ("captured_images", "name") if captured else ("images", "url") + if set(plan) != {"binding_id", "turn_id", "message_id", "answer_sha256", image_field}: + raise ValueError("invalid original image repair fields") + if (not isinstance(plan["message_id"], str) or not LARK_MESSAGE_ID_PATTERN.fullmatch(plan["message_id"]) + or not isinstance(plan["answer_sha256"], str) + or not BARE_SHA256_PATTERN.fullmatch(plan["answer_sha256"]) + or not isinstance(plan[image_field], list) or not 1 <= len(plan[image_field]) <= CHAT_IMAGE_MAX_COUNT): + raise ValueError("invalid original image repair target") + images = [] + for image in plan[image_field]: + if (not isinstance(image, dict) or set(image) != {reference_field, "sha256"} + or not isinstance(reference := image[reference_field], str) + or not isinstance(image["sha256"], str) or not BARE_SHA256_PATTERN.fullmatch(image["sha256"])): + raise ValueError("each original image requires a reference and its SHA-256") + if captured and not REPLY_IMAGE_NAME_PATTERN.fullmatch(reference): + raise ValueError("invalid original image snapshot name") + images.append({reference_field: reference if captured else public_url(reference), "sha256": image["sha256"]}) + if len({image[reference_field] for image in images}) != len(images): + raise ValueError("duplicate original image reference") + historical_image_source(core, record, plan) + return {**plan, image_field: images} diff --git a/tests/control_plane_ts/conversation_binding.test.ts b/tests/control_plane_ts/conversation_binding.test.ts index 56270e85b7..72d6881225 100644 --- a/tests/control_plane_ts/conversation_binding.test.ts +++ b/tests/control_plane_ts/conversation_binding.test.ts @@ -2080,3 +2080,56 @@ test("a steward session migrated via flexible_primary_available preserves web an assert.throws(() => resolveBoundConversation({...webPacket, session: {...migratedSession, ...invalid}})); } }); + + +test("antigravity and flexible-pool steward sessions recover pre-dispatch resume_failed even when failure notice receipt is unknown", () => { + const codexBinding = {...row, context_kind: "steward", grant: "portfolio_read", goal_ids: [], executor_endpoint_id: "codex"}; + const configured = planConversationBinding({...request, binding: codexBinding}).state; + const source = {source_ref: "e".repeat(24), sender_ref: row.operator_ref, private_human_message: true}; + const context = resolveBoundConversation({current: configured, binding_id: row.binding_id, + observation, available_projects: [project], ...source}).context; + const primaryAllocation = {schema_version: "manager_executor_allocation_v0", selection_policy: "flexible", + configured_endpoint: "antigravity", executor_endpoint: "antigravity", configuration_revision: "revision-2", + allocation_reason: "flexible_primary_available", eligible_endpoints: ["antigravity", "codex"]}; + const session = {session_id: "original", goal_id: "loopx-manager", agent_id: "antigravity", + executor_endpoint_id: "antigravity", upstream_mode: "default", status: "ready", + upstream_thread_id: "antigravity-native", active_turn_id: null, + channel_id: `manager.external.native.${row.binding_id}.${source.source_ref}`, steward_context: context, + manager_executor_allocation: primaryAllocation, upstream_context_history: [{agent_id: "codex", thread_id: "original-codex"}]}; + const ref = "f".repeat(24); + const failed = {session_id: "original", turn_id: "prep-failed", client_turn_id: `external-${ref}`, + message: "Compare repository execution value", origin: "lark", status: "failed", delta_count: 0, response: null, + upstream_turn_id: null, started_at: null, first_event_at: null, error_code: "resume_failed", + error: "The previous Agent conversation could not be restored.", completed_at: "2026-01-01T00:00:00Z"}; + const proof = {kind: "core_runtime_preparation_failure", session_id: "original", turn_id: failed.turn_id, + events: [{event_id: "queued", kind: "turn.queued", delivery_mode: "session_queue"}, + {event_id: "failed", kind: "turn.failed", error_code: failed.error_code, message: failed.error}]}; + const packet = { + request: {request_ref: ref, command: null, status: "accepted", delivery_verified: false, + message: failed.message, origin: "lark", binding_id: row.binding_id, session_id: "original", + turn_id: failed.turn_id, source}, + binding: codexBinding, + current_session: session, + context_recovery: {version: 23, failed_turn: failed, observed_at: "2026-01-01T00:02:00Z", + source_verified: true, steering_observed: false, stop_observed: false, attachments_present: false, + native_result_present: false, queued_input_present: false, later_equivalent_input_present: false, + preparation_scope_available: true, preparation_observation: proof, failure_notice_receipt_unknown: true}, + }; + const plan = planBoundConversationRequest(packet); + assert.equal(plan.recovery_candidate, true); + assert.equal(plan.recovery_eligible, true); + assert.equal(plan.preparation_attempt, 1); + assert.equal(plan.client_turn_id, `external-${ref}-recovery-23-preparation-1`); + // Receipt-unknown tolerance stays strictly pre-dispatch; post-dispatch or non-preparation failures remain rejected. + assert.equal(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, failure_notice_receipt_unknown: false}}).recovery_candidate, false); + for (const postDispatch of [ + {started_at: "2026-01-01T00:00:01Z"}, + {upstream_turn_id: "native-1"}, + {delta_count: 1}, + {error_code: "response_timeout", started_at: "2026-01-01T00:00:01Z", upstream_turn_id: "native-1"}, + ]) { + assert.equal(planBoundConversationRequest({...packet, + context_recovery: {...packet.context_recovery, failed_turn: {...failed, ...postDispatch}}}).recovery_candidate, false); + } +}); diff --git a/tests/test_manager_preparation_recovery.py b/tests/test_manager_preparation_recovery.py index 478f7531c5..00bcf38dcb 100644 --- a/tests/test_manager_preparation_recovery.py +++ b/tests/test_manager_preparation_recovery.py @@ -238,3 +238,78 @@ def prepare_then_lose(*args, **kwargs): assert resumed["turn_id"] == key[1] and not resumed.get("context_recovery") assert not any(json.loads(line)["method"] == "turn/start" for line in capture.read_text().splitlines()) assert store.load_turn(*key) == failed + + +def test_antigravity_steward_recovers_pre_dispatch_resume_failed_after_lost_notice_receipt_without_disk_churn( + steward, language_server, monkeypatch # noqa: F811 +): + from loopx.capabilities.steward_executor import STEWARD_EXECUTOR_SCHEMA, effective_steward_executor_defaults + from loopx.chat_agent import CodexChatAgentError + + store, runtime, provider, transport, binding, capture, _ = steward + defaults = effective_steward_executor_defaults({"schema_version": "loopx_machine_configuration_v0", + "namespaces": {"steward_executor": {"schema_version": STEWARD_EXECUTOR_SCHEMA, "selection_policy": "preferred", + "executor_endpoint": "antigravity", "eligible_endpoints": [], + "executor_model": "gemini-4-argon-high", "executor_reasoning_effort": "high"}}}) + monkeypatch.setattr(runtime, "steward_executor_defaults", lambda: defaults) + transport.bindings.configure(transport_ref="steward-app", project_ref=binding["project_ref"], + executor_endpoint_id="antigravity", context_kind="steward") + + ensure = runtime._ensure_adapter + + def resume_unavailable(*_args, **_kwargs): + raise CodexChatAgentError("The previous Agent conversation could not be restored.", + error_code="resume_failed", gate=None) + + monkeypatch.setattr(runtime, "_ensure_adapter", resume_unavailable) + assert transport.admit("steward-app", provider.event("steward-app", "antigravity-resume-loss", "Analyze repo execution value"))["status"] == "durably_accepted" + row = transport.core.pending()[0] + failed = finish(runtime, row) + assert failed["status"] == "failed" and failed["error_code"] == "resume_failed" + assert failed["started_at"] is None and failed["upstream_turn_id"] is None + + provider.lose_next_reply = True + transport.reconcile() + path = transport.root / f"{row['request_ref']}.json" + record = _read_json(path) + assert record["status"] == "accepted" + assert record["deliveries"]["terminal"]["started"] is True + assert not record["deliveries"]["terminal"].get("attempt") + assert not transport.core.read_request(row["request_ref"]).get("delivery_verified") + + # Second reconcile records provider_delivery_receipt_unknown without resending; + # subsequent ticks before cooldown neither queue it as an unsent body nor rewrite disk. + writes_before = list(provider.writes) + transport.reconcile() + assert provider.writes == writes_before + record = _read_json(path) + assert record["deliveries"]["terminal"]["blocker"] == "provider_delivery_receipt_unknown" + bytes_before = path.read_bytes() + mtime_before = path.stat().st_mtime_ns + assert transport.pending_body_paths() == [] + transport._context_recovery_probe_at.clear() + transport.reconcile() + assert provider.writes == writes_before + assert path.read_bytes() == bytes_before + assert path.stat().st_mtime_ns == mtime_before + + monkeypatch.setattr(runtime, "_ensure_adapter", ensure) + advance_recovery_clock(monkeypatch) + transport._context_recovery_probe_at.clear() + transport.reconcile() + + resumed = transport.core.read_request(row["request_ref"]) + assert resumed["turn_id"] != failed["turn_id"] + assert resumed["context_recovery"]["preparation_attempt"] == 1 + assert resumed["context_recovery"]["failure_notice_delivery_verified"] is False + continued = finish(runtime, resumed) + assert continued["status"] == "completed" + transport.reconcile() + + final_record = _read_json(path) + assert final_record["status"] == "delivered" + assert final_record["deliveries"]["terminal"] == record["deliveries"]["terminal"] + assert any(key.startswith("terminal.recovery.") and key.endswith(".preparation.1") and value.get("verified") is True + for key, value in final_record["deliveries"].items()) + assert language_server["sent_count"] == 1 + assert store.load_turn(row["session_id"], failed["turn_id"]) == failed