From 94ba859f4f8dc698c4ef6493d5a20d05fe572d2a Mon Sep 17 00:00:00 2001 From: "lusendong.6789" Date: Sat, 21 Nov 2026 01:11:37 +0800 Subject: [PATCH] refactor(turn): centralize native host failure policy Signed-off-by: lusendong.6789 Co-authored-by: TRAE CLI --- ...hanism-convergence-and-learning-loop-v0.md | 18 +++ ...-convergence-and-learning-loop-v0.zh-CN.md | 13 ++ .../control_plane/effect_runtime_handlers.ts | 2 + .../control_plane/turn_driver/host_failure.py | 64 ++++----- .../control_plane/turn_driver/host_failure.ts | 71 ++++++++-- scripts/external_scheduler_worker.py | 42 +++--- .../test_host_failure_policy_transport.py | 121 ++++++++++++++++++ .../host_failure_policy.test.ts | 68 ++++++++++ tests/test_external_scheduler_worker.py | 8 +- tsconfig.control-plane.json | 1 + 10 files changed, 341 insertions(+), 67 deletions(-) create mode 100644 tests/control_plane/test_host_failure_policy_transport.py create mode 100644 tests/control_plane_ts/host_failure_policy.test.ts diff --git a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md index cecec7e793..96a7fb15df 100644 --- a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md +++ b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.md @@ -315,6 +315,24 @@ one finding and receipt, spends once, and checks effect-free replay plus rejecte new capture. This repairs the existing recovery contract without a new owner or activation; installed Hosts, other backends and benefit retain their own gates. +Native failure allocation and the external worker's Host observation now share +the existing [`typed failure owner`](../../../loopx/control_plane/turn_driver/host_failure.ts). +The Python retry table and worker success/same-session/failure selection are +retired. Native error construction, Mapping refusal order, exact integer IO, +session-ID decoding, polling clocks and the pending-intent witness remain with +their existing adapters. An observed Host success is transport confirmation, +not Turn completion, an ACK or permission to spend. Automatic eligibility stays +narrower than explicit repaired retry; the invocation-only query marker adds no +setting or activation, and omission/false retains the existing explicit query. +The [original worker](../../../tests/test_external_scheduler_worker.py), +[boundary checks](../../../tests/control_plane/test_host_failure_policy_transport.py) +and [File/SQLite managed CLI](../../../tests/test_loopx_turn_managed_step.py) +preserve original-session retry, unknown/foreign/malformed outcomes, refusal +precedence, lost-response recovery and once-only replay. A changed retry ceiling +fails the independent SQLite CLI observation. This source checkpoint does not +qualify installed worker adoption, the remaining C1/C4/C9 cutover, product +composition or accepted outcomes and full cost. + The replay and new-capture assertions execute inside the recovery fixture. An earlier test update put that block in the no-evidence test: the recovery cases skipped those assertions, while ordinary settlement failed on undefined diff --git a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md index 500f003c4b..4055b94156 100644 --- a/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md +++ b/docs/architecture/rfcs/mechanism-convergence-and-learning-loop-v0.zh-CN.md @@ -299,6 +299,19 @@ soft claim/hard lease 及历史反馈仅作上下文:保留独立验证的成 并验证重放无副作用及新捕获拒绝。这修复既有恢复合同,不新增 owner 或 activation; 已安装 Host、其它后端及收益仍沿各自门禁。 +原生故障分配与外部 worker 的 Host 观测现在共用既有 +[`typed 故障 owner`](../../../loopx/control_plane/turn_driver/host_failure.ts)。 +Python 重试表及 worker 的成功/同会话/失败选择已退役。原生错误对象创建、 +Mapping 拒绝顺序、精确整数 IO、会话 ID 解码、轮询时钟及 pending-intent witness +仍归原适配器。Host 成功观测只确认传输,不代表 Turn 完成、ACK 或扣额许可。 +自动重试资格仍窄于显式修复后的重试;仅本次调用的查询标记不增加配置或启用, +省略/false 保留原显式查询语义。[原 worker 检查](../../../tests/test_external_scheduler_worker.py)、 +[边界检查](../../../tests/control_plane/test_host_failure_policy_transport.py)及 +[File/SQLite managed CLI](../../../tests/test_loopx_turn_managed_step.py)保持原会话重试、 +未知/其他会话/损坏结果、拒绝优先级、丢失响应恢复和一次重放结算。改变重试 +上限会使独立 SQLite CLI 观测失败。这是源码 checkpoint,不取得已安装 worker +采用、剩余 C1/C4/C9 切换、产品组合或验收成果及完整成本资格。 + 重放和新捕获断言现于恢复 fixture 内执行。此前测试更新将该块放在无证据测试中: 恢复用例未执行这些断言,普通结算则因 fixture 名称未定义而失败。将既有断言块 移回所属作用域恢复两类检查,不改变 runtime 逻辑。File/SQLite 恢复、无副作用 diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 6fbe7ec4ab..a418097540 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -268,6 +268,8 @@ export function createEffectRuntimeHandlers( lazyHandler(() => import("./turn_driver/turn_journal.ts"), ({interpretTurnJournal}) => (params) => interpretTurnJournal(turnJournalInspectionRequest(params))), ], ["host_failure.retry", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureRetryRequest}) => hostFailureRetryRequest)], + ["host_failure.record", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureRecordRequest}) => hostFailureRecordRequest)], + ["host_failure.observe", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureObservationRequest}) => hostFailureObservationRequest)], ["turn.controller", lazyHandler(() => import("./turn_driver/loop_controller.ts"), ({controllerRequest}) => controllerRequest)], ["turn_journal.write", lazyHandler(() => import("./turn_driver/turn_journal_effects.ts"), ({commitTurnJournal}) => commitTurnJournal)], ["turn_journal.find_settlement", lazyHandler(() => import("./turn_driver/turn_journal_query.ts"), ({findTurnJournalBySettlement}) => findTurnJournalBySettlement)], diff --git a/loopx/control_plane/turn_driver/host_failure.py b/loopx/control_plane/turn_driver/host_failure.py index a625b2bcf0..c990a51828 100644 --- a/loopx/control_plane/turn_driver/host_failure.py +++ b/loopx/control_plane/turn_driver/host_failure.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping +from decimal import Decimal from typing import Any from .session_recovery import require_host_recovery_kind @@ -24,20 +25,6 @@ "unknown", } ) -HOST_RETRY_STRATEGY = "same_configuration" - -# These are scheduling hints. The Turn journal -# remains the authority for the attempt count and the outer controller decides -# when to wake the exact same Turn again. -_RETRY_POLICIES = { - "executor_timeout": (2, 5), - "provider_capacity": (3, 30), - "provider_overloaded": (3, 30), - "rate_limited": (3, 60), - "transport_lost": (3, 10), -} - - class BuiltInHostError(RuntimeError): """A public-safe built-in host failure classification.""" @@ -64,28 +51,23 @@ def require_host_failure_kind(value: str) -> str: def build_host_failure_record(kind: str, *, attempt: int) -> dict[str, Any]: - """Build one content-free failure record from an adapter classification.""" + """Transport a native classification to the existing typed policy owner. + + Native error construction and the Python integer API remain local. Retry + hints grant no launch authority; journals retain their original attempts. + """ + from ..effect_runtime import effect_runtime_result normalized_kind = require_host_failure_kind(kind) if isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1: raise ValueError("host failure attempt must be a positive integer") - record: dict[str, Any] = { - "schema_version": HOST_FAILURE_SCHEMA_VERSION, - "kind": normalized_kind, - "attempt": attempt, - "retryable": normalized_kind in _RETRY_POLICIES, - } - policy = _RETRY_POLICIES.get(normalized_kind) - if policy is not None: - max_attempts, base_backoff_seconds = policy - record["retry"] = { - "strategy": HOST_RETRY_STRATEGY, - "max_attempts": max_attempts, - "backoff_seconds": min( - base_backoff_seconds * (2 ** max(0, attempt - 1)), - 300, - ), - } + record = effect_runtime_result("host_failure.record", { + "kind": normalized_kind, "attempt_text": format(Decimal(int(attempt)), "f"), + }) + if not isinstance(record, dict): + raise RuntimeError("Host failure policy record is unavailable") + # Exact decimal transport avoids JSON rounding of historical Python ints. + record["attempt"] = attempt return record @@ -104,10 +86,9 @@ def normalize_host_failure_record(value: Any) -> dict[str, Any]: retryable = record.get("retryable") if not isinstance(retryable, bool): raise ValueError("host failure record has an invalid retryable marker") - expected_retryable = kind in _RETRY_POLICIES - if retryable is not expected_retryable: - raise ValueError("host failure record retryability does not match its kind") expected = build_host_failure_record(kind, attempt=attempt) + if retryable is not expected["retryable"]: + raise ValueError("host failure record retryability does not match its kind") if retryable: retry = record.get("retry") if not isinstance(retry, Mapping) or dict(retry) != expected["retry"]: @@ -120,11 +101,18 @@ def normalize_host_failure_record(value: Any) -> dict[str, Any]: def host_failure_retry_available(record: Mapping[str, Any]) -> bool: + """Automatic retry remains narrower than an explicit repaired retry.""" + from ..effect_runtime import effect_runtime_result + normalized = normalize_host_failure_record(record) if normalized["retryable"] is not True: return False - retry = dict(normalized["retry"]) - return int(normalized["attempt"]) < int(retry["max_attempts"]) + result = effect_runtime_result("host_failure.retry", { + "failure": normalized, "retry_failed": True, "automatic_only": True, + }) + if not isinstance(result, dict) or type(result.get("allowed")) is not bool: + raise RuntimeError("Host retry decision is unavailable") + return bool(result["allowed"]) def explicit_host_failure_retry_available(record: Mapping[str, Any], *, retry_failed: bool) -> bool: @@ -138,7 +126,7 @@ def explicit_host_failure_retry_available(record: Mapping[str, Any], *, retry_fa }) if not isinstance(result, dict) or type(result.get("allowed")) is not bool: raise RuntimeError("Host retry decision is unavailable") - return result["allowed"] + return bool(result["allowed"]) def record_host_failure( diff --git a/loopx/control_plane/turn_driver/host_failure.ts b/loopx/control_plane/turn_driver/host_failure.ts index fa3c3febff..9247f2a821 100644 --- a/loopx/control_plane/turn_driver/host_failure.ts +++ b/loopx/control_plane/turn_driver/host_failure.ts @@ -13,6 +13,54 @@ const hostRetryPolicies: Readonly> = { rate_limited: [3, 60], transport_lost: [3, 10], }; +/** One retry allocation owner for native records and journal qualification. */ +function hostFailureRecord(kind: string, attempt: bigint): JsonObject { + const policy = hostRetryPolicies[kind]; + return {schema_version: "loopx_turn_host_failure_v0", kind, + attempt: attempt > BigInt(Number.MAX_SAFE_INTEGER) ? String(attempt) : Number(attempt), + retryable: policy !== undefined, + ...(policy === undefined ? {} : {retry: {strategy: "same_configuration", max_attempts: policy[0], + backoff_seconds: Math.min(policy[1] * 2 ** Math.max(0, Number(attempt) - 1), 300)}})}; +} + +export function hostFailureRecordRequest(params: JsonObject): JsonObject { + if (Object.keys(params).some(key => !["kind", "attempt_text"].includes(key))) + throw new EffectRuntimeRequestError("host failure record request contains unsupported fields"); + const kind = params.kind; + if (typeof kind !== "string" || !hostFailureKinds.has(kind)) + throw new EffectRuntimeRequestError("unsupported built-in host failure kind"); + const encoded = params.attempt_text; + if (typeof encoded !== "string" || !/^\d+$/.test(encoded) || BigInt(encoded) < 1n) + throw new EffectRuntimeRequestError("host failure attempt must be a positive integer"); + return hostFailureRecord(kind, BigInt(encoded)); +} + +/** Observe one configured native Host phase. Success is transport confirmation, + * never a Turn completion, scheduler ACK or settlement authorization. + */ +export function hostFailureObservationRequest(params: JsonObject): JsonObject { + const session = typeof params.session_id === "string" ? params.session_id : null; + const prior = typeof params.prior_session_id === "string" ? params.prior_session_id : null; + const sameSession = prior === null || prior === session; + const transported = params.transport_ok === true && params.timed_out === false && params.error_kind === false; + if (transported && params.exit_zero === true && params.ok === true && + params.has_host_failure === false && session && sameSession) { + return {failure: null, session_id: session}; + } + let kind = "unknown"; + if (transported && params.exit_zero === false && params.ok === false && sameSession && + jsonObject(params.native_failure)?.attempt === 1) { + try { + kind = normalizeHostFailure(params.native_failure).kind as string; + } catch (error) { + if (!(error instanceof EffectRuntimeRequestError)) throw error; + // Malformed provider metadata is an unknown phase outcome, not a retry. + } + } + return {failure: hostFailureRecordRequest({kind, attempt_text: params.attempt_text}), + session_id: prior ?? session}; +} + export function normalizeHostFailure(value: unknown, integerAttempt?: string | null): JsonObject { const fail = (message: string): never => { throw new EffectRuntimeRequestError(message); }; const record = jsonObject(value) ?? fail("host failure record must be one object"); @@ -27,15 +75,12 @@ export function normalizeHostFailure(value: unknown, integerAttempt?: string | n : typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 1) fail("host failure record has an invalid attempt"); if (typeof record.retryable !== "boolean") fail("host failure record has an invalid retryable marker"); - const policy = hostRetryPolicies[kind]; - if (record.retryable !== (policy !== undefined)) + const expected: JsonObject = {...hostFailureRecord(kind, exact ?? BigInt(attempt as number)), + // Historical TS callers without exact IO tokens retain their number wire. + attempt: exact !== null && exact > BigInt(Number.MAX_SAFE_INTEGER) ? String(exact) : attempt}; + if (record.retryable !== expected.retryable) fail("host failure record retryability does not match its kind"); - const expected: JsonObject = {schema_version: "loopx_turn_host_failure_v0", kind, - attempt: exact !== null && exact > BigInt(Number.MAX_SAFE_INTEGER) ? String(exact) : attempt, - retryable: policy !== undefined}; - if (policy) { - expected.retry = {strategy: "same_configuration", max_attempts: policy[0], - backoff_seconds: Math.min(policy[1] * 2 ** Math.max(0, Number(attempt) - 1), 300)}; + if (expected.retryable) { if (!isDeepStrictEqual(record.retry, expected.retry)) fail("host failure record has an invalid retry policy"); } else if (Object.hasOwn(record, "retry")) { fail("non-retryable host failure must not declare retry policy"); @@ -69,8 +114,12 @@ export function hostFailureRetryCheck(failure: JsonObject, retryFailed: boolean) export function hostFailureRetryRequest(params: JsonObject): JsonObject { if (typeof params.retry_failed !== "boolean" || Object.keys(params).some( - key => !["failure", "retry_failed"].includes(key))) + key => !["failure", "retry_failed", "automatic_only"].includes(key)) || + (Object.hasOwn(params, "automatic_only") && typeof params.automatic_only !== "boolean")) throw new EffectRuntimeRequestError("Host retry requires an explicit retry intent"); - const check = hostFailureRetryCheck(normalizeHostFailure(params.failure), params.retry_failed); - return {allowed: check === null || check.outcome === "passed", check}; + const failure = normalizeHostFailure(params.failure); + const check = hostFailureRetryCheck(failure, params.retry_failed); + return {allowed: params.automatic_only === true + ? failure.retryable === true && check?.outcome === "passed" + : check === null || check.outcome === "passed", check}; } diff --git a/scripts/external_scheduler_worker.py b/scripts/external_scheduler_worker.py index c97cdf50df..5b2bdc2a28 100644 --- a/scripts/external_scheduler_worker.py +++ b/scripts/external_scheduler_worker.py @@ -33,6 +33,7 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone +from decimal import Decimal from pathlib import Path from typing import Any, Callable, Sequence @@ -628,10 +629,11 @@ def _host_attempt(state: dict[str, Any], scope: str) -> dict[str, Any]: def _host_result(result: CappedProcessResult, attempt: dict[str, Any], now: float) -> dict[str, Any] | None: - """Consume only the configured Host's content-free failure receipt.""" + """Decode native IO; the typed owner qualifies success and same-session failure.""" + from loopx.control_plane.effect_runtime import effect_runtime_result from loopx.control_plane.turn_driver.codex_sessions import _valid_session_id from loopx.control_plane.turn_driver.host_failure import ( - build_host_failure_record, normalize_host_failure_record, + normalize_host_failure_record, ) try: @@ -646,23 +648,29 @@ def _host_result(result: CappedProcessResult, attempt: dict[str, Any], now: floa if session.get("binding_scope") == "agent" and isinstance(candidate, str) else None) if session_id != candidate: session_id = None - same_session = not attempt.get("session_id") or attempt["session_id"] == session_id - if (result.failure_kind is None and result.returncode == 0 and receipt.get("ok") is True - and not receipt.get("timed_out") and not receipt.get("error_kind") - and "host_failure" not in receipt and session_id and same_session): + # Native record decoding retains the existing Mapping API and error + # ordering. No provider prose or unbounded receipt fields cross the RPC. + try: + native_failure = normalize_host_failure_record(receipt.get("host_failure")) + except (ValueError, TypeError): + native_failure = None + count = attempt["attempt"] + observed = effect_runtime_result("host_failure.observe", { + "session_id": session_id, "prior_session_id": attempt.get("session_id") or None, + "ok": receipt.get("ok") if type(receipt.get("ok")) is bool else None, + "timed_out": bool(receipt.get("timed_out")), "error_kind": bool(receipt.get("error_kind")), + "has_host_failure": "host_failure" in receipt, "native_failure": native_failure, + "transport_ok": result.failure_kind is None, "exit_zero": result.returncode == 0, + "attempt_text": format(Decimal(int(count)), "f") if isinstance(count, int) and not isinstance(count, bool) else None, + }) + if not isinstance(observed, dict) or "failure" not in observed: + raise RuntimeError("Native Host observation is unavailable") + failure = observed["failure"] + if failure is None: return None - kind = "unknown" - if (result.failure_kind is None and result.returncode != 0 and receipt.get("ok") is False - and not receipt.get("timed_out") and not receipt.get("error_kind") and same_session): - try: - failure = normalize_host_failure_record(receipt.get("host_failure")) - if failure["attempt"] == 1: - kind = failure["kind"] - except (ValueError, TypeError): - pass - failure = build_host_failure_record(kind, attempt=attempt["attempt"]) + failure["attempt"] = count return {**attempt, "status": "failed", "host_failure": failure, - "session_id": attempt.get("session_id") or session_id, + "session_id": observed["session_id"], "not_before": now + failure.get("retry", {}).get("backoff_seconds", 0)} diff --git a/tests/control_plane/test_host_failure_policy_transport.py b/tests/control_plane/test_host_failure_policy_transport.py new file mode 100644 index 0000000000..3271924722 --- /dev/null +++ b/tests/control_plane/test_host_failure_policy_transport.py @@ -0,0 +1,121 @@ +"""Native failure records keep their historical API and refusal ordering.""" +from copy import deepcopy +import json +from types import MappingProxyType + +import pytest + +from loopx.control_plane.turn_driver.host_failure import ( + BuiltInHostError, build_host_failure_record, host_failure_retry_available, + normalize_host_failure_record, +) + + +@pytest.mark.parametrize("attempt", [2**53 + 1, 10**40, 10**5000], + ids=["json-safe-range-exceeded", "wide-integer", "decimal-codec-limit"]) +def test_non_retryable_attempts_keep_exact_python_integer_readback(attempt): + failure = build_host_failure_record("auth_failed", attempt=attempt) + assert failure == {"schema_version": "loopx_turn_host_failure_v0", "kind": "auth_failed", + "attempt": attempt, "retryable": False} + assert type(failure["attempt"]) is int + assert normalize_host_failure_record(failure) == failure + assert host_failure_retry_available(failure) is False + + +@pytest.mark.parametrize(("changes", "message"), [ + ({"schema_version": "future", "kind": "bad", "attempt": False}, "unsupported schema"), + ({"kind": "bad", "attempt": False}, "unsupported built-in host failure kind"), + ({"attempt": False, "retryable": "true"}, "invalid attempt"), + ({"retryable": "true", "retry": {}}, "invalid retryable marker"), + ({"retryable": False, "retry": {}}, "retryability does not match its kind"), + ({"retry": {}, "extra": "ignored only after validation"}, "invalid retry policy"), + ({"extra": "private provider prose"}, "contains unsupported fields"), +]) +def test_record_refusal_keeps_its_original_precedence_and_input(changes, message): + record = {**build_host_failure_record("provider_capacity", attempt=1), **changes} + before = deepcopy(record) + with pytest.raises(ValueError, match=message): + normalize_host_failure_record(record) + assert record == before + + +def test_mapping_retry_and_equal_float_metadata_keep_the_original_record(): + raw = {"schema_version": "loopx_turn_host_failure_v0", "kind": "executor_timeout", + "attempt": 1, "retryable": True, "retry": MappingProxyType({ + "strategy": "same_configuration", "max_attempts": 2.0, "backoff_seconds": 5.0})} + normalized = normalize_host_failure_record(MappingProxyType(raw)) + assert normalized == {**raw, "retry": {"strategy": "same_configuration", + "max_attempts": 2, "backoff_seconds": 5}} + assert type(normalized["retry"]["max_attempts"]) is int + assert normalized["retry"] is not raw["retry"] + + +def test_native_error_construction_does_not_need_the_effect_runtime(monkeypatch): + from loopx.control_plane import effect_runtime + + def unavailable(*args, **kwargs): + pytest.fail("Native protocol error creation must remain independent of the policy RPC") + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", unavailable) + failure = BuiltInHostError("provider_unavailable", failure_kind="provider_capacity") + assert str(failure) == "provider_unavailable" and failure.failure_kind == "provider_capacity" + with pytest.raises(ValueError, match="unsupported built-in host failure kind"): + BuiltInHostError("provider_unavailable", failure_kind="unregistered") + + +@pytest.mark.parametrize("kind", ["provider_capacity", "rate_limited", "executor_timeout"]) +def test_valid_failure_kind_is_checked_before_bad_attempt(kind): + with pytest.raises(ValueError, match="positive integer"): + build_host_failure_record(kind, attempt=True) + with pytest.raises(ValueError, match="unsupported built-in host failure kind"): + build_host_failure_record("unregistered", attempt=True) + + +@pytest.mark.parametrize(("change", "kind", "session", "transport"), [ + ({}, "provider_capacity", "session-a", None), + ({"timed_out": [], "error_kind": {}}, "provider_capacity", "session-a", None), + ({"ok": True}, "unknown", "session-a", None), + ({"session": {"binding_scope": "agent", "session_id": "session-b"}}, "unknown", "session-a", None), + ({"session": {}}, "unknown", "session-a", None), + ({"host_failure": {"schema_version": "future"}}, "unknown", "session-a", None), + ({}, "unknown", "session-a", "timeout"), +]) +def test_native_observation_keeps_scope_truth_encoding_and_private_receipt_fields( + change, kind, session, transport, monkeypatch, +): + from tests.test_external_scheduler_worker import worker + from loopx.control_plane import effect_runtime + + receipt = {"ok": False, "timed_out": False, "error_kind": None, + "session": {"binding_scope": "agent", "session_id": "session-a"}, + "host_failure": {"schema_version": "loopx_turn_host_failure_v0", "kind": "provider_capacity", + "attempt": 1, "retryable": True, "retry": {"strategy": "same_configuration", + "max_attempts": 3, "backoff_seconds": 30}}, + "private_diagnostic": "private-provider-sentinel", **change} + attempt = {"attempt": 2, "session_id": "session-a", "scope": "original", "attempt_id": "original-attempt"} + before = deepcopy((receipt, attempt)) + observed = [] + invoke = effect_runtime.effect_runtime_result + + def record(method, params, **kwargs): + observed.append((method, params)) + return invoke(method, params, **kwargs) + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", record) + result = worker._host_result(worker.CappedProcessResult(1, json.dumps(receipt).encode(), transport), attempt, 100) + assert result == {**attempt, "status": "failed", "session_id": session, + "host_failure": {"schema_version": "loopx_turn_host_failure_v0", "kind": kind, + "attempt": 2, "retryable": kind == "provider_capacity", + **({"retry": {"strategy": "same_configuration", "max_attempts": 3, "backoff_seconds": 60}} + if kind == "provider_capacity" else {})}, + "not_before": 160 if kind == "provider_capacity" else 100} + assert "private-provider-sentinel" not in json.dumps(observed) + assert (receipt, attempt) == before + + +def test_confirmed_native_success_does_not_require_later_failure_facts(): + from tests.test_external_scheduler_worker import worker + + receipt = {"ok": True, "session": {"binding_scope": "agent", "session_id": "session-a"}} + attempt = {"attempt": False, "session_id": "session-a"} + assert worker._host_result(worker.CappedProcessResult(0, json.dumps(receipt).encode(), None), attempt, 0) is None diff --git a/tests/control_plane_ts/host_failure_policy.test.ts b/tests/control_plane_ts/host_failure_policy.test.ts new file mode 100644 index 0000000000..b8ea65011e --- /dev/null +++ b/tests/control_plane_ts/host_failure_policy.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {hostFailureObservationRequest, hostFailureRecordRequest, hostFailureRetryRequest, normalizeHostFailure} + from "../../loopx/control_plane/turn_driver/host_failure.ts"; + +test("native records and journal qualification reuse the same allocation", () => { + for (const [kind, ceiling, delay] of [["executor_timeout", 2, 5], ["provider_capacity", 3, 30], + ["provider_overloaded", 3, 30], ["rate_limited", 3, 60], ["transport_lost", 3, 10]] as const) { + for (const attempt of [1, 2, 4, 10]) { + const record = hostFailureRecordRequest({kind, attempt_text: String(attempt)}); + assert.deepEqual(record, {schema_version: "loopx_turn_host_failure_v0", kind, attempt, + retryable: true, retry: {strategy: "same_configuration", max_attempts: ceiling, + backoff_seconds: Math.min(delay * 2 ** (attempt - 1), 300)}}); + assert.deepEqual(normalizeHostFailure(record), record); + } + } +}); + +test("automatic availability preserves explicit repair and rejects exhausted policies", () => { + for (const kind of ["auth_failed", "unknown", "output_budget_exhausted", "provider_capacity"]) { + for (const attempt of [1, 3]) for (const retry_failed of [true, false]) { + const failure = hostFailureRecordRequest({kind, attempt_text: String(attempt)}); + const request = {failure, retry_failed}; + const legacy = hostFailureRetryRequest(request); + assert.deepEqual(hostFailureRetryRequest({...request, automatic_only: false}), legacy); + assert.equal(hostFailureRetryRequest({...request, automatic_only: true}).allowed, + kind === "provider_capacity" && attempt === 1 && retry_failed); + assert.equal(legacy.allowed, retry_failed && kind !== "output_budget_exhausted" + && (kind !== "provider_capacity" || attempt === 1)); + } + } +}); + +test("exact native integers do not change the historical no-token journal wire", () => { + const exact = "9007199254740993"; + const record = hostFailureRecordRequest({kind: "auth_failed", attempt_text: exact}); + assert.equal(record.attempt, exact); + const decoded = {...record, attempt: Number(exact)}; + assert.equal(normalizeHostFailure(decoded).attempt, Number(exact)); + assert.equal(normalizeHostFailure(decoded, exact).attempt, exact); +}); + +test("RPC shape rejects malformed attempts and caller-controlled retry fields", () => { + for (const attempt_text of [true, 1, null, "0", "-1", "1.0", "bad"]) { + assert.throws(() => hostFailureRecordRequest({kind: "rate_limited", attempt_text}), /positive integer/); + } + assert.throws(() => hostFailureRecordRequest({kind: "rate_limited", attempt_text: "1", max_attempts: 99}), + /unsupported fields/); + const failure = hostFailureRecordRequest({kind: "rate_limited", attempt_text: "1"}); + for (const automatic_only of [null, "true", 1]) { + assert.throws(() => hostFailureRetryRequest({failure, retry_failed: true, automatic_only}), /explicit retry intent/); + } +}); + +test("observed native phases cannot turn private or forged receipts into a retry", () => { + const request = {session_id: "session", prior_session_id: "session", transport_ok: true, exit_zero: false, + ok: false, timed_out: false, error_kind: false, has_host_failure: true, attempt_text: "2", + native_failure: hostFailureRecordRequest({kind: "provider_capacity", attempt_text: "1"})}; + const observed = hostFailureObservationRequest(request); + assert.deepEqual(observed.failure, hostFailureRecordRequest({kind: "provider_capacity", attempt_text: "2"})); + for (const native_failure of [{attempt: 1, kind: "provider_capacity"}, + {...request.native_failure, retryable: false}, {...request.native_failure, attempt: 2}]) { + assert.equal((hostFailureObservationRequest({...request, native_failure}).failure as {kind: string}).kind, "unknown"); + } + assert.equal((hostFailureObservationRequest({...request, session_id: "replacement"}).failure as {kind: string}).kind, "unknown"); + const success = {...request, ok: true, exit_zero: true, has_host_failure: false, attempt_text: null}; + assert.deepEqual(hostFailureObservationRequest(success), {failure: null, session_id: "session"}); +}); diff --git a/tests/test_external_scheduler_worker.py b/tests/test_external_scheduler_worker.py index 50545b7844..9c0051c26c 100644 --- a/tests/test_external_scheduler_worker.py +++ b/tests/test_external_scheduler_worker.py @@ -725,7 +725,13 @@ def test_core_deadline_during_retry_backoff_does_not_launch_again(tmp_path, monk @pytest.mark.parametrize("stdout,returncode", [(b'{"ok":true}', 0), (b'{"ok":false}', 1), - (b'{"ok":true,"session":', 0), (b'{}', 0)]) + (b'{"ok":true,"session":', 0), (b'{}', 0), + pytest.param(b'{"ok":true,"session":{"binding_scope":"agent","session_id":"session-b"}}', + 0, id="foreign-success"), + pytest.param(json.dumps({"ok": False, "session": {"binding_scope": "agent", "session_id": "session-b"}, + "host_failure": {"schema_version": "loopx_turn_host_failure_v0", "kind": "provider_capacity", + "attempt": 1, "retryable": True, "retry": {"strategy": "same_configuration", + "max_attempts": 3, "backoff_seconds": 30}}}).encode(), 1, id="foreign-capacity")]) def test_unconfirmed_receipt_never_clears_original_session(tmp_path, monkeypatch, stdout, returncode): from loopx.extensions.process_runtime import CappedProcessResult args, count = _guarded_host(tmp_path, kind="provider_capacity") diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index f9057b4a47..eabe5c9634 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -14,6 +14,7 @@ }, "include": [ "loopx/control_plane/**/*.ts", + "tests/control_plane_ts/host_failure_policy.test.ts", "loopx/zcode_goal_mode/**/*.ts", "tests/control_plane_ts/background_services.test.ts", "tests/control_plane_ts/explore_research.test.ts",