Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,24 @@ one finding and receipt, spends once, and checks effect-free replay plus rejecte
new capture. This repairs the existing recovery contract without a new owner or
activation; installed Hosts, other backends and benefit retain their own gates.

Native failure allocation and the external worker's Host observation now share
the existing [`typed failure owner`](../../../loopx/control_plane/turn_driver/host_failure.ts).
The Python retry table and worker success/same-session/failure selection are
retired. Native error construction, Mapping refusal order, exact integer IO,
session-ID decoding, polling clocks and the pending-intent witness remain with
their existing adapters. An observed Host success is transport confirmation,
not Turn completion, an ACK or permission to spend. Automatic eligibility stays
narrower than explicit repaired retry; the invocation-only query marker adds no
setting or activation, and omission/false retains the existing explicit query.
The [original worker](../../../tests/test_external_scheduler_worker.py),
[boundary checks](../../../tests/control_plane/test_host_failure_policy_transport.py)
and [File/SQLite managed CLI](../../../tests/test_loopx_turn_managed_step.py)
preserve original-session retry, unknown/foreign/malformed outcomes, refusal
precedence, lost-response recovery and once-only replay. A changed retry ceiling
fails the independent SQLite CLI observation. This source checkpoint does not
qualify installed worker adoption, the remaining C1/C4/C9 cutover, product
composition or accepted outcomes and full cost.

The replay and new-capture assertions execute inside the recovery fixture.
An earlier test update put that block in the no-evidence test: the recovery
cases skipped those assertions, while ordinary settlement failed on undefined
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,19 @@ soft claim/hard lease 及历史反馈仅作上下文:保留独立验证的成
并验证重放无副作用及新捕获拒绝。这修复既有恢复合同,不新增 owner 或 activation;
已安装 Host、其它后端及收益仍沿各自门禁。

原生故障分配与外部 worker 的 Host 观测现在共用既有
[`typed 故障 owner`](../../../loopx/control_plane/turn_driver/host_failure.ts)。
Python 重试表及 worker 的成功/同会话/失败选择已退役。原生错误对象创建、
Mapping 拒绝顺序、精确整数 IO、会话 ID 解码、轮询时钟及 pending-intent witness
仍归原适配器。Host 成功观测只确认传输,不代表 Turn 完成、ACK 或扣额许可。
自动重试资格仍窄于显式修复后的重试;仅本次调用的查询标记不增加配置或启用,
省略/false 保留原显式查询语义。[原 worker 检查](../../../tests/test_external_scheduler_worker.py)、
[边界检查](../../../tests/control_plane/test_host_failure_policy_transport.py)及
[File/SQLite managed CLI](../../../tests/test_loopx_turn_managed_step.py)保持原会话重试、
未知/其他会话/损坏结果、拒绝优先级、丢失响应恢复和一次重放结算。改变重试
上限会使独立 SQLite CLI 观测失败。这是源码 checkpoint,不取得已安装 worker
采用、剩余 C1/C4/C9 切换、产品组合或验收成果及完整成本资格。

重放和新捕获断言现于恢复 fixture 内执行。此前测试更新将该块放在无证据测试中:
恢复用例未执行这些断言,普通结算则因 fixture 名称未定义而失败。将既有断言块
移回所属作用域恢复两类检查,不改变 runtime 逻辑。File/SQLite 恢复、无副作用
Expand Down
2 changes: 2 additions & 0 deletions loopx/control_plane/effect_runtime_handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,8 @@ export function createEffectRuntimeHandlers(
lazyHandler(() => import("./turn_driver/turn_journal.ts"), ({interpretTurnJournal}) => (params) => interpretTurnJournal(turnJournalInspectionRequest(params))),
],
["host_failure.retry", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureRetryRequest}) => hostFailureRetryRequest)],
["host_failure.record", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureRecordRequest}) => hostFailureRecordRequest)],
["host_failure.observe", lazyHandler(() => import("./turn_driver/host_failure.ts"), ({hostFailureObservationRequest}) => hostFailureObservationRequest)],
["turn.controller", lazyHandler(() => import("./turn_driver/loop_controller.ts"), ({controllerRequest}) => controllerRequest)],
["turn_journal.write", lazyHandler(() => import("./turn_driver/turn_journal_effects.ts"), ({commitTurnJournal}) => commitTurnJournal)],
["turn_journal.find_settlement", lazyHandler(() => import("./turn_driver/turn_journal_query.ts"), ({findTurnJournalBySettlement}) => findTurnJournalBySettlement)],
Expand Down
64 changes: 26 additions & 38 deletions loopx/control_plane/turn_driver/host_failure.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

from collections.abc import Mapping
from decimal import Decimal
from typing import Any

from .session_recovery import require_host_recovery_kind
Expand All @@ -24,20 +25,6 @@
"unknown",
}
)
HOST_RETRY_STRATEGY = "same_configuration"

# These are scheduling hints. The Turn journal
# remains the authority for the attempt count and the outer controller decides
# when to wake the exact same Turn again.
_RETRY_POLICIES = {
"executor_timeout": (2, 5),
"provider_capacity": (3, 30),
"provider_overloaded": (3, 30),
"rate_limited": (3, 60),
"transport_lost": (3, 10),
}


class BuiltInHostError(RuntimeError):
"""A public-safe built-in host failure classification."""

Expand All @@ -64,28 +51,23 @@ def require_host_failure_kind(value: str) -> str:


def build_host_failure_record(kind: str, *, attempt: int) -> dict[str, Any]:
"""Build one content-free failure record from an adapter classification."""
"""Transport a native classification to the existing typed policy owner.

Native error construction and the Python integer API remain local. Retry
hints grant no launch authority; journals retain their original attempts.
"""
from ..effect_runtime import effect_runtime_result

normalized_kind = require_host_failure_kind(kind)
if isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1:
raise ValueError("host failure attempt must be a positive integer")
record: dict[str, Any] = {
"schema_version": HOST_FAILURE_SCHEMA_VERSION,
"kind": normalized_kind,
"attempt": attempt,
"retryable": normalized_kind in _RETRY_POLICIES,
}
policy = _RETRY_POLICIES.get(normalized_kind)
if policy is not None:
max_attempts, base_backoff_seconds = policy
record["retry"] = {
"strategy": HOST_RETRY_STRATEGY,
"max_attempts": max_attempts,
"backoff_seconds": min(
base_backoff_seconds * (2 ** max(0, attempt - 1)),
300,
),
}
record = effect_runtime_result("host_failure.record", {
"kind": normalized_kind, "attempt_text": format(Decimal(int(attempt)), "f"),
})
if not isinstance(record, dict):
raise RuntimeError("Host failure policy record is unavailable")
# Exact decimal transport avoids JSON rounding of historical Python ints.
record["attempt"] = attempt
return record


Expand All @@ -104,10 +86,9 @@ def normalize_host_failure_record(value: Any) -> dict[str, Any]:
retryable = record.get("retryable")
if not isinstance(retryable, bool):
raise ValueError("host failure record has an invalid retryable marker")
expected_retryable = kind in _RETRY_POLICIES
if retryable is not expected_retryable:
raise ValueError("host failure record retryability does not match its kind")
expected = build_host_failure_record(kind, attempt=attempt)
if retryable is not expected["retryable"]:
raise ValueError("host failure record retryability does not match its kind")
if retryable:
retry = record.get("retry")
if not isinstance(retry, Mapping) or dict(retry) != expected["retry"]:
Expand All @@ -120,11 +101,18 @@ def normalize_host_failure_record(value: Any) -> dict[str, Any]:


def host_failure_retry_available(record: Mapping[str, Any]) -> bool:
"""Automatic retry remains narrower than an explicit repaired retry."""
from ..effect_runtime import effect_runtime_result

normalized = normalize_host_failure_record(record)
if normalized["retryable"] is not True:
return False
retry = dict(normalized["retry"])
return int(normalized["attempt"]) < int(retry["max_attempts"])
result = effect_runtime_result("host_failure.retry", {
"failure": normalized, "retry_failed": True, "automatic_only": True,
})
if not isinstance(result, dict) or type(result.get("allowed")) is not bool:
raise RuntimeError("Host retry decision is unavailable")
return bool(result["allowed"])


def explicit_host_failure_retry_available(record: Mapping[str, Any], *, retry_failed: bool) -> bool:
Expand All @@ -138,7 +126,7 @@ def explicit_host_failure_retry_available(record: Mapping[str, Any], *, retry_fa
})
if not isinstance(result, dict) or type(result.get("allowed")) is not bool:
raise RuntimeError("Host retry decision is unavailable")
return result["allowed"]
return bool(result["allowed"])


def record_host_failure(
Expand Down
71 changes: 60 additions & 11 deletions loopx/control_plane/turn_driver/host_failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,54 @@ const hostRetryPolicies: Readonly<Record<string, readonly [number, number]>> = {
rate_limited: [3, 60], transport_lost: [3, 10],
};

/** One retry allocation owner for native records and journal qualification. */
function hostFailureRecord(kind: string, attempt: bigint): JsonObject {
const policy = hostRetryPolicies[kind];
return {schema_version: "loopx_turn_host_failure_v0", kind,
attempt: attempt > BigInt(Number.MAX_SAFE_INTEGER) ? String(attempt) : Number(attempt),
retryable: policy !== undefined,
...(policy === undefined ? {} : {retry: {strategy: "same_configuration", max_attempts: policy[0],
backoff_seconds: Math.min(policy[1] * 2 ** Math.max(0, Number(attempt) - 1), 300)}})};
}

export function hostFailureRecordRequest(params: JsonObject): JsonObject {
if (Object.keys(params).some(key => !["kind", "attempt_text"].includes(key)))
throw new EffectRuntimeRequestError("host failure record request contains unsupported fields");
const kind = params.kind;
if (typeof kind !== "string" || !hostFailureKinds.has(kind))
throw new EffectRuntimeRequestError("unsupported built-in host failure kind");
const encoded = params.attempt_text;
if (typeof encoded !== "string" || !/^\d+$/.test(encoded) || BigInt(encoded) < 1n)
throw new EffectRuntimeRequestError("host failure attempt must be a positive integer");
return hostFailureRecord(kind, BigInt(encoded));
}

/** Observe one configured native Host phase. Success is transport confirmation,
* never a Turn completion, scheduler ACK or settlement authorization.
*/
export function hostFailureObservationRequest(params: JsonObject): JsonObject {
const session = typeof params.session_id === "string" ? params.session_id : null;
const prior = typeof params.prior_session_id === "string" ? params.prior_session_id : null;
const sameSession = prior === null || prior === session;
const transported = params.transport_ok === true && params.timed_out === false && params.error_kind === false;
if (transported && params.exit_zero === true && params.ok === true &&
params.has_host_failure === false && session && sameSession) {
return {failure: null, session_id: session};
}
let kind = "unknown";
if (transported && params.exit_zero === false && params.ok === false && sameSession &&
jsonObject(params.native_failure)?.attempt === 1) {
try {
kind = normalizeHostFailure(params.native_failure).kind as string;
} catch (error) {
if (!(error instanceof EffectRuntimeRequestError)) throw error;
// Malformed provider metadata is an unknown phase outcome, not a retry.
}
}
return {failure: hostFailureRecordRequest({kind, attempt_text: params.attempt_text}),
session_id: prior ?? session};
}

export function normalizeHostFailure(value: unknown, integerAttempt?: string | null): JsonObject {
const fail = (message: string): never => { throw new EffectRuntimeRequestError(message); };
const record = jsonObject(value) ?? fail("host failure record must be one object");
Expand All @@ -27,15 +75,12 @@ export function normalizeHostFailure(value: unknown, integerAttempt?: string | n
: typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 1)
fail("host failure record has an invalid attempt");
if (typeof record.retryable !== "boolean") fail("host failure record has an invalid retryable marker");
const policy = hostRetryPolicies[kind];
if (record.retryable !== (policy !== undefined))
const expected: JsonObject = {...hostFailureRecord(kind, exact ?? BigInt(attempt as number)),
// Historical TS callers without exact IO tokens retain their number wire.
attempt: exact !== null && exact > BigInt(Number.MAX_SAFE_INTEGER) ? String(exact) : attempt};
if (record.retryable !== expected.retryable)
fail("host failure record retryability does not match its kind");
const expected: JsonObject = {schema_version: "loopx_turn_host_failure_v0", kind,
attempt: exact !== null && exact > BigInt(Number.MAX_SAFE_INTEGER) ? String(exact) : attempt,
retryable: policy !== undefined};
if (policy) {
expected.retry = {strategy: "same_configuration", max_attempts: policy[0],
backoff_seconds: Math.min(policy[1] * 2 ** Math.max(0, Number(attempt) - 1), 300)};
if (expected.retryable) {
if (!isDeepStrictEqual(record.retry, expected.retry)) fail("host failure record has an invalid retry policy");
} else if (Object.hasOwn(record, "retry")) {
fail("non-retryable host failure must not declare retry policy");
Expand Down Expand Up @@ -69,8 +114,12 @@ export function hostFailureRetryCheck(failure: JsonObject, retryFailed: boolean)

export function hostFailureRetryRequest(params: JsonObject): JsonObject {
if (typeof params.retry_failed !== "boolean" || Object.keys(params).some(
key => !["failure", "retry_failed"].includes(key)))
key => !["failure", "retry_failed", "automatic_only"].includes(key)) ||
(Object.hasOwn(params, "automatic_only") && typeof params.automatic_only !== "boolean"))
throw new EffectRuntimeRequestError("Host retry requires an explicit retry intent");
const check = hostFailureRetryCheck(normalizeHostFailure(params.failure), params.retry_failed);
return {allowed: check === null || check.outcome === "passed", check};
const failure = normalizeHostFailure(params.failure);
const check = hostFailureRetryCheck(failure, params.retry_failed);
return {allowed: params.automatic_only === true
? failure.retryable === true && check?.outcome === "passed"
: check === null || check.outcome === "passed", check};
}
42 changes: 25 additions & 17 deletions scripts/external_scheduler_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from decimal import Decimal
from pathlib import Path
from typing import Any, Callable, Sequence

Expand Down Expand Up @@ -628,10 +629,11 @@ def _host_attempt(state: dict[str, Any], scope: str) -> dict[str, Any]:


def _host_result(result: CappedProcessResult, attempt: dict[str, Any], now: float) -> dict[str, Any] | None:
"""Consume only the configured Host's content-free failure receipt."""
"""Decode native IO; the typed owner qualifies success and same-session failure."""
from loopx.control_plane.effect_runtime import effect_runtime_result
from loopx.control_plane.turn_driver.codex_sessions import _valid_session_id
from loopx.control_plane.turn_driver.host_failure import (
build_host_failure_record, normalize_host_failure_record,
normalize_host_failure_record,
)

try:
Expand All @@ -646,23 +648,29 @@ def _host_result(result: CappedProcessResult, attempt: dict[str, Any], now: floa
if session.get("binding_scope") == "agent" and isinstance(candidate, str) else None)
if session_id != candidate:
session_id = None
same_session = not attempt.get("session_id") or attempt["session_id"] == session_id
if (result.failure_kind is None and result.returncode == 0 and receipt.get("ok") is True
and not receipt.get("timed_out") and not receipt.get("error_kind")
and "host_failure" not in receipt and session_id and same_session):
# Native record decoding retains the existing Mapping API and error
# ordering. No provider prose or unbounded receipt fields cross the RPC.
try:
native_failure = normalize_host_failure_record(receipt.get("host_failure"))
except (ValueError, TypeError):
native_failure = None
count = attempt["attempt"]
observed = effect_runtime_result("host_failure.observe", {
"session_id": session_id, "prior_session_id": attempt.get("session_id") or None,
"ok": receipt.get("ok") if type(receipt.get("ok")) is bool else None,
"timed_out": bool(receipt.get("timed_out")), "error_kind": bool(receipt.get("error_kind")),
"has_host_failure": "host_failure" in receipt, "native_failure": native_failure,
"transport_ok": result.failure_kind is None, "exit_zero": result.returncode == 0,
"attempt_text": format(Decimal(int(count)), "f") if isinstance(count, int) and not isinstance(count, bool) else None,
})
if not isinstance(observed, dict) or "failure" not in observed:
raise RuntimeError("Native Host observation is unavailable")
failure = observed["failure"]
if failure is None:
return None
kind = "unknown"
if (result.failure_kind is None and result.returncode != 0 and receipt.get("ok") is False
and not receipt.get("timed_out") and not receipt.get("error_kind") and same_session):
try:
failure = normalize_host_failure_record(receipt.get("host_failure"))
if failure["attempt"] == 1:
kind = failure["kind"]
except (ValueError, TypeError):
pass
failure = build_host_failure_record(kind, attempt=attempt["attempt"])
failure["attempt"] = count
return {**attempt, "status": "failed", "host_failure": failure,
"session_id": attempt.get("session_id") or session_id,
"session_id": observed["session_id"],
"not_before": now + failure.get("retry", {}).get("backoff_seconds", 0)}


Expand Down
Loading
Loading