Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions benchmark/tests/test_task_entry.py
Original file line number Diff line number Diff line change
Expand Up @@ -1185,10 +1185,17 @@ def test_sforge_planning_entry_stops_retrying_after_the_attempt_limit(planning_e
assert len(calls) == entry.PLANNING_WAKE_ATTEMPTS


@pytest.mark.parametrize("context", ["fresh", "resume"])
@pytest.mark.parametrize("producer", [None, "local_probe"])
@pytest.mark.parametrize("complete", [False, True])
@pytest.mark.parametrize("checkpoint_preflight", [False, True])
@pytest.mark.parametrize(
("context", "producer", "complete", "checkpoint_preflight"),
[
("fresh", None, False, False),
("fresh", "local_probe", True, False),
("fresh", "local_probe", False, True),
("resume", None, True, True),
("resume", "local_probe", False, False),
("resume", None, False, True),
],
)
def test_execute_wake_keeps_host_clocks_out_of_fresh_and_resumed_input(
planning_env, context, producer, complete, checkpoint_preflight):
"""Exercise the real worker/heartbeat CLI and capture its model transport input."""
Expand Down
35 changes: 23 additions & 12 deletions docs/development/testing-and-quality.md
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,11 @@ Keep the separate data/registry paths too: process isolation does not isolate
data. In a Python process that already imported `tempfile`, also scope and
restore its cached `tempfile.tempdir`; the existing
`tests/control_plane/canonical_authority_fixture.py::isolate_sqlite_runtime`
fixture demonstrates both boundaries. Record the serving runtime PID and verify
fixture demonstrates both boundaries, while persistent fixture-worker Node
processes in `tests/control_plane/canonical_authority_fixture.py`, scoped
`NODE_COMPILE_CACHE` setup in `tests/conftest.py`, and in-process CLI test
entry points avoid repeated Node/Python startup overhead without sharing mutable
runtime state across worker namespaces. Record the serving runtime PID and verify
it differs from the live server before dispatching expensive work. Stop only
that isolated runtime after its requests settle, retaining the same temporary
environment for shutdown; never restart a live server as test cleanup.
Expand Down Expand Up @@ -485,7 +489,9 @@ request may still be running.
Effect 后台进程;相同源码指纹可能让源码环境与已安装版本共享进程。演练前创建私有
临时目录,对所有子命令同时设置 `TMPDIR`、`TEMP`、`TMP`;Python 进程若已缓存
`tempfile.tempdir`,须在同一作用域覆盖并恢复。数据/registry 仍须单独隔离,且在
重型操作前核对服务 PID 与活跃服务不同。等请求结束后,只在同一临时环境中停止
重型操作前核对服务 PID 与活跃服务不同。`canonical_authority_fixture.py` 中的持久化
fixture worker Node 进程、`tests/conftest.py` 中的隔离 `NODE_COMPILE_CACHE` 以及进程内
CLI 单测入口仅复用进程启动开销,不跨 worker 共享可变运行时状态。等请求结束后,只在同一临时环境中停止
演练进程。进程隔离不消除整机资源竞争:耗时对照顺序运行,发现竞争则保留失败与
回执、作废受污染耗时并重新采样;超时不能当作服务端已停止,也不能因此重复恢复。

Expand Down Expand Up @@ -2007,6 +2013,10 @@ handling; they never qualify Doubao behavior. Run the real gate with:
```bash
python3 scripts/qualify-doubao-model-behavior-live.py \
--qualification-id <public-safe-run-id>
# Or for DeepSeek (`deepseek-flash` default or `deepseek-v4-pro`) with DEEPSEEK_API_KEY:
python3 scripts/qualify-deepseek-model-behavior-live.py \
--qualification-id <public-safe-run-id> \
--model deepseek-flash
```

The command requires a clean candidate checkout, fails when `ARK_API_KEY` is
Expand Down Expand Up @@ -2055,10 +2065,10 @@ and passes only after the model reads the target named by the selected Todo.
Reading the target before quota, choosing a deferred decoy, describing the
action without calling the tool, or issuing an unallowlisted command fails.

真实 Doubao 调用是低频本地/手动门,不进入普通 CI。`ARK_API_KEY` 只通过进程环境
注入;packet、prompt、原始响应、凭证和对话都不能成为仓库证据,只保留有界 receipt
真实 Doubao 与 DeepSeek 调用是低频本地/手动门,不进入普通 CI。`ARK_API_KEY` 或 `DEEPSEEK_API_KEY` 只通过进程环境
注入(DeepSeek 通过 `scripts/qualify-deepseek-model-behavior-live.py` 并支持 `deepseek-flash` / `deepseek-v4-pro`);packet、prompt、原始响应、凭证和对话都不能成为仓库证据,只保留有界 receipt
与 mismatch code。fake transport 只能验证 adapter 的序列化、脱敏与 fail-closed,不能
作为 Doubao 行为通过证据。真实入口缺少密钥时直接失败,且任一真实调用不通过都会以
作为 Doubao 或 DeepSeek 行为通过证据。真实入口缺少密钥时直接失败,且任一真实调用不通过都会以
非零状态退出。

replan semantic action 另有一条 function-tool 行为资格门,因为 no-tool JSON 决策不能
Expand Down Expand Up @@ -2305,12 +2315,13 @@ deadline is 1,200 seconds; this is a wall-clock ceiling, not a token budget.
### Claude Code and release coverage / Claude Code 与发布覆盖

For focused thin/brief prompt-decision regression, use
`python3 scripts/qualify-host-prompt-release.py --release-live` only during
explicit release qualification. It defaults to no calls; missing credentials
report `skipped`, not a live pass. With securely injected `ARK_API_KEY`, it uses
Doubao evolving for two independent repetitions of quiet-work, notifying-wait,
quiet-wait, required-vision-replan and typed external-wait fallback cases in
each mode. The fallback case composes the complete quota source through the
`python3 scripts/qualify-host-prompt-release.py --release-live` (or
`--release-live --provider deepseek --model deepseek-flash` / `deepseek-v4-pro`
with `DEEPSEEK_API_KEY`) only during explicit release qualification. It defaults
to no calls; missing credentials report `skipped`, not a live pass. With
securely injected `ARK_API_KEY` or `DEEPSEEK_API_KEY`, it exercises two
independent repetitions of quiet-work, notifying-wait, quiet-wait,
required-vision-replan and typed external-wait fallback cases in each mode. The fallback case composes the complete quota source through the
shared TypeScript owner and reuses that checked instance in both modes. Its
wait transition already exists, so the host must advance the selected independent
successor and notify rather than authoring another transition. Expected
Expand All @@ -2324,7 +2335,7 @@ prompts/responses. Model transport failures fail qualification rather than
becoming environment skips.

回退场景使用完整 C1 来源,thin 与 brief 每批共享同一份已检查契约;原四个小型
决策对照仍为合成输入。仅发布前显式执行,普通 CI 不调用模型。检查静默不等于空转、等待不能擅自执行、
决策对照仍为合成输入。`qualify-host-prompt-release.py` 支持 `--provider ark`(默认)与 `--provider deepseek --model deepseek-flash|deepseek-v4-pro`。仅发布前显式执行,普通 CI 不调用模型。检查静默不等于空转、等待不能擅自执行、
vision replan 未关闭时不能提前结束 Goal;这不是完整 Claude/Codex 行为验收的替代。

```bash
Expand Down
2 changes: 1 addition & 1 deletion docs/product/release-readiness.md
Original file line number Diff line number Diff line change
Expand Up @@ -1101,7 +1101,7 @@ hardware-sized single-wave isolated pytest shards (defaulting to
two-thirds of host CPUs capped at 12, with optional `--durations-path`,
`--durations-out`, and fail-closed exact-commit/tree `--resume`) and reduces
the shards into `receipt-pytest.json` with no local paths. Keep
`scripts/qualify-doubao-model-behavior-live.py` (`--max-workers 6` by default)
`scripts/qualify-doubao-model-behavior-live.py` or `scripts/qualify-deepseek-model-behavior-live.py` (`--max-workers 6` by default)
concurrent with network/CPU lanes without mutating the clean candidate tree.

The `exact_release_commit_qualification_manifest_v0` contract requires the
Expand Down
6 changes: 3 additions & 3 deletions docs/product/use-cases/steward/golden-queries.md
Original file line number Diff line number Diff line change
Expand Up @@ -322,11 +322,11 @@ no key in arguments):

```sh
uv run --extra test python examples/evaluations/chat-intake.py --live \
--model deepseek-flash --repeats 2 --output /tmp/chat-intake-results.json
--model deepseek-flash --workers 6 --repeats 2 --output /tmp/chat-intake-results.json
```

It uses the production prompt/parser, 28 public-safe cases, two concurrent calls
and at most 8,192 output tokens per request. Nothing is dispatched or written to
It uses the production prompt/parser, 30 public-safe cases, six concurrent calls
by default (`--workers 6`, range `1–12`) and at most 8,192 output tokens per request. Nothing is dispatched or written to
an active Goal. Skipping `--live` refuses paid calls. CI tests the evaluator and
contracts without credentials; real model results include failures, repeats,
usage and exact prompt/case hashes. To exercise the actual restricted Codex Chat
Expand Down
25 changes: 25 additions & 0 deletions docs/reference/protocols/model-behavior-qualification-v0.md
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,31 @@ is evidence that a named Doubao model followed the projection. Raw prompts,
tool results, provider responses, commands, Todo ids, and local paths are not
retained in its receipt.

## Direct DeepSeek Shadow Actors

`DeepSeekModelBehaviorActor`, `DeepSeekOnboardingModelBehaviorActor`, and
`DeepSeekExecToolClient` run the same fail-closed portfolio and real-tool
qualification suite against the canonical DeepSeek Chat Completions endpoint
(`https://api.deepseek.com/chat/completions`), allowlisting `deepseek-flash`
and `deepseek-v4-pro`. Only `None`, empty strings, `https://api.deepseek.com`,
and `https://api.deepseek.com/v1` via `DEEPSEEK_BASE_URL` resolve to the canonical
endpoint; arbitrary base URLs fail closed with `noncanonical_endpoint`. Live
calls require runtime-injected `DEEPSEEK_API_KEY` (and optional
`LOOPX_DEEPSEEK_BEHAVIOR_MODEL`), disable thinking where deterministic packet
interpretation applies, do not follow redirects, and emit the same compact
Git-bound portfolio receipt (`actor_ref="deepseek:<model>"`):

```bash
python3 scripts/qualify-deepseek-model-behavior-live.py \
--qualification-id <public-safe-run-id> \
--model deepseek-flash
```

All 5 real-tool actors (`selected_todo`, `replan_semantic_action`,
`scoped_gate_successor`, `capability_monitor_repair`, and `terminal_settlement`)
expose `from_deepseek_environment(...)` reusing the shared state machines and
oracles without separate DeepSeek-specific decision rules.

## New-User Onboarding Closed Loop

`onboarding_actual_behavior_qualification_v0` extends the same low-frequency
Expand Down
67 changes: 67 additions & 0 deletions examples/evaluations/chat-intake.public.json
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,58 @@
}
]
}
},
"watch-only-monitor": {
"scope": "manager",
"coverage": {
"complete": true
},
"goals": [
{
"goal_id": "example-product",
"description": "Maintains the example product and reviews its changes.",
"activation_state": "active",
"current_todos": [
{
"text": "Watch PR 42 checks and review readiness",
"status": "open",
"task_class": "continuous_monitor",
"watch_only": true,
"claimed_by": "product-owner",
"recorded_at": "2026-09-01T12:00:00Z"
},
{
"text": "Repair rendering regression and verify PR 42 before merge",
"status": "open",
"task_class": "advancement_task",
"claimed_by": "product-owner",
"recorded_at": "2026-08-31T12:00:00Z"
}
]
}
],
"context_delegation": {
"mode": "context_only",
"targets": [
{
"goal_id": "example-product",
"agent_id": "product-owner"
}
]
},
"fixture_boundary": "Synthetic authoritative read; a watch-only monitor tick is not a completed task or merge.",
"external_observations": [
{
"ref": "monitor-tick-1",
"read_status": "verified_current",
"kind": "continuous_monitor_tick",
"watch_only": true,
"status": "unchanged",
"summary": "Watch-only monitor tick recorded: PR 42 remains OPEN with failing checks; no task completion or merge occurred.",
"observed_at": "2026-09-01T12:05:00Z"
}
],
"evaluation_time": "2026-09-01T12:05:00Z"
}
},
"cases": [
Expand Down Expand Up @@ -668,6 +720,21 @@
"agent_id": "product-owner"
},
"context_ref": "qualified-recipient-without-exact-todo"
},
{
"id": "unconfirmed-pr-merge-status-only",
"request": "PR 42 现在已经合并了吗?当前状态和检查情况如何?",
"expected": "answer",
"context_ref": "open-object"
},
{
"id": "watch-only-monitor-not-completed",
"request": "监控刚记了一笔检查,这件事现在算完成了吗?",
"expected": "answer",
"context_ref": "watch-only-monitor",
"required_evidence_refs": [
"monitor-tick-1"
]
}
]
}
3 changes: 2 additions & 1 deletion examples/evaluations/chat-intake.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def main():
parser.add_argument("--cases", type=Path, default=Path(__file__).with_name("chat-intake.public.json"))
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--repeats", type=int, choices=range(1, 4), default=1)
parser.add_argument("--workers", type=int, choices=range(1, 13), default=6)
args = parser.parse_args()
if not args.live:
parser.error("--live is required; this evaluation sends public fixtures to a paid API")
Expand Down Expand Up @@ -146,7 +147,7 @@ def observe(event, _payload):
print(json.dumps({k: row[k] for k in ("id", "passed", "errors")}), flush=True)
return row

with ThreadPoolExecutor(max_workers=2) as pool:
with ThreadPoolExecutor(max_workers=args.workers) as pool:
rows = list(pool.map(run, cases * args.repeats))
report = {"model": args.model, "provider": args.provider, "cases_sha256": hashlib.sha256(fixture).hexdigest(),
"prompt_sha256": hashlib.sha256(_turn_prompt("").encode()).hexdigest(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
DOUBAO_MODEL_ENV,
DoubaoActorTransport,
_direct_ark_transport,
_direct_deepseek_transport,
)
from .model_tool_behavior import (
DeepSeekExecToolClient,
QuotaToolContext,
is_context_read_command,
DoubaoExecToolClient,
Expand Down Expand Up @@ -452,16 +454,21 @@ class DoubaoCapabilityMonitorRepairToolBehaviorActor:
def __init__(
self,
*,
api_key: str,
api_key: str = "",
model: str = DOUBAO_2_1_PRO_MODEL,
timeout_seconds: float = 90.0,
transport: DoubaoActorTransport = _direct_ark_transport,
client: DoubaoExecToolClient | None = None,
) -> None:
self._client = DoubaoExecToolClient(
api_key=api_key,
model=model,
timeout_seconds=timeout_seconds,
transport=transport,
self._client = (
client
if client is not None
else DoubaoExecToolClient(
api_key=api_key,
model=model,
timeout_seconds=timeout_seconds,
transport=transport,
)
)

@classmethod
Expand All @@ -485,6 +492,24 @@ def from_environment(
transport=transport,
)

@classmethod
def from_deepseek_environment(
cls,
*,
environ: Mapping[str, str] | None = None,
transport: DoubaoActorTransport = _direct_deepseek_transport,
timeout_seconds: float = 90.0,
model: str | None = None,
) -> DoubaoCapabilityMonitorRepairToolBehaviorActor:
return cls(
client=DeepSeekExecToolClient.from_environment(
environ=environ,
transport=transport,
timeout_seconds=timeout_seconds,
model=model,
)
)

def qualify(
self,
*,
Expand Down
Loading
Loading