diff --git a/benchmark/tests/test_task_entry.py b/benchmark/tests/test_task_entry.py index 7db2a9ba09..715161791b 100644 --- a/benchmark/tests/test_task_entry.py +++ b/benchmark/tests/test_task_entry.py @@ -1185,10 +1185,17 @@ def test_sforge_planning_entry_stops_retrying_after_the_attempt_limit(planning_e assert len(calls) == entry.PLANNING_WAKE_ATTEMPTS -@pytest.mark.parametrize("context", ["fresh", "resume"]) -@pytest.mark.parametrize("producer", [None, "local_probe"]) -@pytest.mark.parametrize("complete", [False, True]) -@pytest.mark.parametrize("checkpoint_preflight", [False, True]) +@pytest.mark.parametrize( + ("context", "producer", "complete", "checkpoint_preflight"), + [ + ("fresh", None, False, False), + ("fresh", "local_probe", True, False), + ("fresh", "local_probe", False, True), + ("resume", None, True, True), + ("resume", "local_probe", False, False), + ("resume", None, False, True), + ], +) def test_execute_wake_keeps_host_clocks_out_of_fresh_and_resumed_input( planning_env, context, producer, complete, checkpoint_preflight): """Exercise the real worker/heartbeat CLI and capture its model transport input.""" diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 7de7a3ee7e..45cac2e5af 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -445,7 +445,11 @@ Keep the separate data/registry paths too: process isolation does not isolate data. In a Python process that already imported `tempfile`, also scope and restore its cached `tempfile.tempdir`; the existing `tests/control_plane/canonical_authority_fixture.py::isolate_sqlite_runtime` -fixture demonstrates both boundaries. Record the serving runtime PID and verify +fixture demonstrates both boundaries, while persistent fixture-worker Node +processes in `tests/control_plane/canonical_authority_fixture.py`, scoped +`NODE_COMPILE_CACHE` setup in `tests/conftest.py`, and in-process CLI test +entry points avoid repeated Node/Python startup overhead without sharing mutable +runtime state across worker namespaces. Record the serving runtime PID and verify it differs from the live server before dispatching expensive work. Stop only that isolated runtime after its requests settle, retaining the same temporary environment for shutdown; never restart a live server as test cleanup. @@ -485,7 +489,9 @@ request may still be running. Effect 后台进程;相同源码指纹可能让源码环境与已安装版本共享进程。演练前创建私有 临时目录,对所有子命令同时设置 `TMPDIR`、`TEMP`、`TMP`;Python 进程若已缓存 `tempfile.tempdir`,须在同一作用域覆盖并恢复。数据/registry 仍须单独隔离,且在 -重型操作前核对服务 PID 与活跃服务不同。等请求结束后,只在同一临时环境中停止 +重型操作前核对服务 PID 与活跃服务不同。`canonical_authority_fixture.py` 中的持久化 +fixture worker Node 进程、`tests/conftest.py` 中的隔离 `NODE_COMPILE_CACHE` 以及进程内 +CLI 单测入口仅复用进程启动开销,不跨 worker 共享可变运行时状态。等请求结束后,只在同一临时环境中停止 演练进程。进程隔离不消除整机资源竞争:耗时对照顺序运行,发现竞争则保留失败与 回执、作废受污染耗时并重新采样;超时不能当作服务端已停止,也不能因此重复恢复。 @@ -2007,6 +2013,10 @@ handling; they never qualify Doubao behavior. Run the real gate with: ```bash python3 scripts/qualify-doubao-model-behavior-live.py \ --qualification-id +# Or for DeepSeek (`deepseek-flash` default or `deepseek-v4-pro`) with DEEPSEEK_API_KEY: +python3 scripts/qualify-deepseek-model-behavior-live.py \ + --qualification-id \ + --model deepseek-flash ``` The command requires a clean candidate checkout, fails when `ARK_API_KEY` is @@ -2055,10 +2065,10 @@ and passes only after the model reads the target named by the selected Todo. Reading the target before quota, choosing a deferred decoy, describing the action without calling the tool, or issuing an unallowlisted command fails. -真实 Doubao 调用是低频本地/手动门,不进入普通 CI。`ARK_API_KEY` 只通过进程环境 -注入;packet、prompt、原始响应、凭证和对话都不能成为仓库证据,只保留有界 receipt +真实 Doubao 与 DeepSeek 调用是低频本地/手动门,不进入普通 CI。`ARK_API_KEY` 或 `DEEPSEEK_API_KEY` 只通过进程环境 +注入(DeepSeek 通过 `scripts/qualify-deepseek-model-behavior-live.py` 并支持 `deepseek-flash` / `deepseek-v4-pro`);packet、prompt、原始响应、凭证和对话都不能成为仓库证据,只保留有界 receipt 与 mismatch code。fake transport 只能验证 adapter 的序列化、脱敏与 fail-closed,不能 -作为 Doubao 行为通过证据。真实入口缺少密钥时直接失败,且任一真实调用不通过都会以 +作为 Doubao 或 DeepSeek 行为通过证据。真实入口缺少密钥时直接失败,且任一真实调用不通过都会以 非零状态退出。 replan semantic action 另有一条 function-tool 行为资格门,因为 no-tool JSON 决策不能 @@ -2305,12 +2315,13 @@ deadline is 1,200 seconds; this is a wall-clock ceiling, not a token budget. ### Claude Code and release coverage / Claude Code 与发布覆盖 For focused thin/brief prompt-decision regression, use -`python3 scripts/qualify-host-prompt-release.py --release-live` only during -explicit release qualification. It defaults to no calls; missing credentials -report `skipped`, not a live pass. With securely injected `ARK_API_KEY`, it uses -Doubao evolving for two independent repetitions of quiet-work, notifying-wait, -quiet-wait, required-vision-replan and typed external-wait fallback cases in -each mode. The fallback case composes the complete quota source through the +`python3 scripts/qualify-host-prompt-release.py --release-live` (or +`--release-live --provider deepseek --model deepseek-flash` / `deepseek-v4-pro` +with `DEEPSEEK_API_KEY`) only during explicit release qualification. It defaults +to no calls; missing credentials report `skipped`, not a live pass. With +securely injected `ARK_API_KEY` or `DEEPSEEK_API_KEY`, it exercises two +independent repetitions of quiet-work, notifying-wait, quiet-wait, +required-vision-replan and typed external-wait fallback cases in each mode. The fallback case composes the complete quota source through the shared TypeScript owner and reuses that checked instance in both modes. Its wait transition already exists, so the host must advance the selected independent successor and notify rather than authoring another transition. Expected @@ -2324,7 +2335,7 @@ prompts/responses. Model transport failures fail qualification rather than becoming environment skips. 回退场景使用完整 C1 来源,thin 与 brief 每批共享同一份已检查契约;原四个小型 -决策对照仍为合成输入。仅发布前显式执行,普通 CI 不调用模型。检查静默不等于空转、等待不能擅自执行、 +决策对照仍为合成输入。`qualify-host-prompt-release.py` 支持 `--provider ark`(默认)与 `--provider deepseek --model deepseek-flash|deepseek-v4-pro`。仅发布前显式执行,普通 CI 不调用模型。检查静默不等于空转、等待不能擅自执行、 vision replan 未关闭时不能提前结束 Goal;这不是完整 Claude/Codex 行为验收的替代。 ```bash diff --git a/docs/product/release-readiness.md b/docs/product/release-readiness.md index 6306f86c51..458e7dbbc0 100644 --- a/docs/product/release-readiness.md +++ b/docs/product/release-readiness.md @@ -1101,7 +1101,7 @@ hardware-sized single-wave isolated pytest shards (defaulting to two-thirds of host CPUs capped at 12, with optional `--durations-path`, `--durations-out`, and fail-closed exact-commit/tree `--resume`) and reduces the shards into `receipt-pytest.json` with no local paths. Keep -`scripts/qualify-doubao-model-behavior-live.py` (`--max-workers 6` by default) +`scripts/qualify-doubao-model-behavior-live.py` or `scripts/qualify-deepseek-model-behavior-live.py` (`--max-workers 6` by default) concurrent with network/CPU lanes without mutating the clean candidate tree. The `exact_release_commit_qualification_manifest_v0` contract requires the diff --git a/docs/product/use-cases/steward/golden-queries.md b/docs/product/use-cases/steward/golden-queries.md index 53d051ad01..01e634ba85 100644 --- a/docs/product/use-cases/steward/golden-queries.md +++ b/docs/product/use-cases/steward/golden-queries.md @@ -322,11 +322,11 @@ no key in arguments): ```sh uv run --extra test python examples/evaluations/chat-intake.py --live \ - --model deepseek-flash --repeats 2 --output /tmp/chat-intake-results.json + --model deepseek-flash --workers 6 --repeats 2 --output /tmp/chat-intake-results.json ``` -It uses the production prompt/parser, 28 public-safe cases, two concurrent calls -and at most 8,192 output tokens per request. Nothing is dispatched or written to +It uses the production prompt/parser, 30 public-safe cases, six concurrent calls +by default (`--workers 6`, range `1–12`) and at most 8,192 output tokens per request. Nothing is dispatched or written to an active Goal. Skipping `--live` refuses paid calls. CI tests the evaluator and contracts without credentials; real model results include failures, repeats, usage and exact prompt/case hashes. To exercise the actual restricted Codex Chat diff --git a/docs/reference/protocols/model-behavior-qualification-v0.md b/docs/reference/protocols/model-behavior-qualification-v0.md index a09e907c5a..4866091792 100644 --- a/docs/reference/protocols/model-behavior-qualification-v0.md +++ b/docs/reference/protocols/model-behavior-qualification-v0.md @@ -253,6 +253,31 @@ is evidence that a named Doubao model followed the projection. Raw prompts, tool results, provider responses, commands, Todo ids, and local paths are not retained in its receipt. +## Direct DeepSeek Shadow Actors + +`DeepSeekModelBehaviorActor`, `DeepSeekOnboardingModelBehaviorActor`, and +`DeepSeekExecToolClient` run the same fail-closed portfolio and real-tool +qualification suite against the canonical DeepSeek Chat Completions endpoint +(`https://api.deepseek.com/chat/completions`), allowlisting `deepseek-flash` +and `deepseek-v4-pro`. Only `None`, empty strings, `https://api.deepseek.com`, +and `https://api.deepseek.com/v1` via `DEEPSEEK_BASE_URL` resolve to the canonical +endpoint; arbitrary base URLs fail closed with `noncanonical_endpoint`. Live +calls require runtime-injected `DEEPSEEK_API_KEY` (and optional +`LOOPX_DEEPSEEK_BEHAVIOR_MODEL`), disable thinking where deterministic packet +interpretation applies, do not follow redirects, and emit the same compact +Git-bound portfolio receipt (`actor_ref="deepseek:"`): + +```bash +python3 scripts/qualify-deepseek-model-behavior-live.py \ + --qualification-id \ + --model deepseek-flash +``` + +All 5 real-tool actors (`selected_todo`, `replan_semantic_action`, +`scoped_gate_successor`, `capability_monitor_repair`, and `terminal_settlement`) +expose `from_deepseek_environment(...)` reusing the shared state machines and +oracles without separate DeepSeek-specific decision rules. + ## New-User Onboarding Closed Loop `onboarding_actual_behavior_qualification_v0` extends the same low-frequency diff --git a/examples/evaluations/chat-intake.public.json b/examples/evaluations/chat-intake.public.json index ab346a7957..6d8d6980e6 100644 --- a/examples/evaluations/chat-intake.public.json +++ b/examples/evaluations/chat-intake.public.json @@ -456,6 +456,58 @@ } ] } + }, + "watch-only-monitor": { + "scope": "manager", + "coverage": { + "complete": true + }, + "goals": [ + { + "goal_id": "example-product", + "description": "Maintains the example product and reviews its changes.", + "activation_state": "active", + "current_todos": [ + { + "text": "Watch PR 42 checks and review readiness", + "status": "open", + "task_class": "continuous_monitor", + "watch_only": true, + "claimed_by": "product-owner", + "recorded_at": "2026-09-01T12:00:00Z" + }, + { + "text": "Repair rendering regression and verify PR 42 before merge", + "status": "open", + "task_class": "advancement_task", + "claimed_by": "product-owner", + "recorded_at": "2026-08-31T12:00:00Z" + } + ] + } + ], + "context_delegation": { + "mode": "context_only", + "targets": [ + { + "goal_id": "example-product", + "agent_id": "product-owner" + } + ] + }, + "fixture_boundary": "Synthetic authoritative read; a watch-only monitor tick is not a completed task or merge.", + "external_observations": [ + { + "ref": "monitor-tick-1", + "read_status": "verified_current", + "kind": "continuous_monitor_tick", + "watch_only": true, + "status": "unchanged", + "summary": "Watch-only monitor tick recorded: PR 42 remains OPEN with failing checks; no task completion or merge occurred.", + "observed_at": "2026-09-01T12:05:00Z" + } + ], + "evaluation_time": "2026-09-01T12:05:00Z" } }, "cases": [ @@ -668,6 +720,21 @@ "agent_id": "product-owner" }, "context_ref": "qualified-recipient-without-exact-todo" + }, + { + "id": "unconfirmed-pr-merge-status-only", + "request": "PR 42 现在已经合并了吗?当前状态和检查情况如何?", + "expected": "answer", + "context_ref": "open-object" + }, + { + "id": "watch-only-monitor-not-completed", + "request": "监控刚记了一笔检查,这件事现在算完成了吗?", + "expected": "answer", + "context_ref": "watch-only-monitor", + "required_evidence_refs": [ + "monitor-tick-1" + ] } ] } diff --git a/examples/evaluations/chat-intake.py b/examples/evaluations/chat-intake.py index aa058d65d1..8776567fbd 100644 --- a/examples/evaluations/chat-intake.py +++ b/examples/evaluations/chat-intake.py @@ -68,6 +68,7 @@ def main(): parser.add_argument("--cases", type=Path, default=Path(__file__).with_name("chat-intake.public.json")) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--repeats", type=int, choices=range(1, 4), default=1) + parser.add_argument("--workers", type=int, choices=range(1, 13), default=6) args = parser.parse_args() if not args.live: parser.error("--live is required; this evaluation sends public fixtures to a paid API") @@ -146,7 +147,7 @@ def observe(event, _payload): print(json.dumps({k: row[k] for k in ("id", "passed", "errors")}), flush=True) return row - with ThreadPoolExecutor(max_workers=2) as pool: + with ThreadPoolExecutor(max_workers=args.workers) as pool: rows = list(pool.map(run, cases * args.repeats)) report = {"model": args.model, "provider": args.provider, "cases_sha256": hashlib.sha256(fixture).hexdigest(), "prompt_sha256": hashlib.sha256(_turn_prompt("").encode()).hexdigest(), diff --git a/loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py b/loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py index 3530bac530..505df3a54f 100644 --- a/loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py +++ b/loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py @@ -17,8 +17,10 @@ DOUBAO_MODEL_ENV, DoubaoActorTransport, _direct_ark_transport, + _direct_deepseek_transport, ) from .model_tool_behavior import ( + DeepSeekExecToolClient, QuotaToolContext, is_context_read_command, DoubaoExecToolClient, @@ -452,16 +454,21 @@ class DoubaoCapabilityMonitorRepairToolBehaviorActor: def __init__( self, *, - api_key: str, + api_key: str = "", model: str = DOUBAO_2_1_PRO_MODEL, timeout_seconds: float = 90.0, transport: DoubaoActorTransport = _direct_ark_transport, + client: DoubaoExecToolClient | None = None, ) -> None: - self._client = DoubaoExecToolClient( - api_key=api_key, - model=model, - timeout_seconds=timeout_seconds, - transport=transport, + self._client = ( + client + if client is not None + else DoubaoExecToolClient( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) ) @classmethod @@ -485,6 +492,24 @@ def from_environment( transport=transport, ) + @classmethod + def from_deepseek_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DoubaoCapabilityMonitorRepairToolBehaviorActor: + return cls( + client=DeepSeekExecToolClient.from_environment( + environ=environ, + transport=transport, + timeout_seconds=timeout_seconds, + model=model, + ) + ) + def qualify( self, *, diff --git a/loopx/control_plane/testing/doubao_model_behavior_actor.py b/loopx/control_plane/testing/doubao_model_behavior_actor.py index 2d0fe51c91..a442d2f47f 100644 --- a/loopx/control_plane/testing/doubao_model_behavior_actor.py +++ b/loopx/control_plane/testing/doubao_model_behavior_actor.py @@ -3,7 +3,7 @@ import json import os import socket -from collections.abc import Mapping, Sequence +from collections.abc import Mapping, Sequence, Set as AbstractSet from typing import Any, Protocol from urllib.error import HTTPError, URLError from urllib.request import HTTPRedirectHandler, Request, build_opener @@ -29,6 +29,14 @@ ) ARK_API_KEY_ENV = "ARK_API_KEY" DOUBAO_MODEL_ENV = "LOOPX_MODEL_BEHAVIOR_MODEL" + +DEEPSEEK_FLASH_MODEL = "deepseek-flash" +DEEPSEEK_V4_PRO_MODEL = "deepseek-v4-pro" +DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT = "https://api.deepseek.com/chat/completions" +DEEPSEEK_API_KEY_ENV = "DEEPSEEK_API_KEY" +DEEPSEEK_BASE_URL_ENV = "DEEPSEEK_BASE_URL" +DEEPSEEK_MODEL_ENV = "LOOPX_DEEPSEEK_BEHAVIOR_MODEL" + MODEL_BEHAVIOR_PROVIDER_INPUT_SCHEMA_VERSION = "model_behavior_provider_input_v1" ALLOWED_MODEL_BEHAVIOR_MODELS = frozenset( @@ -38,6 +46,12 @@ DOUBAO_SEED_EVOLVING_MODEL, } ) +ALLOWED_DEEPSEEK_BEHAVIOR_MODELS = frozenset( + { + DEEPSEEK_FLASH_MODEL, + DEEPSEEK_V4_PRO_MODEL, + } +) _MAX_PROVIDER_RESPONSE_BYTES = 1_048_576 _MAX_DECISION_TOKENS = 4096 @@ -72,16 +86,35 @@ def redirect_request( return None -def _direct_ark_transport( +def resolve_deepseek_chat_completions_endpoint(base_url: str | None = None) -> str: + """Resolve the canonical DeepSeek completions endpoint or fail closed.""" + + raw = "" if base_url is None else str(base_url).strip() + if not raw: + return DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + normalized = raw.rstrip("/") + if normalized in {"https://api.deepseek.com", "https://api.deepseek.com/v1"}: + return DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + raise DoubaoActorTransportError( + "DeepSeek actor endpoint is not the canonical DeepSeek endpoint", + error_code="noncanonical_endpoint", + ) + + +def _direct_chat_completions_transport( *, endpoint: str, + expected_endpoint: str, + endpoint_error_message: str, + provider_label: str, + api_key_env: str, headers: Mapping[str, str], body: bytes, timeout_seconds: float, ) -> Mapping[str, Any]: - if endpoint != DOUBAO_CHAT_COMPLETIONS_ENDPOINT: + if endpoint != expected_endpoint: raise DoubaoActorTransportError( - "Doubao actor endpoint is not the canonical Ark endpoint", + endpoint_error_message, error_code="noncanonical_endpoint", ) request = Request(endpoint, data=body, headers=dict(headers), method="POST") @@ -92,48 +125,86 @@ def _direct_ark_transport( except HTTPError as exc: if exc.code == 401: raise DoubaoActorTransportError( - "Doubao actor authentication failed; refresh ARK_API_KEY before retrying", + f"{provider_label} actor authentication failed; refresh {api_key_env} before retrying", error_code="provider_authentication_failed", ) from None raise DoubaoActorTransportError( - f"Doubao actor request failed with HTTP status {exc.code}", + f"{provider_label} actor request failed with HTTP status {exc.code}", error_code="provider_http_error", ) from None except TimeoutError: raise DoubaoActorTransportError( - "Doubao actor provider timed out", + f"{provider_label} actor provider timed out", error_code="provider_timeout", ) from None except URLError as exc: if isinstance(exc.reason, (TimeoutError, socket.timeout)): raise DoubaoActorTransportError( - "Doubao actor provider timed out", + f"{provider_label} actor provider timed out", error_code="provider_timeout", ) from None raise DoubaoActorTransportError( - "Doubao actor provider transport failed", + f"{provider_label} actor provider transport failed", error_code="provider_transport_failed", ) from None if len(payload) > _MAX_PROVIDER_RESPONSE_BYTES: raise DoubaoActorTransportError( - "Doubao actor response exceeded the size limit", + f"{provider_label} actor response exceeded the size limit", error_code="provider_response_too_large", ) try: decoded = json.loads(payload) except (UnicodeDecodeError, json.JSONDecodeError): raise DoubaoActorTransportError( - "Doubao actor returned invalid JSON", + f"{provider_label} actor returned invalid JSON", error_code="provider_invalid_json", ) from None if not isinstance(decoded, Mapping): raise DoubaoActorTransportError( - "Doubao actor provider response must be an object", + f"{provider_label} actor provider response must be an object", error_code="provider_invalid_shape", ) return decoded +def _direct_ark_transport( + *, + endpoint: str, + headers: Mapping[str, str], + body: bytes, + timeout_seconds: float, +) -> Mapping[str, Any]: + return _direct_chat_completions_transport( + endpoint=endpoint, + expected_endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + endpoint_error_message="Doubao actor endpoint is not the canonical Ark endpoint", + provider_label="Doubao", + api_key_env=ARK_API_KEY_ENV, + headers=headers, + body=body, + timeout_seconds=timeout_seconds, + ) + + +def _direct_deepseek_transport( + *, + endpoint: str, + headers: Mapping[str, str], + body: bytes, + timeout_seconds: float, +) -> Mapping[str, Any]: + return _direct_chat_completions_transport( + endpoint=endpoint, + expected_endpoint=DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + endpoint_error_message="DeepSeek actor endpoint is not the canonical DeepSeek endpoint", + provider_label="DeepSeek", + api_key_env=DEEPSEEK_API_KEY_ENV, + headers=headers, + body=body, + timeout_seconds=timeout_seconds, + ) + + def _provider_input(request: Mapping[str, Any]) -> dict[str, Any]: """Keep qualification metadata out of the model-visible decision input.""" @@ -382,25 +453,49 @@ def _decision_instruction( return instruction -def _provider_decision(response: Mapping[str, Any]) -> Mapping[str, Any]: +def _strip_json_markdown_fence(content: str) -> str: + stripped = content.strip() + if not stripped.startswith("```"): + return stripped + lines = stripped.splitlines() + if len(lines) >= 2 and lines[-1].strip() == "```": + tag = lines[0][3:].strip().lower() + if tag in {"", "json"}: + return "\n".join(lines[1:-1]).strip() + if len(lines) == 1 and len(stripped) >= 6 and stripped.endswith("```"): + inner = stripped[3:-3].strip() + if inner.lower().startswith("json") and ( + len(inner) == 4 or inner[4].isspace() + ): + return inner[4:].strip() + return inner + return stripped + + +def _provider_decision( + response: Mapping[str, Any], + *, + provider_label: str = "Doubao", +) -> Mapping[str, Any]: choices = response.get("choices") if not isinstance(choices, list) or len(choices) != 1: - raise RuntimeError("Doubao actor response must contain exactly one choice") + raise RuntimeError(f"{provider_label} actor response must contain exactly one choice") choice = choices[0] if not isinstance(choice, Mapping): - raise RuntimeError("Doubao actor choice must be an object") + raise RuntimeError(f"{provider_label} actor choice must be an object") message = choice.get("message") if not isinstance(message, Mapping): - raise RuntimeError("Doubao actor choice is missing its message") + raise RuntimeError(f"{provider_label} actor choice is missing its message") content = message.get("content") if not isinstance(content, str) or not content.strip(): - raise RuntimeError("Doubao actor message content must be non-empty JSON") + raise RuntimeError(f"{provider_label} actor message content must be non-empty JSON") + normalized_content = _strip_json_markdown_fence(content) try: - decision = json.loads(content) + decision = json.loads(normalized_content) except json.JSONDecodeError: - raise RuntimeError("Doubao actor message content is not valid JSON") from None + raise RuntimeError(f"{provider_label} actor message content is not valid JSON") from None if not isinstance(decision, Mapping): - raise RuntimeError("Doubao actor decision must be an object") + raise RuntimeError(f"{provider_label} actor decision must be an object") return decision @@ -412,6 +507,8 @@ def _invoke_provider_decision( transport: DoubaoActorTransport, system_instruction: str, provider_input: Mapping[str, Any], + endpoint: str = DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + provider_label: str = "Doubao", ) -> Mapping[str, Any]: body = { "model": model, @@ -435,7 +532,7 @@ def _invoke_provider_decision( } try: response = transport( - endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + endpoint=endpoint, headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -452,10 +549,10 @@ def _invoke_provider_decision( raise except Exception: raise DoubaoActorTransportError( - "Doubao actor provider transport failed", + f"{provider_label} actor provider transport failed", error_code="provider_transport_failed", ) from None - return _provider_decision(response) + return _provider_decision(response, provider_label=provider_label) def _onboarding_decision_instruction(phase: str) -> str: @@ -547,49 +644,60 @@ def _onboarding_provider_input(request: Mapping[str, Any]) -> dict[str, Any]: } -class DoubaoModelBehaviorActor(ModelBehaviorActor): - """Direct Ark actor for low-frequency, no-tool behavior qualification.""" +def _validate_actor_settings( + *, + api_key: str, + model: str, + allowed_models: AbstractSet[str], + timeout_seconds: float, + provider_label: str, + endpoint: str, + expected_endpoint: str, +) -> None: + if not api_key.strip(): + raise RuntimeError(f"{provider_label} actor requires a runtime-injected API key") + if model not in allowed_models: + raise ValueError(f"{provider_label} actor model must be explicitly allowlisted") + if timeout_seconds <= 0 or timeout_seconds > 300: + raise ValueError(f"{provider_label} actor timeout must be between 0 and 300 seconds") + if endpoint != expected_endpoint: + raise DoubaoActorTransportError( + f"{provider_label} actor endpoint is not the canonical endpoint", + error_code="noncanonical_endpoint", + ) + + +class _DirectChatModelBehaviorActor(ModelBehaviorActor): + """Shared direct-provider actor for no-tool behavior qualification.""" + + _provider_label = "Doubao" + _provider_prefix = "ark" + _expected_endpoint = DOUBAO_CHAT_COMPLETIONS_ENDPOINT def __init__( self, *, api_key: str, - model: str = DOUBAO_2_1_PRO_MODEL, - timeout_seconds: float = 90.0, - transport: DoubaoActorTransport = _direct_ark_transport, + model: str, + allowed_models: AbstractSet[str], + timeout_seconds: float, + transport: DoubaoActorTransport, + endpoint: str, ) -> None: - if not api_key.strip(): - raise RuntimeError("Doubao actor requires a runtime-injected API key") - if model not in ALLOWED_MODEL_BEHAVIOR_MODELS: - raise ValueError("Doubao actor model must be explicitly allowlisted") - if timeout_seconds <= 0 or timeout_seconds > 300: - raise ValueError("Doubao actor timeout must be between 0 and 300 seconds") - self._api_key = api_key - self._model = model - self._timeout_seconds = timeout_seconds - self._transport = transport - - @classmethod - def from_environment( - cls, - *, - environ: Mapping[str, str] | None = None, - transport: DoubaoActorTransport = _direct_ark_transport, - timeout_seconds: float = 90.0, - ) -> DoubaoModelBehaviorActor: - values = os.environ if environ is None else environ - api_key = values.get(ARK_API_KEY_ENV, "") - if not api_key.strip(): - raise RuntimeError( - "ARK_API_KEY is not injected; live Doubao qualification is unavailable" - ) - model = values.get(DOUBAO_MODEL_ENV, DOUBAO_2_1_PRO_MODEL) - return cls( + _validate_actor_settings( api_key=api_key, model=model, + allowed_models=allowed_models, timeout_seconds=timeout_seconds, - transport=transport, + provider_label=self._provider_label, + endpoint=endpoint, + expected_endpoint=self._expected_endpoint, ) + self._api_key = api_key + self._model = model + self._timeout_seconds = timeout_seconds + self._transport = transport + self._endpoint = endpoint def __call__(self, request: Mapping[str, Any]) -> Mapping[str, Any]: canonical_request = normalize_model_behavior_actor_request(request) @@ -612,36 +720,140 @@ def __call__(self, request: Mapping[str, Any]) -> Mapping[str, Any]: packet_schema_version=str(canonical_request["packet_schema_version"]), ), provider_input=_provider_input(canonical_request), + endpoint=self._endpoint, + provider_label=self._provider_label, ) return { "schema_version": MODEL_BEHAVIOR_ACTOR_RESULT_SCHEMA_VERSION, - "actor_ref": f"ark:{self._model}", + "actor_ref": f"{self._provider_prefix}:{self._model}", "decision": dict(decision), "tool_calls": [], } -class DoubaoOnboardingModelBehaviorActor: - """Direct Ark actor for low-frequency onboarding closed-loop qualification.""" +class _DirectChatOnboardingModelBehaviorActor: + """Shared direct-provider actor for onboarding closed-loop qualification.""" + + _provider_label = "Doubao" + _provider_prefix = "ark" + _expected_endpoint = DOUBAO_CHAT_COMPLETIONS_ENDPOINT def __init__( self, *, api_key: str, - model: str = DOUBAO_2_1_PRO_MODEL, - timeout_seconds: float = 90.0, - transport: DoubaoActorTransport = _direct_ark_transport, + model: str, + allowed_models: AbstractSet[str], + timeout_seconds: float, + transport: DoubaoActorTransport, + endpoint: str, ) -> None: - if not api_key.strip(): - raise RuntimeError("Doubao actor requires a runtime-injected API key") - if model not in ALLOWED_MODEL_BEHAVIOR_MODELS: - raise ValueError("Doubao actor model must be explicitly allowlisted") - if timeout_seconds <= 0 or timeout_seconds > 300: - raise ValueError("Doubao actor timeout must be between 0 and 300 seconds") + _validate_actor_settings( + api_key=api_key, + model=model, + allowed_models=allowed_models, + timeout_seconds=timeout_seconds, + provider_label=self._provider_label, + endpoint=endpoint, + expected_endpoint=self._expected_endpoint, + ) self._api_key = api_key self._model = model self._timeout_seconds = timeout_seconds self._transport = transport + self._endpoint = endpoint + + def __call__(self, request: Mapping[str, Any]) -> Mapping[str, Any]: + canonical_request = normalize_onboarding_model_behavior_actor_request(request) + phase = str(canonical_request["phase"]) + decision = _invoke_provider_decision( + api_key=self._api_key, + model=self._model, + timeout_seconds=self._timeout_seconds, + transport=self._transport, + system_instruction=_onboarding_decision_instruction(phase), + provider_input=_onboarding_provider_input(canonical_request), + endpoint=self._endpoint, + provider_label=self._provider_label, + ) + return { + "schema_version": ONBOARDING_MODEL_BEHAVIOR_RESULT_SCHEMA_VERSION, + "actor_ref": f"{self._provider_prefix}:{self._model}", + "decision": dict(decision), + "tool_calls": [], + } + + +class DoubaoModelBehaviorActor(_DirectChatModelBehaviorActor): + """Direct Ark actor for low-frequency, no-tool behavior qualification.""" + + _provider_label = "Doubao" + _provider_prefix = "ark" + _expected_endpoint = DOUBAO_CHAT_COMPLETIONS_ENDPOINT + + def __init__( + self, + *, + api_key: str, + model: str = DOUBAO_2_1_PRO_MODEL, + timeout_seconds: float = 90.0, + transport: DoubaoActorTransport = _direct_ark_transport, + ) -> None: + super().__init__( + api_key=api_key, + model=model, + allowed_models=ALLOWED_MODEL_BEHAVIOR_MODELS, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + ) + + @classmethod + def from_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_ark_transport, + timeout_seconds: float = 90.0, + ) -> DoubaoModelBehaviorActor: + values = os.environ if environ is None else environ + api_key = values.get(ARK_API_KEY_ENV, "") + if not api_key.strip(): + raise RuntimeError( + "ARK_API_KEY is not injected; live Doubao qualification is unavailable" + ) + model = values.get(DOUBAO_MODEL_ENV, DOUBAO_2_1_PRO_MODEL) + return cls( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) + + +class DoubaoOnboardingModelBehaviorActor(_DirectChatOnboardingModelBehaviorActor): + """Direct Ark actor for low-frequency onboarding closed-loop qualification.""" + + _provider_label = "Doubao" + _provider_prefix = "ark" + _expected_endpoint = DOUBAO_CHAT_COMPLETIONS_ENDPOINT + + def __init__( + self, + *, + api_key: str, + model: str = DOUBAO_2_1_PRO_MODEL, + timeout_seconds: float = 90.0, + transport: DoubaoActorTransport = _direct_ark_transport, + ) -> None: + super().__init__( + api_key=api_key, + model=model, + allowed_models=ALLOWED_MODEL_BEHAVIOR_MODELS, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + ) @classmethod def from_environment( @@ -665,20 +877,114 @@ def from_environment( transport=transport, ) - def __call__(self, request: Mapping[str, Any]) -> Mapping[str, Any]: - canonical_request = normalize_onboarding_model_behavior_actor_request(request) - phase = str(canonical_request["phase"]) - decision = _invoke_provider_decision( - api_key=self._api_key, - model=self._model, - timeout_seconds=self._timeout_seconds, - transport=self._transport, - system_instruction=_onboarding_decision_instruction(phase), - provider_input=_onboarding_provider_input(canonical_request), + +def _deepseek_env_config( + *, + environ: Mapping[str, str] | None, + model: str | None, +) -> tuple[str, str, str]: + values = os.environ if environ is None else environ + api_key = values.get(DEEPSEEK_API_KEY_ENV, "") + if not api_key.strip(): + raise RuntimeError( + "DEEPSEEK_API_KEY is not injected; live DeepSeek qualification is unavailable" + ) + endpoint = resolve_deepseek_chat_completions_endpoint(values.get(DEEPSEEK_BASE_URL_ENV)) + resolved_model = ( + values.get(DEEPSEEK_MODEL_ENV, DEEPSEEK_FLASH_MODEL) if model is None else model + ) + return api_key, resolved_model, endpoint + + +class DeepSeekModelBehaviorActor(_DirectChatModelBehaviorActor): + """Direct DeepSeek actor for low-frequency, no-tool behavior qualification.""" + + _provider_label = "DeepSeek" + _provider_prefix = "deepseek" + _expected_endpoint = DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + + def __init__( + self, + *, + api_key: str, + model: str = DEEPSEEK_FLASH_MODEL, + timeout_seconds: float = 90.0, + transport: DoubaoActorTransport = _direct_deepseek_transport, + endpoint: str = DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + ) -> None: + super().__init__( + api_key=api_key, + model=model, + allowed_models=ALLOWED_DEEPSEEK_BEHAVIOR_MODELS, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, + ) + + @classmethod + def from_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DeepSeekModelBehaviorActor: + api_key, resolved_model, endpoint = _deepseek_env_config( + environ=environ, + model=model, + ) + return cls( + api_key=api_key, + model=resolved_model, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, + ) + + +class DeepSeekOnboardingModelBehaviorActor(_DirectChatOnboardingModelBehaviorActor): + """Direct DeepSeek actor for low-frequency onboarding closed-loop qualification.""" + + _provider_label = "DeepSeek" + _provider_prefix = "deepseek" + _expected_endpoint = DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + + def __init__( + self, + *, + api_key: str, + model: str = DEEPSEEK_FLASH_MODEL, + timeout_seconds: float = 90.0, + transport: DoubaoActorTransport = _direct_deepseek_transport, + endpoint: str = DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + ) -> None: + super().__init__( + api_key=api_key, + model=model, + allowed_models=ALLOWED_DEEPSEEK_BEHAVIOR_MODELS, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, + ) + + @classmethod + def from_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DeepSeekOnboardingModelBehaviorActor: + api_key, resolved_model, endpoint = _deepseek_env_config( + environ=environ, + model=model, + ) + return cls( + api_key=api_key, + model=resolved_model, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, ) - return { - "schema_version": ONBOARDING_MODEL_BEHAVIOR_RESULT_SCHEMA_VERSION, - "actor_ref": f"ark:{self._model}", - "decision": dict(decision), - "tool_calls": [], - } diff --git a/loopx/control_plane/testing/host_prompt_behavior.py b/loopx/control_plane/testing/host_prompt_behavior.py index 64af2785fa..1bb41a8976 100644 --- a/loopx/control_plane/testing/host_prompt_behavior.py +++ b/loopx/control_plane/testing/host_prompt_behavior.py @@ -9,6 +9,7 @@ from ..quota.effective_action import EffectiveAction from ..quota.turn_envelope import compose_turn_decision from .action_portfolio_scenarios import external_wait_fallback_scenario_source +from .doubao_model_behavior_actor import _strip_json_markdown_fence from .model_tool_behavior import DoubaoExecToolClient @@ -58,8 +59,11 @@ def cases() -> list[dict]: def probe_messages(mode: str, packet: dict) -> list[dict]: if mode not in {"thin", "brief"}: raise ValueError("unsupported host prompt mode") + # Pass an explicit synthetic registry_path so host_prompt_behavior probes do + # not depend on resolving the machine's global ~/.loopx vs ~/.codex/loopx roots. prompt = build_heartbeat_prompt( goal_id="host-prompt-fixture", active_state=Path("ACTIVE_GOAL_STATE.md"), + registry_path=Path("registry.json"), agent_id="worker-a", registered_agents=["worker-a"], runtime_profile="codex_app_heartbeat", **{mode: True}, ) @@ -95,7 +99,7 @@ def run_probe(client: DoubaoExecToolClient, *, repeats: int = 2) -> dict: # repaired by feeding an answer or retried until one passes. response = client.next_final_content(messages) try: - decision = json.loads(response or "") + decision = json.loads(_strip_json_markdown_fence(response or "")) except (ValueError, TypeError): decision = None valid = (isinstance(decision, dict) diff --git a/loopx/control_plane/testing/model_tool_behavior.py b/loopx/control_plane/testing/model_tool_behavior.py index f398fee11f..2543b5ace3 100644 --- a/loopx/control_plane/testing/model_tool_behavior.py +++ b/loopx/control_plane/testing/model_tool_behavior.py @@ -6,17 +6,22 @@ import stat import subprocess import sys -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence, Set as AbstractSet from dataclasses import dataclass from hashlib import sha256 from pathlib import Path from typing import Any from .doubao_model_behavior_actor import ( + ALLOWED_DEEPSEEK_BEHAVIOR_MODELS, ALLOWED_MODEL_BEHAVIOR_MODELS, + DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + DEEPSEEK_FLASH_MODEL, DOUBAO_CHAT_COMPLETIONS_ENDPOINT, DoubaoActorTransport, DoubaoActorTransportError, + _deepseek_env_config, + _direct_deepseek_transport, ) EXEC_COMMAND_TOOL: dict[str, Any] = { @@ -201,7 +206,7 @@ def __call__( class DoubaoExecToolClient: - """One bounded Ark function-tool transport shared by real behavior gates.""" + """One bounded function-tool transport shared by real behavior gates.""" def __init__( self, @@ -210,21 +215,28 @@ def __init__( model: str, timeout_seconds: float, transport: DoubaoActorTransport, + endpoint: str = DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + allowed_models: AbstractSet[str] = ALLOWED_MODEL_BEHAVIOR_MODELS, + provider_prefix: str = "ark", ) -> None: + provider_label = "DeepSeek" if provider_prefix == "deepseek" else "Doubao" if not api_key.strip(): - raise RuntimeError("Doubao actor requires a runtime-injected API key") - if model not in ALLOWED_MODEL_BEHAVIOR_MODELS: - raise ValueError("Doubao actor model must be explicitly allowlisted") + raise RuntimeError(f"{provider_label} actor requires a runtime-injected API key") + if model not in allowed_models: + raise ValueError(f"{provider_label} actor model must be explicitly allowlisted") if timeout_seconds <= 0 or timeout_seconds > 300: - raise ValueError("Doubao actor timeout must be between 0 and 300 seconds") + raise ValueError(f"{provider_label} actor timeout must be between 0 and 300 seconds") self._api_key = api_key self._model = model self._timeout_seconds = timeout_seconds self._transport = transport + self._endpoint = endpoint + self._provider_prefix = provider_prefix + self._provider_label = provider_label @property def actor_ref(self) -> str: - return f"ark:{self._model}" + return f"{self._provider_prefix}:{self._model}" def next_tool_call( self, @@ -261,7 +273,7 @@ def next_step( } try: response = self._transport( - endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + endpoint=self._endpoint, headers={ "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", @@ -279,7 +291,7 @@ def next_step( raise except Exception: # noqa: BLE001 - custom transports may raise any error. raise DoubaoActorTransportError( - "Doubao tool behavior provider transport failed", + f"{self._provider_label} tool behavior provider transport failed", error_code="provider_transport_failed", ) from None @@ -299,7 +311,7 @@ def next_final_content( } try: response = self._transport( - endpoint=DOUBAO_CHAT_COMPLETIONS_ENDPOINT, + endpoint=self._endpoint, headers={ "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", @@ -315,7 +327,7 @@ def next_final_content( step = provider_exec_tool_step(response) if step.tool_call is not None: raise DoubaoActorTransportError( - "Doubao finalization returned an unavailable tool call", + f"{self._provider_label} finalization returned an unavailable tool call", error_code="provider_invalid_tool_call", ) return step.assistant_content @@ -323,11 +335,60 @@ def next_final_content( raise except Exception: # noqa: BLE001 - custom transports may raise any error. raise DoubaoActorTransportError( - "Doubao finalization provider transport failed", + f"{self._provider_label} finalization provider transport failed", error_code="provider_transport_failed", ) from None +class DeepSeekExecToolClient(DoubaoExecToolClient): + """One bounded DeepSeek function-tool transport shared by real behavior gates.""" + + def __init__( + self, + *, + api_key: str, + model: str = DEEPSEEK_FLASH_MODEL, + timeout_seconds: float = 90.0, + transport: DoubaoActorTransport = _direct_deepseek_transport, + endpoint: str = DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + ) -> None: + if endpoint != DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT: + raise DoubaoActorTransportError( + "DeepSeek actor endpoint is not the canonical DeepSeek endpoint", + error_code="noncanonical_endpoint", + ) + super().__init__( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, + allowed_models=ALLOWED_DEEPSEEK_BEHAVIOR_MODELS, + provider_prefix="deepseek", + ) + + @classmethod + def from_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DeepSeekExecToolClient: + api_key, resolved_model, endpoint = _deepseek_env_config( + environ=environ, + model=model, + ) + return cls( + api_key=api_key, + model=resolved_model, + timeout_seconds=timeout_seconds, + transport=transport, + endpoint=endpoint, + ) + + def digest_text(value: str) -> str: return f"sha256:{sha256(value.encode('utf-8')).hexdigest()}" diff --git a/loopx/control_plane/testing/replan_semantic_action_behavior.py b/loopx/control_plane/testing/replan_semantic_action_behavior.py index 91bab06095..803e671acc 100644 --- a/loopx/control_plane/testing/replan_semantic_action_behavior.py +++ b/loopx/control_plane/testing/replan_semantic_action_behavior.py @@ -23,9 +23,11 @@ DOUBAO_MODEL_ENV, DoubaoActorTransport, _direct_ark_transport, + _direct_deepseek_transport, ) from .model_tool_behavior import ( _module_reader_tokens, + DeepSeekExecToolClient, QuotaToolContext, is_context_read_command, LoopxCliExecutionError, @@ -1799,16 +1801,21 @@ class DoubaoReplanSemanticActionBehaviorActor: def __init__( self, *, - api_key: str, + api_key: str = "", model: str = DOUBAO_2_1_PRO_MODEL, timeout_seconds: float = 90.0, transport: DoubaoActorTransport = _direct_ark_transport, + client: DoubaoExecToolClient | None = None, ) -> None: - self._client = DoubaoExecToolClient( - api_key=api_key, - model=model, - timeout_seconds=timeout_seconds, - transport=transport, + self._client = ( + client + if client is not None + else DoubaoExecToolClient( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) ) @classmethod @@ -1832,6 +1839,24 @@ def from_environment( transport=transport, ) + @classmethod + def from_deepseek_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DoubaoReplanSemanticActionBehaviorActor: + return cls( + client=DeepSeekExecToolClient.from_environment( + environ=environ, + transport=transport, + timeout_seconds=timeout_seconds, + model=model, + ) + ) + def qualify( self, *, diff --git a/loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py b/loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py index 0787ac018a..90bb655716 100644 --- a/loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py +++ b/loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py @@ -17,8 +17,10 @@ DOUBAO_MODEL_ENV, DoubaoActorTransport, _direct_ark_transport, + _direct_deepseek_transport, ) from .model_tool_behavior import ( + DeepSeekExecToolClient, QuotaToolContext, is_context_read_command, DoubaoExecToolClient, @@ -322,16 +324,21 @@ class DoubaoScopedGateSuccessorToolBehaviorActor: def __init__( self, *, - api_key: str, + api_key: str = "", model: str = DOUBAO_2_1_PRO_MODEL, timeout_seconds: float = 90.0, transport: DoubaoActorTransport = _direct_ark_transport, + client: DoubaoExecToolClient | None = None, ) -> None: - self._client = DoubaoExecToolClient( - api_key=api_key, - model=model, - timeout_seconds=timeout_seconds, - transport=transport, + self._client = ( + client + if client is not None + else DoubaoExecToolClient( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) ) @classmethod @@ -355,6 +362,24 @@ def from_environment( transport=transport, ) + @classmethod + def from_deepseek_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DoubaoScopedGateSuccessorToolBehaviorActor: + return cls( + client=DeepSeekExecToolClient.from_environment( + environ=environ, + transport=transport, + timeout_seconds=timeout_seconds, + model=model, + ) + ) + def qualify( self, *, diff --git a/loopx/control_plane/testing/selected_todo_tool_behavior.py b/loopx/control_plane/testing/selected_todo_tool_behavior.py index 896bea6bc3..dc1758b89d 100644 --- a/loopx/control_plane/testing/selected_todo_tool_behavior.py +++ b/loopx/control_plane/testing/selected_todo_tool_behavior.py @@ -18,8 +18,10 @@ DOUBAO_MODEL_ENV, DoubaoActorTransport, _direct_ark_transport, + _direct_deepseek_transport, ) from .model_tool_behavior import ( + DeepSeekExecToolClient, QuotaToolContext, is_context_read_command, DoubaoExecToolClient, @@ -1022,16 +1024,21 @@ class DoubaoSelectedTodoToolBehaviorActor: def __init__( self, *, - api_key: str, + api_key: str = "", model: str = DOUBAO_2_1_PRO_MODEL, timeout_seconds: float = 90.0, transport: DoubaoActorTransport = _direct_ark_transport, + client: DoubaoExecToolClient | None = None, ) -> None: - self._client = DoubaoExecToolClient( - api_key=api_key, - model=model, - timeout_seconds=timeout_seconds, - transport=transport, + self._client = ( + client + if client is not None + else DoubaoExecToolClient( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) ) @classmethod @@ -1055,6 +1062,24 @@ def from_environment( transport=transport, ) + @classmethod + def from_deepseek_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DoubaoSelectedTodoToolBehaviorActor: + return cls( + client=DeepSeekExecToolClient.from_environment( + environ=environ, + transport=transport, + timeout_seconds=timeout_seconds, + model=model, + ) + ) + def qualify( self, *, diff --git a/loopx/control_plane/testing/terminal_settlement_tool_behavior.py b/loopx/control_plane/testing/terminal_settlement_tool_behavior.py index 12218c2e9f..78f5dadbea 100644 --- a/loopx/control_plane/testing/terminal_settlement_tool_behavior.py +++ b/loopx/control_plane/testing/terminal_settlement_tool_behavior.py @@ -22,8 +22,10 @@ DOUBAO_MODEL_ENV, DoubaoActorTransport, _direct_ark_transport, + _direct_deepseek_transport, ) from .model_tool_behavior import ( + DeepSeekExecToolClient, QuotaToolContext, DoubaoExecToolClient, QUOTA_FIRST_TOOL_INSTRUCTION, @@ -1051,16 +1053,21 @@ class DoubaoTerminalSettlementToolBehaviorActor: def __init__( self, *, - api_key: str, + api_key: str = "", model: str = DOUBAO_2_1_PRO_MODEL, timeout_seconds: float = 90.0, transport: DoubaoActorTransport = _direct_ark_transport, + client: DoubaoExecToolClient | None = None, ) -> None: - self._client = DoubaoExecToolClient( - api_key=api_key, - model=model, - timeout_seconds=timeout_seconds, - transport=transport, + self._client = ( + client + if client is not None + else DoubaoExecToolClient( + api_key=api_key, + model=model, + timeout_seconds=timeout_seconds, + transport=transport, + ) ) @classmethod @@ -1084,6 +1091,24 @@ def from_environment( transport=transport, ) + @classmethod + def from_deepseek_environment( + cls, + *, + environ: Mapping[str, str] | None = None, + transport: DoubaoActorTransport = _direct_deepseek_transport, + timeout_seconds: float = 90.0, + model: str | None = None, + ) -> DoubaoTerminalSettlementToolBehaviorActor: + return cls( + client=DeepSeekExecToolClient.from_environment( + environ=environ, + transport=transport, + timeout_seconds=timeout_seconds, + model=model, + ) + ) + def qualify( self, *, diff --git a/scripts/qualify-deepseek-model-behavior-live.py b/scripts/qualify-deepseek-model-behavior-live.py new file mode 100755 index 0000000000..365fb02048 --- /dev/null +++ b/scripts/qualify-deepseek-model-behavior-live.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import sys +import threading +from collections.abc import Mapping +from hashlib import sha256 +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +repo_root_text = str(REPO_ROOT) +if sys.path[0] != repo_root_text: + sys.path.insert(0, repo_root_text) + +# The candidate checkout must win over any installed LoopX package. +from loopx.control_plane.testing.actual_default_model_behavior_portfolio import ( # noqa: E402 + build_actual_default_model_behavior_scenario_inputs, + run_actual_default_model_behavior_portfolio, +) +from loopx.control_plane.testing.capability_monitor_repair_tool_behavior import ( # noqa: E402 + DoubaoCapabilityMonitorRepairToolBehaviorActor, +) +from loopx.control_plane.testing.doubao_model_behavior_actor import ( # noqa: E402 + DEEPSEEK_FLASH_MODEL, + DEEPSEEK_MODEL_ENV, + DeepSeekModelBehaviorActor, + DeepSeekOnboardingModelBehaviorActor, + _direct_deepseek_transport, +) +from loopx.control_plane.testing.release_commit_qualification import ( # noqa: E402 + collect_release_source_identity, +) +from loopx.control_plane.testing.replan_semantic_action_behavior import ( # noqa: E402 + DoubaoReplanSemanticActionBehaviorActor, +) +from loopx.control_plane.testing.scoped_gate_successor_tool_behavior import ( # noqa: E402 + DoubaoScopedGateSuccessorToolBehaviorActor, +) +from loopx.control_plane.testing.selected_todo_tool_behavior import ( # noqa: E402 + DoubaoSelectedTodoToolBehaviorActor, +) +from loopx.control_plane.testing.terminal_settlement_tool_behavior import ( # noqa: E402 + DoubaoTerminalSettlementToolBehaviorActor, +) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Run the actual-default behavior portfolio against the real DeepSeek " + "API and print only its bounded receipt." + ) + ) + parser.add_argument("--qualification-id", required=True) + parser.add_argument( + "--model", + default=os.environ.get(DEEPSEEK_MODEL_ENV, DEEPSEEK_FLASH_MODEL), + ) + parser.add_argument("--timeout-seconds", type=float, default=90.0) + parser.add_argument("--required-vision-timeout-seconds", type=float, default=180.0) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--max-workers", type=int, default=6) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.max_workers < 1: + raise ValueError("--max-workers must be at least 1") + source = collect_release_source_identity(args.repo_root) + if source["git_dirty"]: + raise RuntimeError( + "live DeepSeek qualification requires a clean candidate checkout" + ) + provider_call_count = 0 + provider_models: set[str] = set() + transport_lock = threading.Lock() + + def counted_transport( + *, endpoint: str, headers: Mapping[str, str], body: bytes, + timeout_seconds: float, + ) -> Mapping[str, Any]: + nonlocal provider_call_count + model_id = json.loads(body)["model"] + with transport_lock: + provider_call_count += 1 + provider_models.add(model_id) + return _direct_deepseek_transport( + endpoint=endpoint, headers=headers, body=body, + timeout_seconds=timeout_seconds, + ) + + turn_actor = DeepSeekModelBehaviorActor.from_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + onboarding_actor = DeepSeekOnboardingModelBehaviorActor.from_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + selected_todo_actor = DoubaoSelectedTodoToolBehaviorActor.from_deepseek_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + replan_semantic_action_actor = DoubaoReplanSemanticActionBehaviorActor.from_deepseek_environment( + timeout_seconds=args.required_vision_timeout_seconds, + transport=counted_transport, + model=args.model, + ) + scoped_gate_successor_actor = ( + DoubaoScopedGateSuccessorToolBehaviorActor.from_deepseek_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + ) + capability_monitor_repair_actor = ( + DoubaoCapabilityMonitorRepairToolBehaviorActor.from_deepseek_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + ) + terminal_settlement_actor = ( + DoubaoTerminalSettlementToolBehaviorActor.from_deepseek_environment( + timeout_seconds=args.timeout_seconds, + transport=counted_transport, + model=args.model, + ) + ) + with TemporaryDirectory(prefix="loopx-deepseek-live-") as temp_dir: + temp_root = Path(temp_dir).resolve() + + def qualify_selected_todo(run_id: str) -> dict[str, object]: + run_digest = sha256(run_id.encode("utf-8")).hexdigest()[:16] + return selected_todo_actor.qualify( + qualification_id=run_id, + fixture_root=temp_root / "selected-todo" / run_digest, + ) + + def qualify_replan_semantic_action(run_id: str) -> dict[str, object]: + run_digest = sha256(run_id.encode("utf-8")).hexdigest()[:16] + return replan_semantic_action_actor.qualify( + qualification_id=run_id, + fixture_root=temp_root / "replan-semantic-action" / run_digest, + required_vision=True, + ) + + def qualify_scoped_gate_successor(run_id: str) -> dict[str, object]: + run_digest = sha256(run_id.encode("utf-8")).hexdigest()[:16] + return scoped_gate_successor_actor.qualify( + qualification_id=run_id, + fixture_root=temp_root / "scoped-gate-successor" / run_digest, + ) + + def qualify_capability_monitor_repair(run_id: str) -> dict[str, object]: + run_digest = sha256(run_id.encode("utf-8")).hexdigest()[:16] + return capability_monitor_repair_actor.qualify( + qualification_id=run_id, + fixture_root=temp_root / "capability-monitor-repair" / run_digest, + ) + + def qualify_terminal_settlement(run_id: str) -> dict[str, object]: + run_digest = sha256(run_id.encode("utf-8")).hexdigest()[:16] + return terminal_settlement_actor.qualify( + qualification_id=run_id, + fixture_root=temp_root / "terminal-settlement" / run_digest, + ) + + sources, packets = build_actual_default_model_behavior_scenario_inputs( + temp_root / "portfolio" + ) + result = run_actual_default_model_behavior_portfolio( + packets, + scenario_sources=sources, + qualification_id=args.qualification_id, + turn_actor=turn_actor, + onboarding_actor=onboarding_actor, + selected_todo_actor=qualify_selected_todo, + replan_semantic_action_actor=qualify_replan_semantic_action, + scoped_gate_successor_actor=qualify_scoped_gate_successor, + capability_monitor_repair_actor=qualify_capability_monitor_repair, + terminal_settlement_actor=qualify_terminal_settlement, + max_workers=args.max_workers, + ) + result["source"] = source + # An actor invocation can make several HTTP calls while using tools. + # Persist aggregate provenance, never headers or request/response bodies. + result["provider_call_count"] = provider_call_count + result["provider_models"] = sorted(provider_models) + result["model_id"] = next(iter(provider_models)) if len(provider_models) == 1 else None + print(json.dumps(result, ensure_ascii=False, sort_keys=True, indent=2)) + return 0 if result["qualification_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualify-host-prompt-release.py b/scripts/qualify-host-prompt-release.py index de60b12d2a..51abd83b42 100644 --- a/scripts/qualify-host-prompt-release.py +++ b/scripts/qualify-host-prompt-release.py @@ -11,27 +11,59 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from loopx.control_plane.testing.doubao_model_behavior_actor import ( # noqa: E402 - DOUBAO_SEED_EVOLVING_MODEL, _direct_ark_transport, + ARK_API_KEY_ENV, + DEEPSEEK_API_KEY_ENV, + DEEPSEEK_BASE_URL_ENV, + DEEPSEEK_FLASH_MODEL, + DOUBAO_SEED_EVOLVING_MODEL, + _direct_ark_transport, + _direct_deepseek_transport, + resolve_deepseek_chat_completions_endpoint, ) from loopx.control_plane.testing.host_prompt_behavior import run_probe # noqa: E402 -from loopx.control_plane.testing.model_tool_behavior import DoubaoExecToolClient # noqa: E402 +from loopx.control_plane.testing.model_tool_behavior import ( # noqa: E402 + DeepSeekExecToolClient, + DoubaoExecToolClient, +) def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--release-live", action="store_true") + parser.add_argument("--provider", choices=("ark", "deepseek"), default="ark") + parser.add_argument("--model", default=None) parser.add_argument("--repeats", type=int, choices=range(1, 6), default=2) args = parser.parse_args(argv) if not args.release_live: print(json.dumps({"status": "skipped", "reason": "release_opt_in_required", "provider_call_count": 0})) return 0 - key = os.environ.get("ARK_API_KEY", "") - if not key: - print(json.dumps({"status": "skipped", "reason": "provider_credential_unavailable", "provider_call_count": 0})) - return 0 - client = DoubaoExecToolClient(api_key=key, model=DOUBAO_SEED_EVOLVING_MODEL, - timeout_seconds=90, transport=_direct_ark_transport) + if args.provider == "deepseek": + key = os.environ.get(DEEPSEEK_API_KEY_ENV, "") + if not key.strip(): + print(json.dumps({"status": "skipped", "reason": "provider_credential_unavailable", "provider_call_count": 0})) + return 0 + else: + key = os.environ.get(ARK_API_KEY_ENV, "") + if not key.strip(): + print(json.dumps({"status": "skipped", "reason": "provider_credential_unavailable", "provider_call_count": 0})) + return 0 try: + if args.provider == "deepseek": + endpoint = resolve_deepseek_chat_completions_endpoint(os.environ.get(DEEPSEEK_BASE_URL_ENV)) + client: DoubaoExecToolClient = DeepSeekExecToolClient( + api_key=key, + model=args.model or DEEPSEEK_FLASH_MODEL, + timeout_seconds=90, + transport=_direct_deepseek_transport, + endpoint=endpoint, + ) + else: + client = DoubaoExecToolClient( + api_key=key, + model=args.model or DOUBAO_SEED_EVOLVING_MODEL, + timeout_seconds=90, + transport=_direct_ark_transport, + ) report = run_probe(client, repeats=args.repeats) except Exception: # Never publish a transport exception, raw response or credential. diff --git a/tests/conftest.py b/tests/conftest.py index 045a662f7c..4b551f3971 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -202,6 +202,14 @@ def _isolated_worker_effect_runtime(tmp_path_factory, request): for variable in ("TMPDIR", "TEMP", "TMP"): patch.setenv(variable, str(temporary)) patch.setattr(tempfile, "tempdir", str(temporary)) + if ( + os.environ.get("NODE_DISABLE_COMPILE_CACHE") != "1" + and not os.environ.get("NODE_V8_COVERAGE") + ): + compile_cache = temporary / "node-compile-cache" + compile_cache.mkdir(mode=0o700, exist_ok=True) + os.chmod(compile_cache, 0o700) + patch.setenv("NODE_COMPILE_CACHE", str(compile_cache)) yield from loopx.control_plane.effect_runtime import restart_effect_runtime diff --git a/tests/control_plane/canonical_authority_fixture.py b/tests/control_plane/canonical_authority_fixture.py index ebb073aba3..57a7067c5d 100644 --- a/tests/control_plane/canonical_authority_fixture.py +++ b/tests/control_plane/canonical_authority_fixture.py @@ -7,16 +7,196 @@ """ from __future__ import annotations +import atexit +from collections import deque import json import os from datetime import datetime from pathlib import Path +import selectors import subprocess import tempfile +import threading +import time import loopx +_WORKER_LOCK = threading.Lock() +_WORKERS: dict[tuple[int, str, bool], _CanonicalAuthorityWorker] = {} +_SHUTDOWN_REGISTERED = False +_WORKER_REQUEST_TIMEOUT_SECONDS = 45.0 + + +class _CanonicalAuthorityWorker: + """Persistent Node worker for disposable canonical authority fixture setups.""" + + def __init__(self, key: tuple[int, str, bool]) -> None: + self.key = key + self.call_lock = threading.Lock() + self._stderr_tail: deque[str] = deque(maxlen=64) + self._process = self._spawn(Path(key[1])) + + @staticmethod + def _build_script(package: Path) -> str: + module = package / "control_plane/coordination/file_authority_store.ts" + provider_module = module.parent / "local_authority_provider.ts" + fence_module = package / "control_plane/coordination/legacy_writer_fence.ts" + codec_module = package / "control_plane/coordination/authority_store_codec.ts" + return ( + "import {createInterface} from 'node:readline';" + f"import {{FileAuthorityStore}} from {json.dumps(module.as_uri())};" + f"import {{selectLocalSqliteAuthority,openLocalAuthorityStore}} from {json.dumps(provider_module.as_uri())};" + f"import {{engageLegacyCoordinationWriterFence}} from {json.dumps(fence_module.as_uri())};" + f"import {{canonicalAuthoritySha256}} from {json.dumps(codec_module.as_uri())};" + "const rl=createInterface({input:process.stdin,crlfDelay:Infinity});" + "for await(const line of rl){" + "if(!line)continue;" + "try{" + "const req=JSON.parse(line);" + "let result;" + "if(req.op==='init'){" + "if(req.provider==='sqlite')await selectLocalSqliteAuthority(req.root,req.goal,true);" + "const store=await openLocalAuthorityStore(req.root,req.goal);" + "result=await store.commitAuthority({expected_provider_revision:null,operation_id:'canonical-fixture'," + "events:[],next_projection:req.projection,receipts:[]});" + "const fence=await engageLegacyCoordinationWriterFence({schema_version:'loopx_legacy_coordination_writer_fence_engage_request_v0'," + "runtime_root:req.root,goal_id:req.goal,state_path:req.state_path,fence:{schema_version:'loopx_legacy_coordination_writer_fence_v0'," + "state:'engaged',goal_id:req.goal,fence_id:'canonical-fixture',source_version:'canonical-fixture'," + "source_projection_sha256:canonicalAuthoritySha256(req.projection),expected_shadow_provider_revision:result.provider_revision}});" + "if(fence.status!=='applied')throw new Error(JSON.stringify(fence));" + "}else if(req.op==='advance'){" + "const store=await openLocalAuthorityStore(req.root,req.goal);" + "const head=await store.loadAuthority();" + "if(head.status!=='loaded')throw new Error(JSON.stringify(head));" + "result=await store.commitAuthority({expected_provider_revision:head.provider_revision," + "operation_id:req.operation_id,events:[],next_projection:req.projection,receipts:[]});" + "}else{throw new Error('unsupported canonical fixture op');}" + "process.stdout.write(JSON.stringify({ok:true,result})+'\\n');" + "}catch(err){" + "const msg=err instanceof Error?(err.stack||err.message):String(err);" + "process.stdout.write(JSON.stringify({ok:false,error:msg})+'\\n');" + "}}" + ) + + def _spawn(self, package: Path) -> subprocess.Popen[bytes]: + self._stderr_tail.clear() + proc = subprocess.Popen( + ["node", "--no-warnings", "--experimental-strip-types", "--input-type=module", "-e", self._build_script(package)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + assert proc.stderr is not None + + def _drain_stderr() -> None: + try: + for raw_line in iter(proc.stderr.readline, b""): + self._stderr_tail.append(raw_line.decode("utf-8", errors="replace").rstrip()) + except OSError: + pass + + threading.Thread(target=_drain_stderr, daemon=True).start() + return proc + + def _read_line(self, timeout_seconds: float) -> bytes: + stdout = self._process.stdout + assert stdout is not None + deadline = time.monotonic() + timeout_seconds + chunks = bytearray() + selector = selectors.DefaultSelector() + selector.register(stdout, selectors.EVENT_READ) + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("canonical authority worker timed out waiting for response") + events = selector.select(remaining) + if not events: + raise TimeoutError("canonical authority worker timed out waiting for response") + chunk = os.read(stdout.fileno(), 65536) + if not chunk: + raise EOFError("canonical authority worker closed stdout") + chunks.extend(chunk) + if b"\n" in chunk: + line, _rest = bytes(chunks).split(b"\n", 1) + return line + finally: + selector.close() + + def request(self, payload: dict[str, object]) -> dict: + encoded = (json.dumps(payload) + "\n").encode("utf-8") + with self.call_lock: + for attempt in range(2): + if self._process.poll() is not None: + self.close() + self._process = self._spawn(Path(self.key[1])) + try: + assert self._process.stdin is not None + self._process.stdin.write(encoded) + self._process.stdin.flush() + raw_line = self._read_line(_WORKER_REQUEST_TIMEOUT_SECONDS) + except (BrokenPipeError, EOFError, OSError): + self.close() + if attempt == 0: + self._process = self._spawn(Path(self.key[1])) + continue + stderr_text = "\n".join(self._stderr_tail) + raise AssertionError(f"canonical authority worker failed after respawn: {stderr_text}") + except TimeoutError: + self.close() + raise + response = json.loads(raw_line.decode("utf-8")) + if not response.get("ok"): + raise AssertionError(response.get("error") or "\n".join(self._stderr_tail)) + return response["result"] + raise AssertionError("unreachable") + + def close(self) -> None: + proc = self._process + for stream in (proc.stdin, proc.stdout, proc.stderr): + if stream is not None: + try: + stream.close() + except OSError: + pass + if proc.poll() is None: + try: + proc.wait(timeout=1.5) + except subprocess.TimeoutExpired: + proc.terminate() + try: + proc.wait(timeout=1.5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def _shutdown_canonical_fixture_workers() -> None: + current_pid = os.getpid() + with _WORKER_LOCK: + owned = [key for key in _WORKERS if key[0] == current_pid] + workers = [_WORKERS.pop(key) for key in owned] + for worker in workers: + worker.close() + + +def _get_canonical_authority_worker(package: Path) -> _CanonicalAuthorityWorker: + global _SHUTDOWN_REGISTERED + sqlite_enabled = "--experimental-sqlite" in os.environ.get("NODE_OPTIONS", "").split() + key = (os.getpid(), str(package), sqlite_enabled) + with _WORKER_LOCK: + if not _SHUTDOWN_REGISTERED: + atexit.register(_shutdown_canonical_fixture_workers) + _SHUTDOWN_REGISTERED = True + worker = _WORKERS.get(key) + if worker is None: + worker = _CanonicalAuthorityWorker(key) + _WORKERS[key] = worker + return worker + + def assert_todo_read_unchanged(current, previous): """Compare complete readback except validated evaluator/observation clocks.""" def stable(payload): @@ -63,31 +243,15 @@ def initialize_canonical_authority(runtime_root: Path, goal_id: str, projection: if provider not in ("file", "sqlite"): raise ValueError("Canonical local fixtures support File/SQLite only; PostgreSQL requires its service-owned fixture") package = Path(loopx.__file__).resolve().parent - module = package / "control_plane/coordination/file_authority_store.ts" - fence_module = package / "control_plane/coordination/legacy_writer_fence.ts" - codec_module = package / "control_plane/coordination/authority_store_codec.ts" - script = ( - f"import {{FileAuthorityStore}} from {json.dumps(module.as_uri())};" - f"import {{selectLocalSqliteAuthority,openLocalAuthorityStore}} from {json.dumps((module.parent / 'local_authority_provider.ts').as_uri())};" - f"import {{engageLegacyCoordinationWriterFence}} from {json.dumps(fence_module.as_uri())};" - f"import {{canonicalAuthoritySha256}} from {json.dumps(codec_module.as_uri())};" - "import {join} from 'node:path';let input='';for await(const chunk of process.stdin)input+=chunk;" - "const request=JSON.parse(input);" - "if(request.provider==='sqlite')await selectLocalSqliteAuthority(request.root,request.goal,true);" - "const store=await openLocalAuthorityStore(request.root,request.goal);" - "const result=await store.commitAuthority({expected_provider_revision:null,operation_id:'canonical-fixture'," - "events:[],next_projection:request.projection,receipts:[]});" - "const fence=await engageLegacyCoordinationWriterFence({schema_version:'loopx_legacy_coordination_writer_fence_engage_request_v0'," - "runtime_root:request.root,goal_id:request.goal,state_path:request.state_path,fence:{schema_version:'loopx_legacy_coordination_writer_fence_v0'," - "state:'engaged',goal_id:request.goal,fence_id:'canonical-fixture',source_version:'canonical-fixture'," - "source_projection_sha256:canonicalAuthoritySha256(request.projection),expected_shadow_provider_revision:result.provider_revision}});" - "if(fence.status!=='applied')throw new Error(JSON.stringify(fence));process.stdout.write(JSON.stringify(result));" - ) - process = subprocess.run(["node", "--no-warnings", "--experimental-strip-types", "--input-type=module", "-e", script], - input=json.dumps({"root": str(runtime_root), "goal": goal_id, "projection": projection, "state_path": str(state_path), "provider": provider}), - capture_output=True, text=True, check=False, timeout=45) - assert process.returncode == 0, process.stderr - result = json.loads(process.stdout) + worker = _get_canonical_authority_worker(package) + result = worker.request({ + "op": "init", + "root": str(runtime_root), + "goal": goal_id, + "projection": projection, + "state_path": str(state_path), + "provider": provider, + }) assert result["status"] == "applied", result return result @@ -114,26 +278,15 @@ def advance_canonical_fixture_authority( Consumer fixtures may author a later synthetic state, but must not replace storage bytes, reset lineage, re-import Markdown or re-engage its fence. """ - module = Path(loopx.__file__).resolve().parent / "control_plane/coordination/local_authority_provider.ts" - script = ( - f"import {{openLocalAuthorityStore}} from {json.dumps(module.as_uri())};" - "let input='';for await(const chunk of process.stdin)input+=chunk;" - "const request=JSON.parse(input);" - "const store=await openLocalAuthorityStore(request.root,request.goal);" - "const head=await store.loadAuthority();" - "if(head.status!=='loaded')throw new Error(JSON.stringify(head));" - "const result=await store.commitAuthority({expected_provider_revision:head.provider_revision," - "operation_id:request.operation_id,events:[],next_projection:request.projection,receipts:[]});" - "process.stdout.write(JSON.stringify(result));" - ) - process = subprocess.run( - ["node", "--no-warnings", "--experimental-strip-types", "--input-type=module", "-e", script], - input=json.dumps({"root": str(runtime_root), "goal": goal_id, - "projection": projection, "operation_id": operation_id}), - capture_output=True, text=True, check=False, timeout=45, - ) - assert process.returncode == 0, process.stderr - result = json.loads(process.stdout) + package = Path(loopx.__file__).resolve().parent + worker = _get_canonical_authority_worker(package) + result = worker.request({ + "op": "advance", + "root": str(runtime_root), + "goal": goal_id, + "projection": projection, + "operation_id": operation_id, + }) assert result["status"] == "applied", result return result diff --git a/tests/control_plane/test_actual_default_model_behavior_live.py b/tests/control_plane/test_actual_default_model_behavior_live.py index f50d9a68c3..c6d52bff93 100644 --- a/tests/control_plane/test_actual_default_model_behavior_live.py +++ b/tests/control_plane/test_actual_default_model_behavior_live.py @@ -79,3 +79,226 @@ def portfolio(*_, terminal_settlement_actor, replan_semantic_action_actor, **__) assert len(prepared) == 2 assert all(path == path.resolve() and path.is_relative_to(physical.resolve()) for path in prepared) assert json.loads(capsys.readouterr().out)["provider_call_count"] == 0 + + +@pytest.mark.parametrize("base_url", [None, "", " ", "https://api.deepseek.com", "https://api.deepseek.com/", "https://api.deepseek.com/v1", "https://api.deepseek.com/v1/"]) +def test_resolve_deepseek_endpoint_accepts_canonical_urls(base_url): + from loopx.control_plane.testing.doubao_model_behavior_actor import ( + DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + resolve_deepseek_chat_completions_endpoint, + ) + + assert resolve_deepseek_chat_completions_endpoint(base_url) == DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + + +@pytest.mark.parametrize("bad_url", [ + "http://api.deepseek.com", + "https://api.deepseek.com/anthropic", + "https://api.deepseek.com/v2", + "https://evil.example/v1", +]) +def test_resolve_deepseek_endpoint_rejects_noncanonical_urls(bad_url): + from loopx.control_plane.testing.doubao_model_behavior_actor import ( + DoubaoActorTransportError, + resolve_deepseek_chat_completions_endpoint, + ) + + with pytest.raises(DoubaoActorTransportError) as exc_info: + resolve_deepseek_chat_completions_endpoint(bad_url) + assert exc_info.value.error_code == "noncanonical_endpoint" + + +def test_deepseek_actors_strip_markdown_fences_and_preserve_canonical_boundary(monkeypatch): + from io import BytesIO + from urllib.error import HTTPError + from loopx.control_plane.testing.doubao_model_behavior_actor import ( + DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + DEEPSEEK_FLASH_MODEL, + DEEPSEEK_V4_PRO_MODEL, + DeepSeekModelBehaviorActor, + DeepSeekOnboardingModelBehaviorActor, + DoubaoActorTransportError, + _direct_deepseek_transport, + ) + from loopx.control_plane.testing.model_behavior_qualification import ( + build_model_behavior_actor_request, + normalize_model_behavior_actor_result, + ) + from loopx.control_plane.testing.onboarding_model_behavior_qualification import ( + ONBOARDING_MODEL_BEHAVIOR_DECISION_SCHEMA_VERSION, + build_onboarding_model_behavior_actor_request, + normalize_onboarding_model_behavior_actor_result, + onboarding_entry_semantic_contract, + ) + + decision = { + "schema_version": "model_behavior_decision_v0", + "decision": "execute", + "selected_todo_id": "todo_fixture001", + "user_action_required": False, + "must_attempt_work": True, + "delivery_allowed": True, + "quiet_noop_allowed": False, + "external_write_requested": False, + "intended_action_kinds": ["inspect", "test", "writeback"], + "reason_codes": ["bounded_delivery"], + } + captured = {} + + def transport(*, endpoint, headers, body, timeout_seconds): + captured.update(endpoint=endpoint, headers=dict(headers), body=json.loads(body), timeout_seconds=timeout_seconds) + return {"choices": [{"message": {"content": f"```json\n{json.dumps(decision)}\n```"}}]} + + actor = DeepSeekModelBehaviorActor.from_environment( + environ={"DEEPSEEK_API_KEY": "fixture-key-not-a-secret", "DEEPSEEK_BASE_URL": "https://api.deepseek.com/v1"}, + transport=transport, + ) + req = build_model_behavior_actor_request( + {"schema_version": "loopx_turn_envelope_v0", "action": {"selected_todo": {"todo_id": "todo_fixture001"}}}, + qualification_id="case-ds-001", + arm="candidate_packet", + ) + res = normalize_model_behavior_actor_result(actor(req)) + assert res["actor_ref"] == f"deepseek:{DEEPSEEK_FLASH_MODEL}" + assert captured["endpoint"] == DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT + assert captured["body"]["model"] == DEEPSEEK_FLASH_MODEL + + onboarding_packet = { + "schema_version": "loopx_start_goal_guided_v0", + "goal_id": "fixture-goal", + "agent_id": "codex-fixture", + "guided_transaction": {"writes_now": False, "spends_quota_now": False, "ordered_steps": [{"id": "connect_if_needed"}]}, + "commands": { + "goal_start_connect_if_needed": "loopx bootstrap", + "goal_start_refresh_state": "loopx refresh-state", + "goal_start_host_loop_activation": "loopx heartbeat-prompt --thin", + "goal_start_quota_should_run": "loopx quota should-run", + }, + "host_loop_activation": {"agent_id": "codex-fixture", "activation_required_after_todo_write": True}, + } + onboarding_decision = { + "schema_version": ONBOARDING_MODEL_BEHAVIOR_DECISION_SCHEMA_VERSION, + "phase": "entry", + "next_action": "connect_if_needed", + "semantic_contract": onboarding_entry_semantic_contract(onboarding_packet), + "reason_codes": ["source_aligned"], + } + onb_actor = DeepSeekOnboardingModelBehaviorActor.from_environment( + environ={"DEEPSEEK_API_KEY": "fixture-key-not-a-secret", "LOOPX_DEEPSEEK_BEHAVIOR_MODEL": DEEPSEEK_V4_PRO_MODEL}, + transport=lambda **_: {"choices": [{"message": {"content": f"```\n{json.dumps(onboarding_decision)}\n```"}}]}, + ) + onb_res = normalize_onboarding_model_behavior_actor_result( + onb_actor(build_onboarding_model_behavior_actor_request(onboarding_packet, qualification_id="onb-ds-001", phase="entry")), + phase="entry", + ) + assert onb_res["actor_ref"] == f"deepseek:{DEEPSEEK_V4_PRO_MODEL}" + + with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"): + DeepSeekModelBehaviorActor.from_environment(environ={}) + with pytest.raises(ValueError, match="allowlisted"): + DeepSeekModelBehaviorActor.from_environment(environ={"DEEPSEEK_API_KEY": "fixture", "LOOPX_DEEPSEEK_BEHAVIOR_MODEL": "unknown-ds"}) + with pytest.raises(DoubaoActorTransportError) as err: + _direct_deepseek_transport(endpoint="https://evil.example/chat/completions", headers={}, body=b"{}", timeout_seconds=1.0) + assert err.value.error_code == "noncanonical_endpoint" + + class UnauthorizedOpener: + def open(self, *_, **__): + raise HTTPError(DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, 401, "private detail", hdrs=None, fp=BytesIO(b'{"error":"secret"}')) + + monkeypatch.setattr("loopx.control_plane.testing.doubao_model_behavior_actor.build_opener", lambda *_: UnauthorizedOpener()) + with pytest.raises(DoubaoActorTransportError) as auth_err: + _direct_deepseek_transport(endpoint=DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, headers={}, body=b"{}", timeout_seconds=1.0) + assert auth_err.value.error_code == "provider_authentication_failed" + assert "DEEPSEEK_API_KEY" in str(auth_err.value) + assert "secret" not in str(auth_err.value) + + +def test_deepseek_tool_client_and_all_five_tool_actors_reuse_state_machines(): + from loopx.control_plane.testing.capability_monitor_repair_tool_behavior import DoubaoCapabilityMonitorRepairToolBehaviorActor + from loopx.control_plane.testing.doubao_model_behavior_actor import ( + DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + DEEPSEEK_FLASH_MODEL, + DEEPSEEK_V4_PRO_MODEL, + DoubaoActorTransportError, + ) + from loopx.control_plane.testing.model_tool_behavior import DeepSeekExecToolClient, DoubaoExecToolClient + from loopx.control_plane.testing.replan_semantic_action_behavior import DoubaoReplanSemanticActionBehaviorActor + from loopx.control_plane.testing.scoped_gate_successor_tool_behavior import DoubaoScopedGateSuccessorToolBehaviorActor + from loopx.control_plane.testing.selected_todo_tool_behavior import DoubaoSelectedTodoToolBehaviorActor + from loopx.control_plane.testing.terminal_settlement_tool_behavior import DoubaoTerminalSettlementToolBehaviorActor + + seen = [] + + def transport(*, endpoint, headers, body, timeout_seconds): + seen.append((endpoint, json.loads(body)["model"])) + return {"choices": [{"message": {"role": "assistant", "content": "done"}}]} + + client = DeepSeekExecToolClient.from_environment( + environ={"DEEPSEEK_API_KEY": "fixture-key-not-a-secret"}, + transport=transport, + ) + assert client.actor_ref == f"deepseek:{DEEPSEEK_FLASH_MODEL}" + assert client.next_final_content([{"role": "user", "content": "hi"}]) == "done" + assert seen == [(DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, DEEPSEEK_FLASH_MODEL)] + + custom = DoubaoExecToolClient( + api_key="fixture-key", model=DEEPSEEK_V4_PRO_MODEL, timeout_seconds=10, + transport=transport, endpoint=DEEPSEEK_CHAT_COMPLETIONS_ENDPOINT, + allowed_models={DEEPSEEK_V4_PRO_MODEL}, provider_prefix="deepseek", + ) + assert custom.actor_ref == f"deepseek:{DEEPSEEK_V4_PRO_MODEL}" + + with pytest.raises(DoubaoActorTransportError) as bad_ep: + DeepSeekExecToolClient(api_key="fixture-key", endpoint="https://evil.example/chat/completions") + assert bad_ep.value.error_code == "noncanonical_endpoint" + + for actor_cls in ( + DoubaoSelectedTodoToolBehaviorActor, + DoubaoReplanSemanticActionBehaviorActor, + DoubaoScopedGateSuccessorToolBehaviorActor, + DoubaoCapabilityMonitorRepairToolBehaviorActor, + DoubaoTerminalSettlementToolBehaviorActor, + ): + actor = actor_cls.from_deepseek_environment( + environ={"DEEPSEEK_API_KEY": "fixture-key-not-a-secret"}, + model=DEEPSEEK_V4_PRO_MODEL, + ) + assert actor._client.actor_ref == f"deepseek:{DEEPSEEK_V4_PRO_MODEL}" + with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"): + actor_cls.from_deepseek_environment(environ={}) + + +def test_deepseek_live_runner_wires_all_actors_and_defaults_cleanly(tmp_path, monkeypatch, capsys): + source = Path(__file__).resolve().parents[2] + spec = importlib.util.spec_from_file_location( + "live_deepseek_portfolio_fixture", source / "scripts/qualify-deepseek-model-behavior-live.py", + ) + runner = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runner) + assert runner._parser().parse_args(["--qualification-id", "q1"]).max_workers == 6 + assert runner._parser().parse_args(["--qualification-id", "q1"]).model == "deepseek-flash" + + monkeypatch.setattr(runner, "collect_release_source_identity", lambda _: {"git_dirty": False}) + monkeypatch.setattr(runner, "build_actual_default_model_behavior_scenario_inputs", lambda _: ({}, {})) + seen_models = [] + + def factory(*, model=None, **_): + seen_models.append(model) + return SimpleNamespace(qualify=lambda **__: {"qualification_passed": True}) + + for name in ("DeepSeekModelBehaviorActor", "DeepSeekOnboardingModelBehaviorActor"): + monkeypatch.setattr(getattr(runner, name), "from_environment", factory) + for name in ( + "DoubaoSelectedTodoToolBehaviorActor", + "DoubaoReplanSemanticActionBehaviorActor", + "DoubaoScopedGateSuccessorToolBehaviorActor", + "DoubaoCapabilityMonitorRepairToolBehaviorActor", + "DoubaoTerminalSettlementToolBehaviorActor", + ): + monkeypatch.setattr(getattr(runner, name), "from_deepseek_environment", factory) + monkeypatch.setattr(runner, "run_actual_default_model_behavior_portfolio", lambda *_, max_workers, **__: {"qualification_passed": True, "max_workers": max_workers}) + + assert runner.main(["--qualification-id", "ds-offline", "--model", "deepseek-v4-pro", "--max-workers", "4"]) == 0 + assert seen_models == ["deepseek-v4-pro"] * 7 + payload = json.loads(capsys.readouterr().out) + assert payload["provider_call_count"] == 0 and payload["max_workers"] == 4 diff --git a/tests/control_plane/test_canonical_authority_fixture.py b/tests/control_plane/test_canonical_authority_fixture.py index 5e48ef6066..3eddb833f0 100644 --- a/tests/control_plane/test_canonical_authority_fixture.py +++ b/tests/control_plane/test_canonical_authority_fixture.py @@ -1,6 +1,8 @@ """Canonical CLI fixtures retain runtime isolation and actual backend identity.""" from __future__ import annotations +import os +from pathlib import Path import subprocess import sys import tempfile @@ -70,3 +72,37 @@ def test_local_fixture_reads_its_actual_provider_through_cli(tmp_path, monkeypat assert result["todo"]["todo_id"] == fixture.TODO_ID assert result["authority_read"]["decision_read_from_provider"] assert not result["authority_read"]["legacy_fallback_used"] + + +def test_session_effect_runtime_sets_private_node_compile_cache(tmp_path): + cache = os.environ.get("NODE_COMPILE_CACHE") + if os.environ.get("NODE_DISABLE_COMPILE_CACHE") == "1" or os.environ.get("NODE_V8_COVERAGE"): + pytest.skip("compile cache disabled by caller environment") + assert cache + cache_path = Path(cache) + assert cache_path.is_dir() + assert cache_path.name == "node-compile-cache" + assert cache_path.stat().st_mode & 0o077 == 0 + + +def test_persistent_canonical_worker_reuses_and_auto_respawns_once(tmp_path, monkeypatch): + import canonical_authority_fixture as fixture_mod + from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection + + isolate_sqlite_runtime(tmp_path, monkeypatch) + runtime = tmp_path / "runtime" + state = tmp_path / "state.md" + state.write_text("# Goal\n", encoding="utf-8") + projection = build_todo_runtime_shadow_projection(goal_id="goal-respawn", todos=[], handoff_mode="soft_claim") + first = fixture_mod.initialize_canonical_authority(runtime, "goal-respawn", projection, state_path=state, provider="sqlite") + assert first["status"] == "applied" + + worker = fixture_mod._get_canonical_authority_worker(Path(fixture_mod.loopx.__file__).resolve().parent) + original_pid = worker._process.pid + worker._process.kill() + worker._process.wait(timeout=5) + + second = fixture_mod.advance_canonical_fixture_authority(runtime, "goal-respawn", projection, operation_id="after-kill") + assert second["status"] == "applied" + assert worker._process.pid != original_pid + assert worker._process.poll() is None diff --git a/tests/control_plane/test_canonical_lease_scopes.py b/tests/control_plane/test_canonical_lease_scopes.py index dd869aa01e..4e3328b459 100644 --- a/tests/control_plane/test_canonical_lease_scopes.py +++ b/tests/control_plane/test_canonical_lease_scopes.py @@ -1,4 +1,6 @@ """Scope comparison must preserve request identity and accepted declarations.""" +from contextlib import redirect_stdout +import io import json import os import subprocess @@ -8,6 +10,7 @@ import pytest from canonical_authority_fixture import initialize_canonical_authority, isolate_sqlite_runtime +from loopx.cli import main as cli_main from loopx.control_plane.coordination.local_authority import read_canonical_todos_if_promoted from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection @@ -36,6 +39,18 @@ def scope_goal(tmp_path, monkeypatch, request): initialize_canonical_authority(runtime, goal, projection, state_path=state, provider=request.param) def cli(*args, cwd=None, expected_exit=0): + if cwd is None and not any(name.startswith("LOOPX_LEASE_SCOPE_") for name in os.environ): + output = io.StringIO() + with redirect_stdout(output): + code = cli_main([ + "--registry", str(registry), + "--runtime-root", str(runtime), + "--format", "json", + *args, + "--goal-id", goal, + ]) + assert code == expected_exit, output.getvalue() + return json.loads(output.getvalue()) selected_repo = Path(os.environ.get("LOOPX_LEASE_SCOPE_CONSUMER", REPO)) if cwd is None else cwd child = subprocess.run([sys.executable, "-m", "loopx.cli", "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", *args, "--goal-id", goal], diff --git a/tests/control_plane/test_dependency_vision_wait_cli.py b/tests/control_plane/test_dependency_vision_wait_cli.py index 7d2c74844c..7501e42d18 100644 --- a/tests/control_plane/test_dependency_vision_wait_cli.py +++ b/tests/control_plane/test_dependency_vision_wait_cli.py @@ -154,7 +154,7 @@ def worker_args(goal): error_backoff_seconds=5, quota_timeout_seconds=90, wake_timeout_seconds=None) -def test_pending_dependency_stays_quiet_for_100_polls_restart_and_100_more(public_goal, monkeypatch): +def test_pending_dependency_stays_quiet_for_12_polls_restart_and_12_more(public_goal, monkeypatch): goal = public_goal source, waiting = wait_on(goal, "todo_done") before = goal.call("todo", "list", "--goal-id", GOAL) @@ -189,14 +189,14 @@ class RestartWorker(Exception): def advance(seconds): sleeps.append(seconds) - if len(sleeps) in (100, 200): + if len(sleeps) in (12, 24): raise RestartWorker monkeypatch.setattr(worker, "run_quota_should_run", probe) monkeypatch.setattr(worker, "_run_wake", unexpected_wake) args = worker_args(goal) args.once = False - for expected in (100, 200): + for expected in (12, 24): # Interrupt only after the actual worker has persisted the tick, then # reopen that same state route. No real hours or model calls are used. with pytest.raises(RestartWorker): @@ -215,7 +215,7 @@ def advance(seconds): events = [json.loads(line) for line in events_path.read_text().splitlines()] assert not any(event.get("event_kind") in ("quota_monitor_poll", "quota_spend") for event in events) assert counts == {"D": 0, "P": 0, "E": 0, "Q": 0} - assert len(sleeps) == 200 and sleeps[-1] == 3600 + assert len(sleeps) == 24 and sleeps[-1] == 3600 @pytest.mark.parametrize("kind", ["todo_done", "monitor_changed"]) diff --git a/tests/control_plane/test_host_prompt_behavior.py b/tests/control_plane/test_host_prompt_behavior.py index d50e55f8b5..94d4bafccd 100644 --- a/tests/control_plane/test_host_prompt_behavior.py +++ b/tests/control_plane/test_host_prompt_behavior.py @@ -176,3 +176,46 @@ def fail(*args, **kwargs): result = capsys.readouterr().out assert "private provider" not in result assert json.loads(result)["status"] == "failed" + + +def test_release_probe_supports_deepseek_provider_and_fails_closed(monkeypatch, capsys): + script = Path(__file__).resolve().parents[2] / "scripts/qualify-host-prompt-release.py" + main = runpy.run_path(str(script))["main"] + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.delenv("DEEPSEEK_BASE_URL", raising=False) + assert main(["--release-live", "--provider", "deepseek"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "skipped" + + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-only-unused") + monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://evil.example/v1") + assert main(["--release-live", "--provider", "deepseek"]) == 1 + assert json.loads(capsys.readouterr().out) == {"status": "failed", "reason": "probe_execution_failed"} + + seen = [] + + def fake_probe(client, *, repeats): + seen.append((client.actor_ref, repeats)) + return {"qualification_passed": True, "actor_ref": client.actor_ref, "provider_call_count": 0} + + monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com") + monkeypatch.setitem(main.__globals__, "run_probe", fake_probe) + assert main(["--release-live", "--provider", "deepseek", "--model", "deepseek-v4-pro", "--repeats", "1"]) == 0 + out = json.loads(capsys.readouterr().out) + assert out["actor_ref"] == "deepseek:deepseek-v4-pro" + assert seen == [("deepseek:deepseek-v4-pro", 1)] + + +def test_probe_handles_dual_global_roots_and_json_fences(monkeypatch, tmp_path): + (tmp_path / ".loopx").mkdir() + (tmp_path / ".loopx" / "registry.global.json").write_text("{}", encoding="utf-8") + (tmp_path / ".codex" / "loopx").mkdir(parents=True) + (tmp_path / ".codex" / "loopx" / "registry.global.json").write_text("{}", encoding="utf-8") + monkeypatch.setenv("HOME", str(tmp_path)) + + class FencedClient(ScriptedClient): + def next_final_content(self, messages): + return f"```json\n{super().next_final_content(messages)}\n```" + + report = run_probe(FencedClient(), repeats=1) + assert report["qualification_passed"] is True + assert report["provider_call_count"] == 10 diff --git a/tests/test_chat_intake_evaluation.py b/tests/test_chat_intake_evaluation.py index 58a213065f..afff0e948f 100644 --- a/tests/test_chat_intake_evaluation.py +++ b/tests/test_chat_intake_evaluation.py @@ -516,3 +516,69 @@ def urlopen(request, **_): if row["id"] in routed: assert row["review_response"]["context_handoff"]["brief"]["constraints"] == ["App only; do not publish."] assert "synthetic-test-only" not in output.read_text() + + +def test_chat_intake_suite_includes_30_cases_and_v240_status_and_watch_oracles(): + suite = json.loads((ROOT / "examples/evaluations/chat-intake.public.json").read_text()) + assert len(suite["cases"]) == 30 + cases = {row["id"]: row for row in suite["cases"]} + + pr_status = cases["unconfirmed-pr-merge-status-only"] + assert pr_status["expected"] == "answer" and pr_status["context_ref"] == "open-object" + assert evaluation.score(pr_status, {"message": "PR 42 is still OPEN and unmerged; checks are failing."})["passed"] + for bad in ( + {"message": "Handing off repair.", "context_handoff": {"goal_id": "example-product", "agent_id": "product-owner"}}, + {"message": "Proposing merge.", "protected_action": {"operation": "merge", "target": "42"}}, + {"message": "Drafting.", "goal_draft": {"objective": "Merge PR 42"}}, + {"message": "Adding todo.", "proposals": [{"kind": "todo"}]}, + ): + assert not evaluation.score(pr_status, bad)["passed"] + + watch = cases["watch-only-monitor-not-completed"] + assert watch["expected"] == "answer" and watch["context_ref"] == "watch-only-monitor" + assert watch["required_evidence_refs"] == ["monitor-tick-1"] + watch_context = suite["contexts"]["watch-only-monitor"] + assert watch_context["external_observations"][0]["ref"] == "monitor-tick-1" + assert watch_context["goals"][0]["current_todos"][0]["watch_only"] is True + assert evaluation.score(watch, {"message": "monitor-tick-1 recorded unchanged watch-only status; task work is not completed."})["passed"] + assert not evaluation.score(watch, {"message": "The watch-only check completed the task."})["passed"] + assert not evaluation.score(watch, {"message": "monitor-tick-1", "context_handoff": {"goal_id": "example-product", "agent_id": "product-owner"}})["passed"] + + +@pytest.mark.parametrize("workers_args, expected_workers", [([], 6), (["--workers", "1"], 1), (["--workers", "12"], 12)]) +def test_chat_intake_cli_supports_bounded_workers_and_rejects_out_of_range( + tmp_path, monkeypatch, workers_args, expected_workers +): + from io import BytesIO + + cases = tmp_path / "cases.json" + cases.write_text(json.dumps({"contexts": {"ordinary": {"scope": "public", "goals": []}}, "cases": [ + {"id": "ordinary", "context_ref": "ordinary", "expected": "answer", "request": "Explain Goals."}]})) + output = tmp_path / "report.json" + captured_workers = [] + real_pool = evaluation.ThreadPoolExecutor + + def capturing_pool(*, max_workers): + captured_workers.append(max_workers) + return real_pool(max_workers=max_workers) + + monkeypatch.setattr(evaluation, "ThreadPoolExecutor", capturing_pool) + monkeypatch.setattr(evaluation, "operator_provider_environ", lambda _: {"DEEPSEEK_API_KEY": "synthetic-test-only"}) + monkeypatch.setattr( + evaluation.urllib.request, + "urlopen", + lambda *_, **__: BytesIO(json.dumps({"choices": [{"finish_reason": "stop", "message": { + "content": evaluation.CHAT_REVIEW_OPEN_TAG + json.dumps({"message": "A Goal defines an outcome."}) + evaluation.CHAT_REVIEW_CLOSE_TAG + }}]}).encode()), + ) + monkeypatch.setattr(sys, "argv", ["chat-intake", "--live", "--model", "fixture-model", + "--cases", str(cases), "--output", str(output), *workers_args]) + assert evaluation.main() == 0 + assert captured_workers == [expected_workers] + + for invalid in ("0", "13"): + monkeypatch.setattr(sys, "argv", ["chat-intake", "--live", "--model", "fixture-model", + "--cases", str(cases), "--output", str(output), "--workers", invalid]) + with pytest.raises(SystemExit) as err: + evaluation.main() + assert err.value.code == 2