From ff85f6e460978c4bfa13574be2a076b3ad99cd26 Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 15 Sep 2026 17:07:47 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8C=89=E4=BB=BB=E5=8A=A1=E5=A4=8D?= =?UTF-8?q?=E6=9D=82=E5=BA=A6=E8=B7=AF=E7=94=B1=E5=A7=94=E6=B4=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 + README.zh-CN.md | 6 + docs/task-routing-pilot-2026-09-15.json | 202 +++++++++++++++ docs/task-routing-pilot-2026-09-15.md | 83 ++++++ plugins/deepseek-code-agent/README.md | 6 +- plugins/deepseek-code-agent/package.json | 2 +- .../deepseek-coding-delegation/SKILL.md | 9 +- .../tests/routing-policy.test.mjs | 239 ++++++++++++++++++ 8 files changed, 549 insertions(+), 4 deletions(-) create mode 100644 docs/task-routing-pilot-2026-09-15.json create mode 100644 docs/task-routing-pilot-2026-09-15.md create mode 100644 plugins/deepseek-code-agent/tests/routing-policy.test.mjs diff --git a/README.md b/README.md index f9405b6..5f10a35 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,12 @@ Codex selects the relevant repository instructions, passes bounded `scope_paths` ## Queen token efficiency +### Route by task complexity + +Choose the route from reasoning, risk, and exploration/verification burden, not line or file counts. **Small** known edits that take a few minutes stay with Codex. **Medium** mechanical work with clear acceptance criteria prefers one persistent worker. **Large or high-risk** work — state machines, concurrency, permissions, transactions, migrations — may still delegate the labor, but Codex must first fix the design, interfaces, invariants, failure semantics and acceptance cases, then review the actual diff and run independent verification; a worker's self-report or passing self-tests are never acceptance. An explicit user request to delegate or to stay direct overrides these defaults. + +A 2026-09-15 pilot paired one direct and one delegated execution per size from the same commit: [task routing pilot](./docs/task-routing-pilot-2026-09-15.md). It ran one execution per cell in parallel without alternating order, so it is not the repeated benchmark the queen protocol asks for, and its numbers neither generalize nor promise a fixed saving. In that pilot, delegated worker non-cached input plus output was 13.78% below the three direct runs, while the controller's own interval was 55.41% below the direct proxy total; the controller interval also covers managing the native control arm and reviewing both runs, so it is not a pure delegation bill. Worker totals including cached reads were higher than direct totals, so combined tokens and latency can still increase; medium and large worker usage is marked incomplete. + ### Design, implement, verify, review Codex reads the critical call chain and decides interfaces, invariants and acceptance cases before delegation. An optional `task_spec` gives the worker those decisions and the exact check commands. The worker completes implementation, self-tests and routine repairs in one session. Codex reviews the patch and batches feedback; after two unsuccessful correction rounds it can take over after confirming the worker and verification have stopped. diff --git a/README.zh-CN.md b/README.zh-CN.md index 2e7c7b6..6ddd2c9 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -149,6 +149,12 @@ Codex 会选择相关仓库规则，传入明确的 `scope_paths`、任务相关 ## Queen Token 优化 +### 按任务复杂度路由 + +路由依据判断量、风险以及探索与验证负担，而不是行数或文件数。**小型**已知改动几分钟可完成，由 Codex 直接处理；**中型**机械实现且验收标准清晰，优先交给一个持久执行者；**大型或高风险**任务（状态机、并发、权限、事务、迁移等）可以把实现交给执行者，但 Codex 必须先确定设计、接口、不变量、失败语义和验收案例，再审查实际 diff 并独立验证，绝不按 worker 自报或自测通过来接受。用户明确要求委派或直接处理时，以用户要求为准。 + +2026-09-15 的 pilot 在同一基线 commit 上为每个规模各做了一次 direct 与委派执行：[任务分级 pilot](./docs/task-routing-pilot-2026-09-15.md)。每格仅一次、双臂并行、顺序未交替，因此它不是 queen 协议要求的三次重复正式基准，结果不能推广，也不承诺固定节省。该 pilot 中，worker 非缓存输入加输出合计比三次 direct 低 13.78%，controller 自身区间的 proxy 比 direct proxy 合计低 55.41%；controller 区间也包含原生对照管理和两边审查，不能当作纯委派账单。计入缓存后 worker total 高于 direct total，总 Token 和耗时仍可能增加；medium 和 large 的 worker 用量标记为不完整。 + ### Codex 设计与审核，GLM 实现 Codex 先阅读关键调用链，明确接口、必须保持的行为和验收案例，通过可选 `task_spec` 一次交付设计决定和检查命令。执行模型在同一会话中完成实现、自测和常规修复，Codex 集中审核并批量反馈；两轮修正仍未通过时，确认 worker 和验收进程停止后再接管。 diff --git a/docs/task-routing-pilot-2026-09-15.json b/docs/task-routing-pilot-2026-09-15.json new file mode 100644 index 0000000..03622d3 --- /dev/null +++ b/docs/task-routing-pilot-2026-09-15.json @@ -0,0 +1,202 @@ +{ + "schema_version": 1, + "pilot_id": "task-routing-pilot-2026-09-15", + "kind": "transparent_single_pair_pilot", + "date": "2026-09-15", + "baseline_commit": "90d84be225fe2751ed0583917e7391ff29245559", + "task_sizes": ["small", "medium", "large"], + "tasks": { + "small": { + "task_id": "notice-export-content-disposition-ascii-fallback", + "definition": "Content-Disposition fallback ASCII whitelist hardening", + "scope": ["notice_export_service", "service test"] + }, + "medium": { + "task_id": "notification-priority-dirty-input-degradation", + "definition": "Notification priority dirty-input stable degradation and enrich pure-function extraction", + "scope": ["service", "new test"] + }, + "large": { + "task_id": "export-lease-thread-safe-success-cooldown", + "definition": "Thread-safe export lease with success-commit cooldown, wired into three routes and covering failure rollback", + "scope": ["service", "2 routes", "tests"] + } + }, + "design": { + "arms": ["direct_codex", "delegated_deepseek"], + "repetitions_per_cell": 1, + "execution": "parallel", + "order_alternated": false, + "matches_queen_protocol": false, + "queen_protocol_requirement": "The queen protocol asks for at least three alternated repetitions per variant; this pilot must not be presented as that formal benchmark.", + "wall_time_used_for_token_reduction": false + }, + "unit_conventions": { + "codex_total": "input + output, where input is inclusive of cached_input", + "codex_proxy": "(input - cached_input) + output", + "worker_total": "input + cache_read + output + reasoning", + "worker_proxy": "input + output", + "reduction_percent": "1 - compared_proxy / direct_proxy_total, rounded to two decimals", + "wall_time_seconds": "observed duration from the recorded wall_time_source; never used in token reduction calculations" + }, + "direct_codex": { + "small": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 554251, + "cached_input": 508032, + "output": 3508, + "reasoning": 1034, + "total": 557759, + "acceptance": true, + "corrections": 0, + "wall_time_seconds": 108.921, + "wall_time_source": "Codex session first-to-last timestamp", + "controller_acceptance": { + "tests_total": 32, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + }, + "medium": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 812011, + "cached_input": 751872, + "output": 7019, + "reasoning": 2209, + "total": 819030, + "acceptance": true, + "corrections": 0, + "wall_time_seconds": 178.325, + "wall_time_source": "Codex session first-to-last timestamp", + "controller_acceptance": { + "tests_total": 51, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + }, + "large": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 1696112, + "cached_input": 1626240, + "output": 15444, + "reasoning": 4958, + "total": 1711556, + "acceptance": true, + "corrections": 0, + "wall_time_seconds": 379.631, + "wall_time_source": "Codex session first-to-last timestamp", + "controller_acceptance": { + "tests_service": 21, + "tests_api": 49, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + } + }, + "delegated_deepseek": { + "small": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 26939, + "cache_read": 333952, + "output": 3475, + "reasoning": 1842, + "total": 366208, + "cost_usd": 0.008232906, + "acceptance": true, + "corrections": 0, + "usage_complete": true, + "wall_time_seconds": 210.517, + "wall_time_source": "worker creation through controller verification", + "controller_acceptance": { + "tests_total": 41, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + }, + "medium": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 27079, + "cache_read": 554240, + "output": 8597, + "reasoning": 3316, + "total": 593232, + "cost_usd": 0.01287237, + "acceptance": true, + "corrections": 0, + "usage_complete": false, + "wall_time_seconds": 414.657, + "wall_time_source": "worker creation through controller verification", + "controller_acceptance": { + "tests_total": 120, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + }, + "large": { + "commit": "90d84be225fe2751ed0583917e7391ff29245559", + "input": 84304, + "cache_read": 5069568, + "output": 23944, + "reasoning": 18216, + "total": 5196032, + "cost_usd": 0.053150304, + "acceptance": true, + "corrections": 1, + "usage_complete": false, + "wall_time_seconds": 811.445, + "wall_time_source": "worker creation through controller verification", + "wall_time_includes_correction_rounds": 1, + "controller_acceptance": { + "tests_service": 25, + "tests_api": 53, + "ruff": "passed", + "format": "passed", + "diff": "passed" + } + } + }, + "controller_interval": { + "description": "Codex controller increment from launching the six executions through acceptance of both arms", + "input": 6358712, + "cached_input": 6287488, + "output": 18933, + "reasoning": 6389, + "total": 6377645, + "noncached_input": 71224, + "proxy": 90157, + "includes": ["native control-arm management", "review of both arms"], + "pure_delegation_bill": false + }, + "derived": { + "direct_proxy_total": 202201, + "worker_proxy_total": 174338, + "worker_proxy_reduction_vs_direct_percent": 13.78, + "controller_proxy_total": 90157, + "controller_proxy_reduction_vs_direct_percent": 55.41, + "direct_total_sum": 3088345, + "worker_total_sum": 6155472 + }, + "review_corrections": [ + { + "arm": "delegated_deepseek", + "task_size": "large", + "rounds": 1, + "finding": "Codex review found the first revision let a stale lease break a new lease and let empty results enter cooldown.", + "resolution": "The worker corrected both defects; the revised patch passed review and independent verification and was accepted." + } + ], + "limitations": [ + "One execution per size and arm; the arms ran in parallel and the order was not alternated, so this is not the repeated benchmark the queen protocol requests.", + "One repository, one task set, one model configuration and one date; results do not generalize and promise no fixed or billing-grade saving.", + "The controller interval covers launch of all six executions through acceptance of both arms, including native control-arm management and review of both arms, so it is not a pure delegation bill.", + "Worker usage for medium and large is marked usage_complete=false; those worker totals and costs may understate actual usage.", + "Worker total including cached reads (6155472) exceeds direct total (3088345), so combined tokens and latency can increase.", + "Both arms ran in parallel, so each wall time can include contention for the same machine and provider capacity; the durations are single observations, not a stable performance baseline.", + "This pilot is separate from the 2026-09-14 combined-token benchmark and must not be merged with it." + ] +} diff --git a/docs/task-routing-pilot-2026-09-15.md b/docs/task-routing-pilot-2026-09-15.md new file mode 100644 index 0000000..94ebb9c --- /dev/null +++ b/docs/task-routing-pilot-2026-09-15.md @@ -0,0 +1,83 @@ +# Task Routing Pilot — 2026-09-15 + +A transparent single-pair pilot of the task routing policy. It records one direct Codex execution and one delegated DeepSeek worker execution for each of a small, medium and large task, all from the same baseline commit `90d84be225fe2751ed0583917e7391ff29245559`. The machine-readable source is [task-routing-pilot-2026-09-15.json](./task-routing-pilot-2026-09-15.json); this page summarizes it and adds no new measurements. + +This is not a formal benchmark. There is exactly one execution per cell, both arms ran in parallel, and the order was not alternated. The queen protocol asks for at least three alternated repetitions per variant, so this pilot does not satisfy it. Do not generalize these numbers, promise a fixed saving, or merge them with the [2026-09-14 combined-token report](./token-routing-benchmark-2026-09-14.md). Combined tokens and latency can still increase. + +## Tasks + +| Size | Task ID | Definition | Scope | +|---|---|---|---| +| small | notice-export-content-disposition-ascii-fallback | Content-Disposition fallback ASCII whitelist hardening | notice_export_service, service test | +| medium | notification-priority-dirty-input-degradation | Notification priority dirty-input stable degradation and enrich pure-function extraction | service, new test | +| large | export-lease-thread-safe-success-cooldown | Thread-safe export lease with success-commit cooldown, wired into three routes and covering failure rollback | service, 2 routes, tests | + +The JSON records these IDs and definitions under `tasks`, matched to the three sizes. + +## Raw results + +### Direct Codex + +| Size | Input | Cached input | Output | Reasoning | Total | Acceptance | Corrections | +|---|---:|---:|---:|---:|---:|---|---:| +| small | 554251 | 508032 | 3508 | 1034 | 557759 | true | 0 | +| medium | 812011 | 751872 | 7019 | 2209 | 819030 | true | 0 | +| large | 1696112 | 1626240 | 15444 | 4958 | 1711556 | true | 0 | + +Direct total = input + output, where input includes cached input. These are model usage counters, not a billing statement. + +### Delegated DeepSeek worker + +| Size | Input | Cache read | Output | Reasoning | Total | Cost (USD) | Acceptance | Corrections | Usage complete | +|---|---:|---:|---:|---:|---:|---:|---|---:|---| +| small | 26939 | 333952 | 3475 | 1842 | 366208 | 0.008232906 | true | 0 | true | +| medium | 27079 | 554240 | 8597 | 3316 | 593232 | 0.01287237 | true | 0 | false | +| large | 84304 | 5069568 | 23944 | 18216 | 5196032 | 0.053150304 | true | 1 | false | + +Worker total = input + cache_read + output + reasoning. Medium and large are marked `usage_complete=false`; their totals and costs may understate actual usage. The provider-reported cost is retained as raw data and is not converted into a saving claim. + +### Codex controller interval + +From launching the six executions through accepting both arms, the controller increment was: input 6358712, cached input 6287488, output 18933, reasoning 6389, total 6377645, non-cached input 71224, proxy 90157. + +This interval covers the whole two-arm experiment, including managing the native control arm and reviewing both arms. It is not a pure delegation bill and must not be presented as one. + +### Wall time and independent acceptance + +| Size | Direct wall time (s) | Delegated wall time (s) | Direct acceptance | Delegated acceptance | +|---|---:|---:|---|---| +| small | 108.921 | 210.517 | 32 tests | 41 tests | +| medium | 178.325 | 414.657 | 51 tests | 120 tests | +| large | 379.631 | 811.445 | 21 service + 49 API tests | 25 service + 53 API tests | + +Wall time sources are recorded per execution in the JSON: direct durations span the Codex session's first-to-last timestamp; delegated durations run from worker creation through controller verification. The delegated large duration includes its one correction round. Independent controller verification passed for all six executions, with ruff, format and diff checks passing for each. Wall time values are single observations from arms that ran in parallel, so they can reflect contention for the same machine and provider capacity; they are not a stable performance baseline and are never used in token reduction calculations. + +## Derived comparisons + +Definitions: direct proxy = (input − cached input) + output; worker proxy = input + output; reduction = 1 − compared proxy / direct proxy total. + +| Quantity | Value | +|---|---:| +| Direct proxy total (three tasks) | 202201 | +| Worker proxy total (three tasks) | 174338 | +| Worker proxy reduction vs direct proxy total | 13.78% | +| Controller proxy | 90157 | +| Controller proxy reduction vs direct proxy total | 55.41% | +| Direct total sum | 3088345 | +| Worker total sum (cache included) | 6155472 | + +The 55.41% is a conservative controller-side comparison inside this pilot: the controller interval also absorbs native control-arm management and review of both arms, so it is neither a pure delegation bill nor a promised saving. Worker proxy was 13.78% below the direct proxy total, but worker total including cached reads was 6155472 against 3088345 direct, so combined tokens can increase. No fixed saving is claimed, here or elsewhere. + +## Corrections and failures + +All six executions were accepted. The three direct executions each needed 0 corrections, and the delegated small and medium executions were accepted with 0 corrections. The delegated large execution was rejected on first review: the first revision let a stale lease break a new lease and let empty results enter cooldown. The worker corrected both defects, and the revised patch passed review and independent verification. Nothing was discarded, and `usage_complete=false` for medium and large is retained as reported. + +## Limitations + +- One execution per size and arm, parallel, non-alternating. This is not the repeated formal benchmark required by the queen protocol and supports no statistical claim. +- One repository, one task set, one model configuration and one date. Nothing here generalizes, and no billing-grade conclusion follows. +- The controller interval includes native control-arm management and both-arm review, so 55.41% must not be treated as pure delegated savings. +- Worker usage completeness is false for two of three tasks, so the worker-side comparison may understate worker cost. +- Cache-inclusive worker total exceeded direct total, so combined tokens and latency may increase even when the worker proxy looks lower. +- Both arms ran in parallel, so each wall time can include contention for the same machine and provider capacity; the durations are single observations, not a stable performance baseline, and they feed no token reduction calculation. +- This pilot is independent of the earlier combined-token benchmark, and the two must not be mixed. diff --git a/plugins/deepseek-code-agent/README.md b/plugins/deepseek-code-agent/README.md index 7e53ce5..51d3a06 100644 --- a/plugins/deepseek-code-agent/README.md +++ b/plugins/deepseek-code-agent/README.md @@ -46,7 +46,11 @@ The live test makes a small request to the selected model in a temporary Git rep ## Routing expectation -Codex decides the design, interfaces, invariants and acceptance cases. A persistent worker implements that plan, self-tests and fixes routine failures. Codex reviews the scoped patch and gives one batch of feedback; after two unsuccessful review correction rounds it may take over, after confirming the worker and verification have stopped. Tiny edits can be handled directly. +Route by judgment, risk, and exploration/verification burden, not line counts. Small known edits that take a few minutes stay with Codex. Medium mechanical work with clear acceptance criteria prefers one persistent worker. Large or high-risk work (state machines, concurrency, permissions, transactions, migrations) may delegate the labor, but Codex must first decide the design, interfaces, data flows, invariants, failure semantics and acceptance examples, then review the actual diff and run independent verification; worker self-reports and passing self-tests are never acceptance. An explicit user request overrides these defaults. + +Codex decides the design, interfaces, invariants and acceptance cases. A persistent worker implements that plan, self-tests and fixes routine failures. Codex reviews the scoped patch and gives one batch of feedback; after two unsuccessful review correction rounds it may take over, after confirming the worker and verification have stopped. + +The repository's 2026-09-15 [task routing pilot](../../docs/task-routing-pilot-2026-09-15.md) records one direct/delegated pair per size from one commit. It is a single non-repeated pilot, not a benchmark; its results do not generalize, and combined tokens and latency can still increase. The link targets the source repository layout; a standalone plugin package does not include the `docs/` directory. `ds_spawn_agent` accepts an optional structured `task_spec`. `ds_verify_agent` runs its preselected commands and preserves full logs, binding results to a source snapshot. Verification can run beyond one MCP wait window; repeated calls observe the same job rather than executing checks again. Inspect remains read-only. Neither worker completion nor passing checks means Codex has accepted the patch. diff --git a/plugins/deepseek-code-agent/package.json b/plugins/deepseek-code-agent/package.json index 56064b6..f9fc2e4 100644 --- a/plugins/deepseek-code-agent/package.json +++ b/plugins/deepseek-code-agent/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "check": "node ./scripts/check-prereqs.mjs", - "test": "node --test ./tests/controller.test.mjs ./tests/mcp.test.mjs ./tests/workflow-core.test.mjs ./tests/verification.test.mjs", + "test": "node --test ./tests/controller.test.mjs ./tests/mcp.test.mjs ./tests/workflow-core.test.mjs ./tests/verification.test.mjs ./tests/routing-policy.test.mjs", "test:live": "node ./tests/live-smoke.mjs" }, "engines": { diff --git a/plugins/deepseek-code-agent/skills/deepseek-coding-delegation/SKILL.md b/plugins/deepseek-code-agent/skills/deepseek-coding-delegation/SKILL.md index 529fa08..9aae7e3 100644 --- a/plugins/deepseek-code-agent/skills/deepseek-coding-delegation/SKILL.md +++ b/plugins/deepseek-code-agent/skills/deepseek-coding-delegation/SKILL.md @@ -1,6 +1,6 @@ --- name: deepseek-coding-delegation -description: Keep the queen (Codex) on quality-critical engineering decisions while a DeepSeek or GLM worker handles bounded discovery, implementation, tests, routine fixes and evidence preparation. Use for coding tasks, coding goals, supporting engineering work, and selecting, querying or resetting this plugin’s worker model defaults; handle tiny known edits directly. +description: Keep the queen (Codex) on quality-critical engineering decisions while a DeepSeek or GLM worker handles bounded discovery, implementation, tests, routine fixes and evidence preparation. Use for coding tasks, coding goals, supporting engineering work, and selecting, querying or resetting this plugin’s worker model defaults; route small known edits directly, prefer delegation for clear medium work, and keep design, diff review and independent verification for large or high-risk work. --- # DeepSeek Coding Delegation @@ -25,7 +25,12 @@ Use the queen where its judgment most affects quality: intent, acceptance criter ## Route and delegate -- Handle a tiny, understood edit directly when describing it and reviewing it would take more work than doing it. Hand off a coherent behavior or investigation with acceptance criteria, rather than one function or tool command per turn. +Choose the route from how much judgment, risk, and exploration/verification burden the task carries — not from line counts or file counts. This is a decision guide for Codex’s situational judgment, not an automatic classifier, and an explicit user request to delegate or to stay direct overrides it. + +- **Small (direct):** a known, clearly understood change that takes a few minutes and needs neither broad discovery nor risky design. Handle it in Codex directly; briefing and reviewing a worker would cost more than the edit. Report the route as direct. +- **Medium (prefer delegation):** mechanical implementation, or a larger amount of testing, with clear and objective acceptance criteria. Prefer one persistent worker, carrying the design decisions and exact checks in `task_spec`. +- **Large or high-risk (delegated labor, queen-owned design):** state machines, concurrency, permissions, transactions, migrations, or changes where a wrong invariant, interface or failure semantic is expensive. The labor may be delegated, but Codex must first decide the design, interface, data flow, invariants, failure semantics and acceptance examples; then Codex reviews the actual diff and runs independent verification. Never accept a worker’s self-report or passing self-tests as acceptance. +- Hand off a coherent behavior or investigation with acceptance criteria, rather than one function or tool command per turn. - Prefer one persistent worker. Avoid additional queen-model workers for ordinary implementation. Reuse the worker for related corrections; use a fresh session for unrelated tasks. - Before spawning, read applicable repository instructions and the critical call chain. Codex must decide the interface, data flow, invariants, failure behavior and acceptance examples that determine correctness. Leave routine implementation and broader discovery to the worker. Do not substitute a vague task restatement for a design, or prewrite every implementation detail. - Send a `task_spec` with the design decisions, observable acceptance criteria and exact checks selected by Codex. Keep it grounded in existing contracts; do not add speculative constraints. For an unfamiliar subsystem, first request a bounded discovery report from the same worker, then resolve the consequential decisions before implementation. diff --git a/plugins/deepseek-code-agent/tests/routing-policy.test.mjs b/plugins/deepseek-code-agent/tests/routing-policy.test.mjs new file mode 100644 index 0000000..72cdd30 --- /dev/null +++ b/plugins/deepseek-code-agent/tests/routing-policy.test.mjs @@ -0,0 +1,239 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; + +const PILOT_URL = new URL("../../../docs/task-routing-pilot-2026-09-15.json", import.meta.url); +const BASELINE_COMMIT = "90d84be225fe2751ed0583917e7391ff29245559"; +const SIZES = ["small", "medium", "large"]; +const EXPECTED = { + direct_proxy_total: 202201, + worker_proxy_total: 174338, + controller_proxy_total: 90157, + worker_proxy_reduction_percent: 13.78, + controller_proxy_reduction_percent: 55.41, + direct_total_sum: 3088345, + worker_total_sum: 6155472, +}; + +const pilot = JSON.parse(readFileSync(PILOT_URL, "utf8")); + +function round2(value) { + return Math.round(value * 100) / 100; +} + +function assertNumber(value, label) { + assert.equal(typeof value, "number", `${label} must be a number`); + assert.ok(Number.isFinite(value), `${label} must be finite`); +} + +test("pilot JSON declares one non-alternating pair per size", () => { + assert.equal(pilot.schema_version, 1); + assert.equal(pilot.baseline_commit, BASELINE_COMMIT); + assert.equal(pilot.date, "2026-09-15"); + assert.deepEqual(pilot.task_sizes, SIZES); + assert.deepEqual(pilot.design.arms, ["direct_codex", "delegated_deepseek"]); + assert.equal(pilot.design.repetitions_per_cell, 1); + assert.equal(pilot.design.execution, "parallel"); + assert.equal(pilot.design.order_alternated, false); + assert.equal(pilot.design.matches_queen_protocol, false); + assert.equal(pilot.design.wall_time_used_for_token_reduction, false); +}); + +test("every execution is pinned to the same baseline commit", () => { + for (const size of SIZES) { + assert.equal(pilot.direct_codex[size].commit, BASELINE_COMMIT, `direct ${size} commit`); + assert.equal( + pilot.delegated_deepseek[size].commit, + BASELINE_COMMIT, + `delegated ${size} commit`, + ); + } +}); + +test("raw totals recompute from reported counters", () => { + for (const size of SIZES) { + const direct = pilot.direct_codex[size]; + for (const field of ["input", "cached_input", "output", "reasoning", "total"]) { + assertNumber(direct[field], `direct_codex.${size}.${field}`); + } + assert.equal(direct.total, direct.input + direct.output, `direct ${size} total`); + assert.equal(direct.acceptance, true, `direct ${size} acceptance`); + assertNumber(direct.corrections, `direct_codex.${size}.corrections`); + assert.ok(!("cost_usd" in direct), "unknown direct cost must not be fabricated as a number"); + + const worker = pilot.delegated_deepseek[size]; + for (const field of ["input", "cache_read", "output", "reasoning", "total", "cost_usd"]) { + assertNumber(worker[field], `delegated_deepseek.${size}.${field}`); + } + assert.equal( + worker.total, + worker.input + worker.cache_read + worker.output + worker.reasoning, + `worker ${size} total`, + ); + assert.equal(worker.acceptance, true, `delegated ${size} acceptance`); + assertNumber(worker.corrections, `delegated_deepseek.${size}.corrections`); + assert.equal(typeof worker.usage_complete, "boolean", `worker ${size} usage_complete`); + } + + const controller = pilot.controller_interval; + for (const field of ["input", "cached_input", "output", "reasoning", "total", "proxy"]) { + assertNumber(controller[field], `controller_interval.${field}`); + } + assert.equal(controller.total, controller.input + controller.output, "controller total"); + assert.equal(controller.noncached_input, controller.input - controller.cached_input); + assert.equal(controller.proxy, controller.noncached_input + controller.output); + assert.equal(controller.pure_delegation_bill, false); +}); + +test("proxy aggregates recompute from raw fields", () => { + const directProxy = SIZES.reduce((sum, size) => { + const record = pilot.direct_codex[size]; + return sum + (record.input - record.cached_input) + record.output; + }, 0); + const workerProxy = SIZES.reduce((sum, size) => { + const record = pilot.delegated_deepseek[size]; + return sum + record.input + record.output; + }, 0); + + assert.equal(directProxy, EXPECTED.direct_proxy_total); + assert.equal(workerProxy, EXPECTED.worker_proxy_total); + assert.equal(pilot.controller_interval.proxy, EXPECTED.controller_proxy_total); + + assert.equal(pilot.derived.direct_proxy_total, EXPECTED.direct_proxy_total); + assert.equal(pilot.derived.worker_proxy_total, EXPECTED.worker_proxy_total); + assert.equal(pilot.derived.controller_proxy_total, EXPECTED.controller_proxy_total); +}); + +test("reduction percentages recompute to the published values", () => { + const directProxy = pilot.derived.direct_proxy_total; + const workerReduction = round2( + ((directProxy - pilot.derived.worker_proxy_total) / directProxy) * 100, + ); + const controllerReduction = round2( + ((directProxy - pilot.derived.controller_proxy_total) / directProxy) * 100, + ); + + assert.equal(workerReduction, EXPECTED.worker_proxy_reduction_percent); + assert.equal(controllerReduction, EXPECTED.controller_proxy_reduction_percent); + assert.equal( + pilot.derived.worker_proxy_reduction_vs_direct_percent, + EXPECTED.worker_proxy_reduction_percent, + ); + assert.equal( + pilot.derived.controller_proxy_reduction_vs_direct_percent, + EXPECTED.controller_proxy_reduction_percent, + ); +}); + +test("published totals, acceptance and corrections are preserved", () => { + const directTotal = SIZES.reduce((sum, size) => sum + pilot.direct_codex[size].total, 0); + const workerTotal = SIZES.reduce( + (sum, size) => sum + pilot.delegated_deepseek[size].total, + 0, + ); + assert.equal(directTotal, EXPECTED.direct_total_sum); + assert.equal(workerTotal, EXPECTED.worker_total_sum); + assert.equal(pilot.derived.direct_total_sum, EXPECTED.direct_total_sum); + assert.equal(pilot.derived.worker_total_sum, EXPECTED.worker_total_sum); + assert.ok(workerTotal > directTotal, "cache-inclusive worker total must remain visible"); + + assert.deepEqual( + SIZES.map((size) => pilot.direct_codex[size].corrections), + [0, 0, 0], + ); + assert.deepEqual( + SIZES.map((size) => pilot.delegated_deepseek[size].corrections), + [0, 0, 1], + ); + assert.equal(pilot.delegated_deepseek.small.usage_complete, true); + assert.equal(pilot.delegated_deepseek.medium.usage_complete, false); + assert.equal(pilot.delegated_deepseek.large.usage_complete, false); + + assert.equal(pilot.review_corrections.length, 1); + assert.equal(pilot.review_corrections[0].task_size, "large"); + assert.equal(pilot.review_corrections[0].rounds, 1); + assert.match(pilot.review_corrections[0].finding, /stale lease/); + assert.match(pilot.review_corrections[0].finding, /cooldown/); +}); + +test("task definitions cover all three sizes with stable non-empty IDs", () => { + const ids = new Set(); + for (const size of SIZES) { + const task = pilot.tasks[size]; + assert.ok(task, `tasks.${size} is required`); + assert.equal(typeof task.task_id, "string", `tasks.${size}.task_id`); + assert.ok(task.task_id.trim().length > 0, `tasks.${size}.task_id must not be empty`); + assert.ok(!ids.has(task.task_id), `task_id ${task.task_id} must be unique`); + ids.add(task.task_id); + assert.equal(typeof task.definition, "string", `tasks.${size}.definition`); + assert.ok(task.definition.trim().length > 0, `tasks.${size}.definition must not be empty`); + assert.ok(Array.isArray(task.scope) && task.scope.length > 0, `tasks.${size}.scope`); + for (const entry of task.scope) { + assert.equal(typeof entry, "string", `tasks.${size}.scope entry`); + assert.ok(entry.trim().length > 0, `tasks.${size}.scope entry must not be empty`); + } + } +}); + +test("wall times are positive observations with recorded sources", () => { + const durations = []; + for (const size of SIZES) { + for (const arm of ["direct_codex", "delegated_deepseek"]) { + const record = pilot[arm][size]; + assertNumber(record.wall_time_seconds, `${arm}.${size}.wall_time_seconds`); + assert.ok(record.wall_time_seconds > 0, `${arm}.${size}.wall_time_seconds must be positive`); + assert.equal(typeof record.wall_time_source, "string", `${arm}.${size}.wall_time_source`); + assert.ok( + record.wall_time_source.trim().length > 0, + `${arm}.${size}.wall_time_source must not be empty`, + ); + durations.push({ arm, size, seconds: record.wall_time_seconds }); + } + } + for (const size of SIZES) { + assert.ok( + pilot.delegated_deepseek[size].wall_time_seconds > + pilot.direct_codex[size].wall_time_seconds, + `delegated ${size} wall time should exceed direct`, + ); + } + const longest = durations.reduce((max, entry) => (entry.seconds > max.seconds ? entry : max)); + assert.equal(longest.arm, "delegated_deepseek"); + assert.equal(longest.size, "large"); +}); + +test("large delegated wall time includes its single correction round", () => { + const large = pilot.delegated_deepseek.large; + assert.equal(large.corrections, 1); + assert.equal(large.wall_time_includes_correction_rounds, 1); + assert.ok(large.wall_time_seconds > pilot.delegated_deepseek.medium.wall_time_seconds); +}); + +test("controller acceptance counts and quality checks are recorded", () => { + const expected = { + direct_codex: { + small: { tests_total: 32 }, + medium: { tests_total: 51 }, + large: { tests_service: 21, tests_api: 49 }, + }, + delegated_deepseek: { + small: { tests_total: 41 }, + medium: { tests_total: 120 }, + large: { tests_service: 25, tests_api: 53 }, + }, + }; + for (const arm of Object.keys(expected)) { + for (const size of SIZES) { + const acceptance = pilot[arm][size].controller_acceptance; + assert.ok(acceptance, `${arm}.${size}.controller_acceptance is required`); + assert.equal(acceptance.ruff, "passed", `${arm}.${size} ruff`); + assert.equal(acceptance.format, "passed", `${arm}.${size} format`); + assert.equal(acceptance.diff, "passed", `${arm}.${size} diff`); + for (const [field, value] of Object.entries(expected[arm][size])) { + assertNumber(acceptance[field], `${arm}.${size}.${field}`); + assert.ok(acceptance[field] > 0, `${arm}.${size}.${field} must be positive`); + assert.equal(acceptance[field], value, `${arm}.${size}.${field}`); + } + } + } +});