feat(bench): slim the report, shrink the fixtures, rewrite the manual (#379)

bench/cli.mjs

@@ -69,7 +69,7 @@ export function abUsage() {
     '                            [--base <label>] [--cand <label>] [--show] [runner flags...]',
     '',
     'Runs the baseline side, the candidate side, then the comparison, all inside ONE',
-    'bench/runs/benchmark-test-NN group: the number is allocated once here because a',
+    'bench/runs/benchmark-YYYY-MM-DD-NN group: the name is allocated once here because a',
     'per-process default would put each side in a different group and leave nothing to pair.',
     'Flag meanings are in `--help`; the group is created by the runner (P7/P8) and the',
     'comparison by P11.',
@@ -114,15 +114,16 @@ export function parseAbArgs(argv) {
  * sharing the one group folder allocated here. Pure: it reserves the name but writes
  * nothing, so `--show` and a real run report the same plan.
  * @param {string[]} argv - the `ab` arguments, parsed by {@link parseAbArgs}.
- * @param {{runsDir?: string, repoRoot?: string}} [options] - where group folders live and
- *   the checkout the three commands run in; injectable so a suite allocates nothing real.
+ * @param {{runsDir?: string, repoRoot?: string, now?: Date}} [options] - where group folders live,
+ *   the checkout the three commands run in, and the clock the auto folder name stamps
+ *   (injectable so a suite allocates nothing real and asserts exact names).
  * @returns {{base: string, cand: string, tasks: string[], repeat: string|null, folder: string, show: boolean, extra: string[], help: boolean, allocated: boolean, commands: {label: string, target: string, args: string[], argv: string[]}[], repoRoot: string}} the plan.
  */
 export function planAb(argv, options = {}) {
   const { runsDir = RUNS_DIR, repoRoot = REPO_ROOT } = options
   const parsed = parseAbArgs(argv)
   const allocated = parsed.folder === null
-  const folder = parsed.folder ?? nextFolderName(runsDir)
+  const folder = parsed.folder ?? nextFolderName(runsDir, options.now)
   const shared = parsed.tasks.flatMap((task) => ['--task', task])
   if (parsed.repeat !== null) shared.push('--repeat', parsed.repeat)
   shared.push('--folder', folder, ...parsed.extra)

bench/lib/compare-core.mjs

@@ -228,6 +228,10 @@ export function aggregateTask(metricsList = []) {
   const steps = numeric((metrics) => metrics?.steps ?? 0)
   const toolCalls = numeric((metrics) => metrics?.toolCalls ?? 0)
   const toolErrors = numeric((metrics) => metrics?.toolErrors ?? 0)
+  // Wall-clock per repeat, aggregated like steps. A failed row records 0 ms (it never
+  // ran), which drags the median down exactly the way a 0-steps row does — the pass
+  // columns are what tell the reader which rows actually measured something.
+  const durations = numeric((metrics) => metrics?.durationMs ?? 0)
   // The prompt shape: the first request (no history yet — system prompt + tool
   // schemas + task text) is the per-request fixed cost, and the per-step average
   // says how much context each subsequent request re-sent. Older archives predate
@@ -282,6 +286,8 @@ export function aggregateTask(metricsList = []) {
     stepsMin: minValue(steps),
     toolCallsMedian: median(toolCalls),
     toolCallsMin: minValue(toolCalls),
+    durationMsMedian: median(durations),
+    durationMsMin: minValue(durations),
     toolCallsByName,
     toolErrorsMedian: median(toolErrors),
     toolErrorsMin: minValue(toolErrors),
@@ -368,6 +374,7 @@ function statisticKeys(aggregate) {
     toolsKey: useMin ? 'toolCallsMin' : 'toolCallsMedian',
     errorsKey: useMin ? 'toolErrorsMin' : 'toolErrorsMedian',
     firstPromptKey: useMin ? 'firstRequestPromptMin' : 'firstRequestPromptMedian',
+    durationKey: useMin ? 'durationMsMin' : 'durationMsMedian',
     promptTotalKey: useMin ? 'promptTotalMin' : 'promptTotalMedian',
     inputKey: useMin ? 'inputMin' : 'inputMedian',
     outputKey: useMin ? 'outputMin' : 'outputMedian',
@@ -745,6 +752,7 @@ export function compareSummaries(base, cand, options = {}) {
         stepsPct: percentChange(baseSide[keys.stepsKey], candSide[keys.stepsKey]),
         toolCallsPct: percentChange(baseSide[keys.toolsKey], candSide[keys.toolsKey]),
         toolErrorsPct: percentChange(baseSide[keys.errorsKey], candSide[keys.errorsKey]),
+        durationPct: percentChange(baseSide[keys.durationKey], candSide[keys.durationKey]),
         passRateAbs: candSide.passRate - baseSide.passRate,
       },
     }
@@ -874,6 +882,8 @@ export function compareSummaries(base, cand, options = {}) {
       candToolCalls: sumStat(tasks, (task) => task.cand[keys.toolsKey]),
       baseToolErrors: sumStat(tasks, (task) => task.base[keys.errorsKey]),
       candToolErrors: sumStat(tasks, (task) => task.cand[keys.errorsKey]),
+      baseDurationMs: sumStat(tasks, (task) => task.base[keys.durationKey]),
+      candDurationMs: sumStat(tasks, (task) => task.cand[keys.durationKey]),
       basePasses,
       baseRuns: baseRunsCount,
       candPasses,

bench/lib/next-folder.mjs

@@ -1,17 +1,18 @@
 #!/usr/bin/env node
 /**
- * Allocate the next free `benchmark-test-NN` group directory under `bench/runs/`.
+ * Allocate the next free date-stamped group directory under `bench/runs/`.
  *
- * `ab` allocates this **exactly once per sweep** and hands the answer to both sides
- * and to the comparison: letting each process default on its own would put the
- * baseline in `benchmark-test-01` and the candidate in `benchmark-test-02` (that
- * group exists by then), and the comparison would have nothing to pair.
+ * The default name is `benchmark-YYYY-MM-DD-NN` — the local calendar date, numbered
+ * from `-00` — and `ab` allocates it **exactly once per sweep**, handing the answer
+ * to both sides and to the comparison: letting each process default on its own would
+ * put the baseline and the candidate in two different groups, and the comparison
+ * would have nothing to pair.
  *
  * The scan is read-only and non-exclusive on purpose: two sweeps started in the same
- * instant can both see `benchmark-test-01` free, and the failure mode is two runs
- * landing in one group — visible in the report, never lost data. A number is skipped
- * when *any* entry has that name, so a leftover empty directory from a failed sweep
- * is skipped rather than reused.
+ * instant can both see `-00` free, and the failure mode is two runs landing in one
+ * group — visible in the report, never lost data. A number is skipped when *any* entry
+ * has that name, so a leftover empty directory from a failed sweep is skipped rather
+ * than reused.
  *
  * @module dsh-ops/bench/next-folder
  */
@@ -24,10 +25,23 @@ import { BENCH_DIR } from './env.mjs'
 /** Where `run.json` archives land, one directory per sweep. */
 export const RUNS_DIR = path.join(BENCH_DIR, 'runs')
 /** The group prefix both the CLI help text and this scan agree on. */
-export const FOLDER_PREFIX = 'benchmark-test-'
-/** Zero-padded to two digits; numbers past 99 simply grow (`benchmark-test-100`). */
+export const FOLDER_PREFIX = 'benchmark-'
+/** Zero-padded to two digits; numbers past 99 simply grow (`benchmark-2026-10-24-100`). */
 const WIDTH = 2
 
+/**
+ * The local calendar date of `now` as `YYYY-MM-DD`.
+ * @param {Date} [now] - the instant to stamp (default the current time).
+ * @returns {string} the zero-padded local date.
+ */
+export function dayStamp(now = new Date()) {
+  return [
+    String(now.getFullYear()).padStart(4, '0'),
+    String(now.getMonth() + 1).padStart(2, '0'),
+    String(now.getDate()).padStart(2, '0'),
+  ].join('-')
+}
+
 /**
  * The group names already present under the runs root, lower-cased so the answer
  * holds on a case-insensitive filesystem (Windows) as well as a case-sensitive one.
@@ -45,14 +59,17 @@ export function takenFolders(runsDir) {
 }
 
 /**
- * The first `benchmark-test-NN` nobody is using yet.
+ * The first `benchmark-YYYY-MM-DD-NN` nobody is using yet.
  * @param {string} [runsDir] - the runs root to scan (default {@link RUNS_DIR}).
- * @returns {string} the free group name, zero-padded to two digits.
+ * @param {Date} [now] - the instant whose local date names the group (default now);
+ *   injectable so a suite can assert exact names instead of patterns.
+ * @returns {string} the free group name, numbered from `-00` on that date.
  */
-export function nextFolderName(runsDir = RUNS_DIR) {
+export function nextFolderName(runsDir = RUNS_DIR, now = new Date()) {
   const taken = takenFolders(runsDir)
-  for (let number = 1; ; number += 1) {
-    const name = `${FOLDER_PREFIX}${String(number).padStart(WIDTH, '0')}`
+  const prefix = `${FOLDER_PREFIX}${dayStamp(now)}-`
+  for (let number = 0; ; number += 1) {
+    const name = `${prefix}${String(number).padStart(WIDTH, '0')}`
     if (!taken.has(name.toLowerCase())) return name
   }
 }

bench/llm.patch.example.yml

@@ -1,44 +1,20 @@
-# dsh-ops benchmark —— LLM 路由 overlay 模板
-#
-# 用法（详见 bench/PLAN.md §8、bench/README.md）：
-#   1) 复制本文件为 bench/llm.patch.yml（已在 .gitignore，不进仓库）；
-#   2) 填写你的 OpenAI 兼容服务器与模型 id（下面所有 <...> 占位符）；
-#   3) 设置 apiKeyEnv 指向的环境变量（如 BENCH_SERVER_API_KEY），
-#      密钥只放环境变量或 $DSH_HOME/.credentials.yaml，**绝不写进本文件**；
-#   4) node bench/run.mjs 会自动把它以 --patch 传给每次 dsh 调用
-#      （含 dump-config 存档）→ bench-base / bench-ops 两侧
-#      provider/model 一致性由构造保证（PLAN §2.7）；
-#      换别的 overlay 用 --llm-patch <path>；确实不想用 overlay（两侧各跑
-#      各自 profile 自带的路由）用 --no-llm-patch，sweep 会把"两侧不共享
-#      任何路由"这句话说出口。文件不存在又没给 --no-llm-patch 时，
-#      runner 在 spawn 之前就拒绝（退 2），不会静默换一条路由开跑。
-#
-# 注意：
-#   - 路由名 bench-server 可自定，但 llm-pi-ai 与 agent-default-model
-#     两处必须一致；
-#   - api 按服务器协议选择，自托管（vLLM / sglang / llama.cpp 的
-#     OpenAI 兼容端点）用 openai-completions；
-#   - 服务器必须在流式响应里回报 usage（stream_options.include_usage），
-#     否则 harness 拒绝给出 token 结论（PLAN §2.6）；
-#   - contextWindow 要按模型真实窗口填：现有语料里最长的题峰值 prompt 约 2 万 token，
-#     这个值不影响它们；但填小了仍会让 compaction-basic（0.8 × contextWindow）
-#     提前触发压缩，想跑长上下文任务时尤其要注意；
-#   - patch 语义是"整段替换 config"：llm-pi-ai 的 config 只有 providers
-#     一个字段、agent-default-model 只有 provider/model，直接照抄结构即可。
+- insert:
+    - id: credentials-local
+      name: "@deepseek-ai/dsh-credentials-local"
 
 - id: llm-pi-ai
   config:
     providers:
       bench-server:
-        baseURL: https://<你的服务器>/v1
-        apiKeyEnv: BENCH_SERVER_API_KEY
+        baseURL: https://<your-server>/v1
+        apiKeyEnv: DSH_API_KEY_ENV_NAME
         api: openai-completions
         models:
-          - id: <模型id>
+          - id: <model-id>
             contextWindow: 262144
-            maxTokens: 16384
+            maxTokens: 32768
 
 - id: agent-default-model
   config:
     provider: bench-server
-    model: <模型id>
+    model: <model-id>