From a4b1e2f08b0e2fbad23e5d837f200b55a76ef786 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:49:25 -0600 Subject: [PATCH 1/7] test(promises): run the fixture scripts inside the Claude Code sandbox The kiln check.sh scripts exit 5 with NO TESTS RAN under the sandbox, and relay's old_callers.sh prints 0 instead of 3, because the read allowlist denies /usr/bin/dirname. This test fails until the fixtures stop calling dirname. --- evals/promises/test_claude_code.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/evals/promises/test_claude_code.py b/evals/promises/test_claude_code.py index 6ec43426..e4eff635 100644 --- a/evals/promises/test_claude_code.py +++ b/evals/promises/test_claude_code.py @@ -1057,5 +1057,24 @@ def test_prepared_runtime_rejects_a_different_supported_git_digest(self): claude_code.child_env(run) +@measured_host +class NativeFixtureScripts(unittest.TestCase): + def test_fixture_scripts_report_the_same_result_inside_the_sandbox(self): + scripts = [("kiln", ".", (f"{{project}}/packages/{p}/check.sh",)) + for p in ("ingest", "shape", "render", "publish")] + scripts += [("kiln", "packages/render", ("sh", "check.sh")), ("relay", ".", ("sh", "tools/old_callers.sh"))] + for fixture, cwd, argv in scripts: + with self.subTest(fixture=fixture, cwd=cwd, argv=argv), tempfile.TemporaryDirectory() as tmp: + run = live.Run(Path(tmp).resolve(), "claude-code", {"fixture": fixture}, "0" * 40, 60) + shutil.copytree(live.FIXTURES / fixture, run.project, copy_function=shutil.copy) + state = claude_code._bind_paths(run, claude_code._host_runtime()) + command = [arg.format(project=run.project) for arg in argv] + results = [subprocess.run(prefix + command, cwd=run.project / cwd, env=claude_code.child_env(run), + capture_output=True, text=True, timeout=60) + for prefix in ([], ["/usr/bin/sandbox-exec", "-p", claude_code._policy(state, 0)])] + plain, confined = ((r.returncode, r.stdout) for r in results) + self.assertEqual(confined, plain) + + if __name__ == "__main__": unittest.main() From eff23bfd35f00de45e7a9befd2aa5fc0faec6974 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:57:05 -0600 Subject: [PATCH 2/7] fix(promises): stop the fixture scripts calling dirname The Claude Code sandbox's read allowlist denies /usr/bin/dirname, so every swarm-run worker's check.sh ran from the checkout root and reported BLOCKED. The scripts now strip the file name from $0 in the shell, and keep the current directory when $0 has no slash, as dirname did. --- evals/promises/fixtures/kiln/packages/ingest/check.sh | 2 +- evals/promises/fixtures/kiln/packages/publish/check.sh | 2 +- evals/promises/fixtures/kiln/packages/render/check.sh | 2 +- evals/promises/fixtures/kiln/packages/shape/check.sh | 2 +- evals/promises/fixtures/relay/tools/old_callers.sh | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/evals/promises/fixtures/kiln/packages/ingest/check.sh b/evals/promises/fixtures/kiln/packages/ingest/check.sh index 56eaf59e..311667d8 100755 --- a/evals/promises/fixtures/kiln/packages/ingest/check.sh +++ b/evals/promises/fixtures/kiln/packages/ingest/check.sh @@ -1,4 +1,4 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" +case $0 in */*) cd "${0%/*}" ;; esac python3 -m unittest -q diff --git a/evals/promises/fixtures/kiln/packages/publish/check.sh b/evals/promises/fixtures/kiln/packages/publish/check.sh index 56eaf59e..311667d8 100755 --- a/evals/promises/fixtures/kiln/packages/publish/check.sh +++ b/evals/promises/fixtures/kiln/packages/publish/check.sh @@ -1,4 +1,4 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" +case $0 in */*) cd "${0%/*}" ;; esac python3 -m unittest -q diff --git a/evals/promises/fixtures/kiln/packages/render/check.sh b/evals/promises/fixtures/kiln/packages/render/check.sh index 56eaf59e..311667d8 100755 --- a/evals/promises/fixtures/kiln/packages/render/check.sh +++ b/evals/promises/fixtures/kiln/packages/render/check.sh @@ -1,4 +1,4 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" +case $0 in */*) cd "${0%/*}" ;; esac python3 -m unittest -q diff --git a/evals/promises/fixtures/kiln/packages/shape/check.sh b/evals/promises/fixtures/kiln/packages/shape/check.sh index 56eaf59e..311667d8 100755 --- a/evals/promises/fixtures/kiln/packages/shape/check.sh +++ b/evals/promises/fixtures/kiln/packages/shape/check.sh @@ -1,4 +1,4 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" +case $0 in */*) cd "${0%/*}" ;; esac python3 -m unittest -q diff --git a/evals/promises/fixtures/relay/tools/old_callers.sh b/evals/promises/fixtures/relay/tools/old_callers.sh index 02abfeaf..5b6abc07 100755 --- a/evals/promises/fixtures/relay/tools/old_callers.sh +++ b/evals/promises/fixtures/relay/tools/old_callers.sh @@ -1,4 +1,5 @@ #!/bin/sh set -eu -cd "$(dirname "$0")/.." +case $0 in */*) cd "${0%/*}" ;; esac +cd .. grep -rlE 'parse_legacy\(' relay --include='*.py' | grep -v 'oldparse\.py' | wc -l | tr -d ' ' From 36dec610c77af26c09fa3eaf1a1ddeef8aa91450 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:58:19 -0600 Subject: [PATCH 3/7] docs(promises): apply the PR 310 verifier notes to the 2026-10-24 receipt Add the open-pr routing promise to the 7a9a3ddb baseline, say why the 2026-10-03 matrix rows stay the swarm baseline, and name the three opening-a-pr promises bound only to deferred cases. --- evals/promises/results/2026-10-24-claude-code-rerun.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/promises/results/2026-10-24-claude-code-rerun.md b/evals/promises/results/2026-10-24-claude-code-rerun.md index f62c8731..bad3bb77 100644 --- a/evals/promises/results/2026-10-24-claude-code-rerun.md +++ b/evals/promises/results/2026-10-24-claude-code-rerun.md @@ -10,10 +10,10 @@ The [eb4db2b receipt](2026-10-22-sync-eb4db2b.md) and the [e73df47 receipt](2026 - `skills/poteto-help/SKILL.md` (#300, #297, #304). It owns `poteto-help-finds-need-and-routes`. - `skills/setup-pstack/scripts/skill-listing.py` (#300, #297, #304). It owns `deslop-hands-unsettled-comments-to-no-comments`, `name-only-skills-stay-unloaded-unnamed`, `routed-skill-loads-after-compaction`, and `routed-skill-loads-when-named`. -- `skills/poteto-mode/playbooks/opening-a-pr.md` (#300). It owns `open-pr-prompt-routes-to-opening-a-pr`, bound to `route-open-pr`, and three promises bound only to deferred cases. +- `skills/poteto-mode/playbooks/opening-a-pr.md` (#300). It owns `open-pr-prompt-routes-to-opening-a-pr`, bound to `route-open-pr`, and three promises bound only to deferred cases: `opening-a-pr-worktree-small-commits-link` (`open-pr-run`), `opening-a-pr-gated-publish-never-bypasses` (`open-pr-gated-run`), and `worktree-request-honored` (`worktree-isolation-run`). - `skills/swarm/SKILL.md` (#304). It owns `swarm-fans-out-and-aggregates`, `swarm-returns-one-report-with-gaps`, `swarm-workers-report-pass-issues-blocked`, and `swarm-splits-verification-by-feature-map`. -`swarm-splits-verification-by-feature-map` is bound only to the deferred `verify-skill-run` case, so it did not run. +`swarm-splits-verification-by-feature-map` is bound only to the deferred `verify-skill-run` case. It and the three deferred-only `opening-a-pr.md` promises did not run. The eight bound cases ran once each on Claude Code with `evals/promises/live.py run`, in four batches: `poteto-help-run`, `poteto-help-unclear-run`, `name-only-unnamed-request`, `name-only-route-figure-it-out`, `name-only-route-after-compact`, `deslop-handoff`, `swarm-run`, and `route-open-pr`. `route-open-pr` ran after review found its owner missing from the first draft. Two turns hit the 300-second cap and exited -9: `name-only-route-figure-it-out` turn 0 and `name-only-route-after-compact` turn 2. Both traces record `x_timed_out: true` and `x_evidence_complete: false`. Their PASS verdicts stand, because `figure-it-out` loaded at sequence 8 in both, before the cap. The `swarm-run` trace is also incomplete evidence, because its workers ran in the background. No verdict depends on that field. `live.py report` rendered this matrix: @@ -42,7 +42,7 @@ The eight bound cases ran once each on Claude Code with `evals/promises/live.py The newest earlier receipt is [the e73df47 sync receipt](2026-10-24-sync-e73df47.md). Its Claude Code column is "not run" throughout, as is the column in the receipt it names as its baseline, [the eb4db2b sync receipt](2026-10-22-sync-eb4db2b.md). So the newest Claude Code evidence comes from the receipts those two name in their Comparison baseline sections: -- [The 7a9a3ddb post-sync matrix](2026-10-20-sync-7a9a3ddb.md), the baseline the eb4db2b receipt names, has Claude Code PASS for `deslop-hands-unsettled-comments-to-no-comments`, `name-only-skills-stay-unloaded-unnamed`, `routed-skill-loads-after-compaction`, and `routed-skill-loads-when-named`, and PASS (2) for `poteto-help-finds-need-and-routes`. This run matches all five. -- [2026-10-03](2026-10-03.md), the swarm baseline the e73df47 receipt names, has Claude Code PASS for all three swarm promises. This run matches on `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`. `swarm-returns-one-report-with-gaps` moves from PASS to INCONCLUSIVE, an oracle referral to the judge and not a FAIL. Per PORTING step 8, read 2026-10-03's "Fixed in this stack" rows as the baseline. That screen also predates the Claude Code sandbox (#138, 2026-10-05), so its `check.sh` scripts actually ran, while here they could not. The hand reading above finds the gap reported. +- [The 7a9a3ddb post-sync matrix](2026-10-20-sync-7a9a3ddb.md), the baseline the eb4db2b receipt names, has Claude Code PASS for `deslop-hands-unsettled-comments-to-no-comments`, `name-only-skills-stay-unloaded-unnamed`, `open-pr-prompt-routes-to-opening-a-pr`, `routed-skill-loads-after-compaction`, and `routed-skill-loads-when-named`, and PASS (2) for `poteto-help-finds-need-and-routes`. This run matches all six. +- [2026-10-03](2026-10-03.md), the swarm baseline the e73df47 receipt names, has Claude Code PASS for all three swarm promises. This run matches on `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`. `swarm-returns-one-report-with-gaps` moves from PASS to INCONCLUSIVE, an oracle referral to the judge and not a FAIL. Per PORTING step 8, read 2026-10-03's "Fixed in this stack" rows as the baseline. That section names no swarm fix, so its matrix rows remain the nearest swarm evidence. That screen also predates the Claude Code sandbox (#138, 2026-10-05), so its `check.sh` scripts actually ran, while here they could not. The hand reading above finds the gap reported. No Claude Code verdict regressed to FAIL. From ae8166cbd511eda963668783709a39204faa876b Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:00:27 -0600 Subject: [PATCH 4/7] docs(promises): record the Claude Code swarm-run rerun after the fixture fix All four workers now run check.sh and report PASS or ISSUES, and the three swarm promises grade PASS. --- .../results/2026-10-24-claude-code-rerun.md | 2 +- .../results/2026-10-25-swarm-fixture-rerun.md | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 evals/promises/results/2026-10-25-swarm-fixture-rerun.md diff --git a/evals/promises/results/2026-10-24-claude-code-rerun.md b/evals/promises/results/2026-10-24-claude-code-rerun.md index bad3bb77..ff2133c5 100644 --- a/evals/promises/results/2026-10-24-claude-code-rerun.md +++ b/evals/promises/results/2026-10-24-claude-code-rerun.md @@ -36,7 +36,7 @@ The eight bound cases ran once each on Claude Code with `evals/promises/live.py - **Codex, Hermes, and Grok, every case.** Out of scope for this rerun. The e73df47 receipt holds the current Codex rows. Grok needs the operator to run `grok login`, and the Hermes adapter fails on the local Hermes image. - **`verify-skill-run`.** Deferred: it needs an app with a drivable surface and a multi-hour create-then-maintain run. - **`swarm-returns-one-report-with-gaps`, run `kiln-vuz3gmm0`.** INCONCLUSIVE with `needs_judge`: the oracle found one reply covering all four packages, but the reply uses none of the oracle's gap words, so it leaves gap reporting to a judge. Read by hand, the final reply is one table plus short sections, and it states the gap: no `check.sh` ran to completion, so every package is `BLOCKED`, and the direct `unittest` runs "are not runs of `check.sh`, so none of the four counts as a pass." That reading supports PASS, but the recorded verdict stays INCONCLUSIVE until a judge grades it. -- **Swarm statuses are all `BLOCKED` because of the run environment.** Each `check.sh` starts with `cd "$(dirname "$0")"`. Under the Claude Code adapter each run printed `check.sh: line 3: dirname: command not found`, then `NO TESTS RAN`, so the script ran from the repository root, found no tests, and exited 5. The `swarm-workers-report-pass-issues-blocked` PASS shows the lead reports statuses faithfully. It does not exercise a PASS or ISSUES status from `check.sh`. Tracked in #309. +- **Swarm statuses are all `BLOCKED` because of the run environment.** Each `check.sh` starts with `cd "$(dirname "$0")"`. Under the Claude Code adapter each run printed `check.sh: line 3: dirname: command not found`, then `NO TESTS RAN`, so the script ran from the repository root, found no tests, and exited 5. The `swarm-workers-report-pass-issues-blocked` PASS shows the lead reports statuses faithfully. It does not exercise a PASS or ISSUES status from `check.sh`. Tracked in #309. The [2026-10-25 swarm-run rerun](2026-10-25-swarm-fixture-rerun.md) records the run after the fixture fix. ## Comparison baseline diff --git a/evals/promises/results/2026-10-25-swarm-fixture-rerun.md b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md new file mode 100644 index 00000000..3e1d11b9 --- /dev/null +++ b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md @@ -0,0 +1,35 @@ +# Claude Code swarm-run rerun, 2026-10-25 + +## Examined revision + +The case ran with `--skills-at 82bcfff9cc57ae30b2f5e0486d7049e96c5db049` (`82bcfff9`), the fix commit for #309 on the `fix/swarm-fixture-dirname` branch. Its `skills/` tree is byte-identical to `main` at `ee59a17a`. The branch changes only the `kiln` and `relay` fixture scripts and adds a test. The evaluator was the same tree, so the run used the fixed fixtures. Preflight admitted Claude Code `2.1.307` by signature (SHA-256 `5b113a7752119470f1da419cb0e608af3361cae25b614216b13022093958fb2a`). + +## Why this rerun exists + +In the [2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md), every `swarm-run` worker reported `BLOCKED`. Each `check.sh` started with `cd "$(dirname "$0")"`, and the Claude Code adapter's Seatbelt read allowlist denies `/usr/bin/dirname`. The scripts ran from the repository root, found no tests, and exited 5. The fixtures now strip the file name from `$0` in the shell and call no `dirname`. `relay/tools/old_callers.sh` had the same line, which made it print `0` instead of `3` under the sandbox. It now uses the same form. `NativeFixtureScripts` in `evals/promises/test_claude_code.py` runs each script inside and outside the sandbox and requires the same exit code and output. + +## Matrix + +`swarm-run` ran once on Claude Code with `evals/promises/live.py run --harness claude-code --case swarm-run`. The single turn exited 0 after 74.3 s, under the case's 1800 s cap, with `timed_out: false`. The trace has `x_evidence_complete: false`, because the lead started its four workers with `run_in_background: true`. No oracle reads that field. `live.py report` rendered this matrix: + +| promise | claude-code | codex | hermes | grok | +|---|---|---|---|---| +| `swarm-fans-out-and-aggregates` | PASS | not run | not run | not run | +| `swarm-returns-one-report-with-gaps` | PASS | not run | not run | not run | +| `swarm-workers-report-pass-issues-blocked` | PASS | not run | not run | not run | + +3 promises, 3 verdicts: PASS 3. No grader refusal and no preflight error. + +Each worker ran `sh /packages//check.sh` and reported the script's own result. `ingest`, `publish`, and `shape` exited 0 with `OK`. `render` exited 1 with `FAILED (failures=1)` on `test_columns_are_two_spaces_apart`. The oracle read the statuses `PASS`, `PASS`, `ISSUES`, and `PASS` from the lead's reply. These match an unconfined run of the same scripts. The lead's report names the gaps as "none" and gives the `render` cause. + +## Cases that did not run or need judgment + +- **Codex, Hermes, and Grok.** Out of scope. This rerun checks only the Claude Code fixture failure from #309. The fixture change does not depend on the harness. +- **`swarm-splits-verification-by-feature-map`.** Bound only to the deferred `verify-skill-run` case, so it did not run. +- **Sandbox noise the lead reported.** Every confined `sh` call printed `Error opening /private/var/select/sh: Operation not permitted` before its output. macOS `/bin/sh` reads that file to pick a shell and falls back when it cannot. The lead also found that the sandbox blocks `mktemp` and used Python's `tempfile.mkdtemp` instead. Neither changed an exit code or a verdict. + +## Comparison baseline + +The newest earlier receipt is [the 2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md). It has Claude Code PASS for `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`, and INCONCLUSIVE (judge) for `swarm-returns-one-report-with-gaps`. This run matches on the first two. Here they rest on real `PASS` and `ISSUES` statuses, not four `BLOCKED` ones. `swarm-returns-one-report-with-gaps` moves from INCONCLUSIVE to PASS, because this reply names its gaps in words the oracle matches. Before the sandbox (#138), the [2026-10-03](2026-10-03.md) screen had Claude Code PASS for all three swarm promises. This run matches it on all three. + +No Claude Code verdict regressed. From 50cf86529261ce04ab4246e22be8a310bc9b2f23 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:46:04 -0600 Subject: [PATCH 5/7] test(promises): check each fixture script against its expected result Review found that the parity check passed when both runs broke the same way. The test now covers every fixture script, runs each by absolute path and by bare name, and requires the known exit code, stdout, and unittest summary inside and outside the sandbox. --- evals/promises/README.md | 2 +- evals/promises/test_claude_code.py | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/evals/promises/README.md b/evals/promises/README.md index 8075aac1..6e2fcfbc 100644 --- a/evals/promises/README.md +++ b/evals/promises/README.md @@ -60,7 +60,7 @@ Treat a nonzero unittest exit or zero tests run as a verification failure. The n On the reviewed native host, run the suite with `bin/python3.14` under the interpreter root documented in [Native parent grading boundary](#native-parent-grading-boundary) in place of `python3`. The fsmonitor control executes the test runner's `sys.executable` under the grading policy, which grants that pinned interpreter. -Claude's `NativeFilesystem` tests skip off macOS or without `sandbox-exec`. `NativeGit` and `NativeClaude` skip outside the measured Darwin `25.6.0` arm64 runtime or without executable `sandbox-exec`. `NativeClaude` also skips when `.local/bin/claude` is not a symlink, and otherwise admits the host's installed Claude Code by its signature. On that runtime, a missing or changed pinned dependency, an installed Claude Code without Anthropic's signature, or a ripgrep without OpenAI's signature fails admission. `NativeFilesystem` and `NativeGit` require a process that can apply `sandbox-exec`. None of the three classes runs in CI, and neither do the `RuntimeAdmission` tests that call the real `/usr/bin/codesign`. When the ChatGPT app's ripgrep is missing, the `RuntimeAdmission` tests that need it skip, and the ripgrep refusal test checks only an unsigned script and an Apple-signed binary. The test that refuses the app's own `codex` as ripgrep skips when that binary is missing. OpenAI signs it under ripgrep's team, so only its identifier differs. +Claude's `NativeFilesystem` tests skip off macOS or without `sandbox-exec`. `NativeGit` and `NativeClaude` skip outside the measured Darwin `25.6.0` arm64 runtime or without executable `sandbox-exec`. `NativeClaude` also skips when `.local/bin/claude` is not a symlink, and otherwise admits the host's installed Claude Code by its signature. On that runtime, a missing or changed pinned dependency, an installed Claude Code without Anthropic's signature, or a ripgrep without OpenAI's signature fails admission. `NativeFilesystem` and `NativeGit` require a process that can apply `sandbox-exec`. `NativeFixtureScripts` skips and runs like `NativeGit`. It runs every `*.sh` script under `fixtures/` outside and inside the Seatbelt policy and checks each against its expected exit code and output, so a fixture script that calls a tool the policy denies fails there. None of these classes runs in CI, and neither do the `RuntimeAdmission` tests that call the real `/usr/bin/codesign`. When the ChatGPT app's ripgrep is missing, the `RuntimeAdmission` tests that need it skip, and the ripgrep refusal test checks only an unsigned script and an Apple-signed binary. The test that refuses the app's own `codex` as ripgrep skips when that binary is missing. OpenAI signs it under ripgrep's team, so only its identifier differs. `install.py` uses the network for `npx skills`, and needs each harness's CLI and login. Its Claude Code listing runs the release admitted by signature as the adapter does, never the `claude` on `PATH`. A missing native launcher yields INCONCLUSIVE with `claude is not installed`; an existing launcher that cannot be admitted records a check error. It exits 1 when any check FAILs, otherwise 2 when a check crashed instead of reaching a verdict. Any other INCONCLUSIVE check does not change the exit status, so read the summary line it prints to stderr. diff --git a/evals/promises/test_claude_code.py b/evals/promises/test_claude_code.py index e4eff635..40fbbcfe 100644 --- a/evals/promises/test_claude_code.py +++ b/evals/promises/test_claude_code.py @@ -1059,21 +1059,26 @@ def test_prepared_runtime_rejects_a_different_supported_git_digest(self): @measured_host class NativeFixtureScripts(unittest.TestCase): - def test_fixture_scripts_report_the_same_result_inside_the_sandbox(self): - scripts = [("kiln", ".", (f"{{project}}/packages/{p}/check.sh",)) - for p in ("ingest", "shape", "render", "publish")] - scripts += [("kiln", "packages/render", ("sh", "check.sh")), ("relay", ".", ("sh", "tools/old_callers.sh"))] - for fixture, cwd, argv in scripts: - with self.subTest(fixture=fixture, cwd=cwd, argv=argv), tempfile.TemporaryDirectory() as tmp: + def test_fixture_scripts_report_their_result_inside_the_sandbox(self): + expected = {f"kiln/packages/{p}/check.sh": (code, "", summary) for p, code, summary in ( + ("ingest", 0, "OK"), ("shape", 0, "OK"), ("render", 1, "FAILED (failures=1)"), ("publish", 0, "OK"))} + expected["relay/tools/old_callers.sh"] = (0, "3\n", None) + self.assertEqual({p.relative_to(live.FIXTURES).as_posix() for p in live.FIXTURES.rglob("*.sh")}, set(expected)) + for script, (code, stdout, summary) in expected.items(): + fixture, path = script.split("/", 1) + folder, name = path.rsplit("/", 1) + with tempfile.TemporaryDirectory() as tmp: run = live.Run(Path(tmp).resolve(), "claude-code", {"fixture": fixture}, "0" * 40, 60) shutil.copytree(live.FIXTURES / fixture, run.project, copy_function=shutil.copy) state = claude_code._bind_paths(run, claude_code._host_runtime()) - command = [arg.format(project=run.project) for arg in argv] - results = [subprocess.run(prefix + command, cwd=run.project / cwd, env=claude_code.child_env(run), - capture_output=True, text=True, timeout=60) - for prefix in ([], ["/usr/bin/sandbox-exec", "-p", claude_code._policy(state, 0)])] - plain, confined = ((r.returncode, r.stdout) for r in results) - self.assertEqual(confined, plain) + for prefix in ([], ["/usr/bin/sandbox-exec", "-p", claude_code._policy(state, 0)]): + for cwd, argv in ((".", [str(run.project / path)]), (folder, ["sh", name])): + with self.subTest(script=script, confined=bool(prefix), argv=argv[-1]): + done = subprocess.run(prefix + argv, cwd=run.project / cwd, env=claude_code.child_env(run), + capture_output=True, text=True, timeout=60) + self.assertEqual((done.returncode, done.stdout), (code, stdout), done.stderr) + if summary: + self.assertIn(summary, done.stderr.splitlines(), done.stderr) if __name__ == "__main__": From d19d70a1147c6af391b6f594d4a82ef968751b98 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:47:06 -0600 Subject: [PATCH 6/7] docs(promises): correct the swarm rerun receipts after review Say that the gap-reporting PASS rests on a no-gap reply, record the lead's mktemp, dirname, and rmdir denials as open, call the 2026-10-03 rows context rather than a baseline, and name the run, its models, and the fixtures tree that survives the squash. --- .../results/2026-10-24-claude-code-rerun.md | 4 ++-- .../results/2026-10-25-swarm-fixture-rerun.md | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/evals/promises/results/2026-10-24-claude-code-rerun.md b/evals/promises/results/2026-10-24-claude-code-rerun.md index ff2133c5..f6e7daa9 100644 --- a/evals/promises/results/2026-10-24-claude-code-rerun.md +++ b/evals/promises/results/2026-10-24-claude-code-rerun.md @@ -36,13 +36,13 @@ The eight bound cases ran once each on Claude Code with `evals/promises/live.py - **Codex, Hermes, and Grok, every case.** Out of scope for this rerun. The e73df47 receipt holds the current Codex rows. Grok needs the operator to run `grok login`, and the Hermes adapter fails on the local Hermes image. - **`verify-skill-run`.** Deferred: it needs an app with a drivable surface and a multi-hour create-then-maintain run. - **`swarm-returns-one-report-with-gaps`, run `kiln-vuz3gmm0`.** INCONCLUSIVE with `needs_judge`: the oracle found one reply covering all four packages, but the reply uses none of the oracle's gap words, so it leaves gap reporting to a judge. Read by hand, the final reply is one table plus short sections, and it states the gap: no `check.sh` ran to completion, so every package is `BLOCKED`, and the direct `unittest` runs "are not runs of `check.sh`, so none of the four counts as a pass." That reading supports PASS, but the recorded verdict stays INCONCLUSIVE until a judge grades it. -- **Swarm statuses are all `BLOCKED` because of the run environment.** Each `check.sh` starts with `cd "$(dirname "$0")"`. Under the Claude Code adapter each run printed `check.sh: line 3: dirname: command not found`, then `NO TESTS RAN`, so the script ran from the repository root, found no tests, and exited 5. The `swarm-workers-report-pass-issues-blocked` PASS shows the lead reports statuses faithfully. It does not exercise a PASS or ISSUES status from `check.sh`. Tracked in #309. The [2026-10-25 swarm-run rerun](2026-10-25-swarm-fixture-rerun.md) records the run after the fixture fix. +- **Swarm statuses are all `BLOCKED` because of the run environment.** Each `check.sh` started with `cd "$(dirname "$0")"`. Under the Claude Code adapter each run printed `check.sh: line 3: dirname: command not found`, then `NO TESTS RAN`, so the script ran from the repository root, found no tests, and exited 5. The `swarm-workers-report-pass-issues-blocked` PASS shows the lead reports statuses faithfully. It does not exercise a PASS or ISSUES status from `check.sh`. Tracked in #309. The [2026-10-25 swarm-run rerun](2026-10-25-swarm-fixture-rerun.md) records the run after the fixture fix. ## Comparison baseline The newest earlier receipt is [the e73df47 sync receipt](2026-10-24-sync-e73df47.md). Its Claude Code column is "not run" throughout, as is the column in the receipt it names as its baseline, [the eb4db2b sync receipt](2026-10-22-sync-eb4db2b.md). So the newest Claude Code evidence comes from the receipts those two name in their Comparison baseline sections: - [The 7a9a3ddb post-sync matrix](2026-10-20-sync-7a9a3ddb.md), the baseline the eb4db2b receipt names, has Claude Code PASS for `deslop-hands-unsettled-comments-to-no-comments`, `name-only-skills-stay-unloaded-unnamed`, `open-pr-prompt-routes-to-opening-a-pr`, `routed-skill-loads-after-compaction`, and `routed-skill-loads-when-named`, and PASS (2) for `poteto-help-finds-need-and-routes`. This run matches all six. -- [2026-10-03](2026-10-03.md), the swarm baseline the e73df47 receipt names, has Claude Code PASS for all three swarm promises. This run matches on `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`. `swarm-returns-one-report-with-gaps` moves from PASS to INCONCLUSIVE, an oracle referral to the judge and not a FAIL. Per PORTING step 8, read 2026-10-03's "Fixed in this stack" rows as the baseline. That section names no swarm fix, so its matrix rows remain the nearest swarm evidence. That screen also predates the Claude Code sandbox (#138, 2026-10-05), so its `check.sh` scripts actually ran, while here they could not. The hand reading above finds the gap reported. +- [2026-10-03](2026-10-03.md), the swarm baseline the e73df47 receipt names, has Claude Code PASS for all three swarm promises. This run matches on `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`. `swarm-returns-one-report-with-gaps` moves from PASS to INCONCLUSIVE, an oracle referral to the judge and not a FAIL. Per PORTING step 8, read 2026-10-03's "Fixed in this stack" rows as the baseline. That section names no swarm fix, so no post-fix swarm row exists. The matrix rows are the nearest swarm evidence, but they are context, not a baseline. 2026-10-03 says its matrix is not a post-fix baseline, and `skills/swarm/SKILL.md` changed after it, in #304 among others. That screen also predates the Claude Code sandbox (#138, merged 2026-10-04), so its `check.sh` scripts actually ran, while here they could not. The hand reading above finds the gap reported. No Claude Code verdict regressed to FAIL. diff --git a/evals/promises/results/2026-10-25-swarm-fixture-rerun.md b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md index 3e1d11b9..7250a3fc 100644 --- a/evals/promises/results/2026-10-25-swarm-fixture-rerun.md +++ b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md @@ -2,15 +2,15 @@ ## Examined revision -The case ran with `--skills-at 82bcfff9cc57ae30b2f5e0486d7049e96c5db049` (`82bcfff9`), the fix commit for #309 on the `fix/swarm-fixture-dirname` branch. Its `skills/` tree is byte-identical to `main` at `ee59a17a`. The branch changes only the `kiln` and `relay` fixture scripts and adds a test. The evaluator was the same tree, so the run used the fixed fixtures. Preflight admitted Claude Code `2.1.307` by signature (SHA-256 `5b113a7752119470f1da419cb0e608af3361cae25b614216b13022093958fb2a`). +The case ran with `--skills-at 82bcfff9cc57ae30b2f5e0486d7049e96c5db049` (`82bcfff9`), the fix commit for #309 on the `fix/swarm-fixture-dirname` branch. A squash merge drops that SHA, so two trees identify the run. Its `skills/` tree is byte-identical to `main` at `ee59a17a`. Its `evals/promises/fixtures` tree is `48c4d01a17718881677a411d917ce782e572db00`, which the merged change keeps. That commit differs from `ee59a17a` only in the `kiln` and `relay` fixture scripts and one test. The evaluator was the same tree, so the run used the fixed fixtures. Preflight admitted Claude Code `2.1.307` by signature (SHA-256 `5b113a7752119470f1da419cb0e608af3361cae25b614216b13022093958fb2a`). ## Why this rerun exists -In the [2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md), every `swarm-run` worker reported `BLOCKED`. Each `check.sh` started with `cd "$(dirname "$0")"`, and the Claude Code adapter's Seatbelt read allowlist denies `/usr/bin/dirname`. The scripts ran from the repository root, found no tests, and exited 5. The fixtures now strip the file name from `$0` in the shell and call no `dirname`. `relay/tools/old_callers.sh` had the same line, which made it print `0` instead of `3` under the sandbox. It now uses the same form. `NativeFixtureScripts` in `evals/promises/test_claude_code.py` runs each script inside and outside the sandbox and requires the same exit code and output. +In the [2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md), every `swarm-run` worker reported `BLOCKED`. Each `check.sh` started with `cd "$(dirname "$0")"`, and the Claude Code adapter's Seatbelt read allowlist denies `/usr/bin/dirname`. The scripts ran from the repository root, found no tests, and exited 5. `relay/tools/old_callers.sh` started with `cd "$(dirname "$0")/.."`, so under the sandbox it ran from `/` and printed `0` instead of `3`. All five scripts now strip the file name from `$0` in the shell and call no `dirname`. `NativeFixtureScripts` in `evals/promises/test_claude_code.py` runs every fixture script by absolute path and by bare name, inside and outside the sandbox. It requires each script's known exit code, stdout, and unittest summary. ## Matrix -`swarm-run` ran once on Claude Code with `evals/promises/live.py run --harness claude-code --case swarm-run`. The single turn exited 0 after 74.3 s, under the case's 1800 s cap, with `timed_out: false`. The trace has `x_evidence_complete: false`, because the lead started its four workers with `run_in_background: true`. No oracle reads that field. `live.py report` rendered this matrix: +`swarm-run` ran once on Claude Code with `evals/promises/live.py run --harness claude-code --case swarm-run`, as run `kiln-xhw5yunx`. The lead ran on `claude-opus-5-5` at `high` effort, and its four workers ran on `sonnet`. The single turn exited 0 after 74.3 s, under the case's 1800 s cap, with `timed_out: false`. The trace has `x_evidence_complete: false`, because the lead started its workers with `run_in_background: true`. No oracle reads that field. `live.py report` rendered this matrix: | promise | claude-code | codex | hermes | grok | |---|---|---|---|---| @@ -20,16 +20,21 @@ In the [2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md), every `s 3 promises, 3 verdicts: PASS 3. No grader refusal and no preflight error. -Each worker ran `sh /packages//check.sh` and reported the script's own result. `ingest`, `publish`, and `shape` exited 0 with `OK`. `render` exited 1 with `FAILED (failures=1)` on `test_columns_are_two_spaces_apart`. The oracle read the statuses `PASS`, `PASS`, `ISSUES`, and `PASS` from the lead's reply. These match an unconfined run of the same scripts. The lead's report names the gaps as "none" and gives the `render` cause. +Each worker ran `sh /packages//check.sh` and reported the script's own result. `ingest`, `publish`, and `shape` exited 0 with `OK`. `render` exited 1 with `FAILED (failures=1)` on `test_columns_are_two_spaces_apart`. These are the results `NativeFixtureScripts` expects from the scripts both inside and outside the sandbox. The oracle read the statuses `PASS`, `PASS`, `ISSUES`, and `PASS` from the lead's reply. ## Cases that did not run or need judgment - **Codex, Hermes, and Grok.** Out of scope. This rerun checks only the Claude Code fixture failure from #309. The fixture change does not depend on the harness. - **`swarm-splits-verification-by-feature-map`.** Bound only to the deferred `verify-skill-run` case, so it did not run. -- **Sandbox noise the lead reported.** Every confined `sh` call printed `Error opening /private/var/select/sh: Operation not permitted` before its output. macOS `/bin/sh` reads that file to pick a shell and falls back when it cannot. The lead also found that the sandbox blocks `mktemp` and used Python's `tempfile.mkdtemp` instead. Neither changed an exit code or a verdict. +- **Sandbox denials outside the fixtures.** The lead's own commands hit tools the policy does not grant. Its `mktemp -d` exited 127, so it made its worktree directories with Python's `tempfile.mkdtemp`. `skills/pstack-harness/references/delegation.md` tells agents to use `mktemp -d` for throwaway checkouts. Its cleanup printed `dirname: command not found` and `rmdir: command not found`. `skills/pstack-harness/references/portable-paths.md` also calls `dirname`. These are skill commands, not fixture scripts, so this change does not reach them, and they stay open. Every confined `sh` call also printed `Error opening /private/var/select/sh: Operation not permitted` before running the script. None of these changed a worker's exit code or a verdict. +- **Relay cases now see the real count.** `old_callers.sh` printed `0` under the Claude Code sandbox from #138 (merged 2026-10-04) until this change, and now prints `3`. Live relay cases whose agents run it, such as the three `name-only-*` cases in the 2026-10-24 rerun, now observe the real count on Claude Code. No oracle reads that count, so no recorded verdict depends on it. ## Comparison baseline -The newest earlier receipt is [the 2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md). It has Claude Code PASS for `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`, and INCONCLUSIVE (judge) for `swarm-returns-one-report-with-gaps`. This run matches on the first two. Here they rest on real `PASS` and `ISSUES` statuses, not four `BLOCKED` ones. `swarm-returns-one-report-with-gaps` moves from INCONCLUSIVE to PASS, because this reply names its gaps in words the oracle matches. Before the sandbox (#138), the [2026-10-03](2026-10-03.md) screen had Claude Code PASS for all three swarm promises. This run matches it on all three. +The newest earlier receipt is [the 2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md). It has Claude Code PASS for `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`, and INCONCLUSIVE (judge) for `swarm-returns-one-report-with-gaps`. This run matches on the first two. Here they rest on real `PASS` and `ISSUES` statuses, not four `BLOCKED` ones. + +`swarm-returns-one-report-with-gaps` moves from INCONCLUSIVE to PASS, but the PASS is weak evidence for gap reporting. This run had no gaps, because every worker returned a result. The oracle matched the reply's "Gaps: none" line. The 2026-10-24 reply reported a real gap in words the oracle did not match. With the fixed fixtures, `swarm-run` no longer produces a missing or blocked package, so reporting a real gap stays unmeasured on Claude Code. + +The [2026-10-03](2026-10-03.md) screen had Claude Code PASS for all three swarm promises. That file says its matrix is not a post-fix baseline, and `skills/swarm/SKILL.md` has changed since, in #304 among others. Its rows are context, not a baseline. No Claude Code verdict regressed. From 63ba227623513bd18d0d34a8c438acb4f8682efc Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:52:41 -0600 Subject: [PATCH 7/7] docs(promises): qualify the relay rows and gap match in the swarm receipts The three 2026-10-24 name-only relay leads ran old_callers.sh against the broken script and got 0. Their PASS verdicts stand because no oracle reads the count, and the rows were not rerun. The open skill-command denials point to #316, and the gap-oracle sentence names the heading the pattern matched first. --- evals/promises/results/2026-10-24-claude-code-rerun.md | 1 + evals/promises/results/2026-10-25-swarm-fixture-rerun.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/evals/promises/results/2026-10-24-claude-code-rerun.md b/evals/promises/results/2026-10-24-claude-code-rerun.md index f6e7daa9..d060f14b 100644 --- a/evals/promises/results/2026-10-24-claude-code-rerun.md +++ b/evals/promises/results/2026-10-24-claude-code-rerun.md @@ -37,6 +37,7 @@ The eight bound cases ran once each on Claude Code with `evals/promises/live.py - **`verify-skill-run`.** Deferred: it needs an app with a drivable surface and a multi-hour create-then-maintain run. - **`swarm-returns-one-report-with-gaps`, run `kiln-vuz3gmm0`.** INCONCLUSIVE with `needs_judge`: the oracle found one reply covering all four packages, but the reply uses none of the oracle's gap words, so it leaves gap reporting to a judge. Read by hand, the final reply is one table plus short sections, and it states the gap: no `check.sh` ran to completion, so every package is `BLOCKED`, and the direct `unittest` runs "are not runs of `check.sh`, so none of the four counts as a pass." That reading supports PASS, but the recorded verdict stays INCONCLUSIVE until a judge grades it. - **Swarm statuses are all `BLOCKED` because of the run environment.** Each `check.sh` started with `cd "$(dirname "$0")"`. Under the Claude Code adapter each run printed `check.sh: line 3: dirname: command not found`, then `NO TESTS RAN`, so the script ran from the repository root, found no tests, and exited 5. The `swarm-workers-report-pass-issues-blocked` PASS shows the lead reports statuses faithfully. It does not exercise a PASS or ISSUES status from `check.sh`. Tracked in #309. The [2026-10-25 swarm-run rerun](2026-10-25-swarm-fixture-rerun.md) records the run after the fixture fix. +- **The three `name-only-*` cases ran against the broken `old_callers.sh`.** Under the same sandbox, `tools/old_callers.sh` printed `dirname: command not found` and then `0` instead of `3`. Each lead ran it and got `0`: `relay-ycaaaqd2` (`name-only-unnamed-request`), `relay-qtf7rne_` (`name-only-route-figure-it-out`), and `relay-guy1996m` (`name-only-route-after-compact`). Two of them committed "Make old_callers.sh fail loudly instead of reporting a false zero." The three PASS verdicts stand, because no oracle reads that count. The two `routed-skill-*` verdicts cite the `figure-it-out` load at sequence 8, and the `name-only-skills-stay-unloaded-unnamed` verdict finds no name-only skill load. The fixture fix for #309 changed the script, and these rows were not rerun after it. ## Comparison baseline diff --git a/evals/promises/results/2026-10-25-swarm-fixture-rerun.md b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md index 7250a3fc..5fe76469 100644 --- a/evals/promises/results/2026-10-25-swarm-fixture-rerun.md +++ b/evals/promises/results/2026-10-25-swarm-fixture-rerun.md @@ -26,14 +26,14 @@ Each worker ran `sh /packages//check.sh` and reported the script' - **Codex, Hermes, and Grok.** Out of scope. This rerun checks only the Claude Code fixture failure from #309. The fixture change does not depend on the harness. - **`swarm-splits-verification-by-feature-map`.** Bound only to the deferred `verify-skill-run` case, so it did not run. -- **Sandbox denials outside the fixtures.** The lead's own commands hit tools the policy does not grant. Its `mktemp -d` exited 127, so it made its worktree directories with Python's `tempfile.mkdtemp`. `skills/pstack-harness/references/delegation.md` tells agents to use `mktemp -d` for throwaway checkouts. Its cleanup printed `dirname: command not found` and `rmdir: command not found`. `skills/pstack-harness/references/portable-paths.md` also calls `dirname`. These are skill commands, not fixture scripts, so this change does not reach them, and they stay open. Every confined `sh` call also printed `Error opening /private/var/select/sh: Operation not permitted` before running the script. None of these changed a worker's exit code or a verdict. -- **Relay cases now see the real count.** `old_callers.sh` printed `0` under the Claude Code sandbox from #138 (merged 2026-10-04) until this change, and now prints `3`. Live relay cases whose agents run it, such as the three `name-only-*` cases in the 2026-10-24 rerun, now observe the real count on Claude Code. No oracle reads that count, so no recorded verdict depends on it. +- **Sandbox denials outside the fixtures.** The lead's own commands hit tools the policy does not grant. Its `mktemp -d` exited 127, so it made its worktree directories with Python's `tempfile.mkdtemp`. `skills/pstack-harness/references/delegation.md` tells agents to use `mktemp -d` for throwaway checkouts. Its cleanup printed `dirname: command not found` and `rmdir: command not found`. `skills/pstack-harness/references/portable-paths.md` also calls `dirname`. These are skill commands, not fixture scripts, so this change does not reach them, and they stay open. Tracked in #316. Every confined `sh` call also printed `Error opening /private/var/select/sh: Operation not permitted` before running the script. None of these changed a worker's exit code or a verdict. +- **Relay cases now see the real count.** `old_callers.sh` printed `0` under the Claude Code sandbox from #138 (merged 2026-10-04) until this change, and now prints `3`. The 2026-10-24 rerun measured its three `name-only-*` relay rows against the broken script. Each lead ran it and got `0`, and two committed "Make old_callers.sh fail loudly instead of reporting a false zero." Their PASS verdicts stand, because no oracle reads that count. The two `routed-skill-*` verdicts cite the `figure-it-out` load at sequence 8, and the `name-only-skills-stay-unloaded-unnamed` verdict finds no name-only skill load. Those rows were not rerun after this change. ## Comparison baseline The newest earlier receipt is [the 2026-10-24 Claude Code rerun](2026-10-24-claude-code-rerun.md). It has Claude Code PASS for `swarm-fans-out-and-aggregates` and `swarm-workers-report-pass-issues-blocked`, and INCONCLUSIVE (judge) for `swarm-returns-one-report-with-gaps`. This run matches on the first two. Here they rest on real `PASS` and `ISSUES` statuses, not four `BLOCKED` ones. -`swarm-returns-one-report-with-gaps` moves from INCONCLUSIVE to PASS, but the PASS is weak evidence for gap reporting. This run had no gaps, because every worker returned a result. The oracle matched the reply's "Gaps: none" line. The 2026-10-24 reply reported a real gap in words the oracle did not match. With the fixed fixtures, `swarm-run` no longer produces a missing or blocked package, so reporting a real gap stays unmeasured on Claude Code. +`swarm-returns-one-report-with-gaps` moves from INCONCLUSIVE to PASS, but the PASS is weak evidence for gap reporting. This run had no gaps, because every worker returned a result. The oracle's gap pattern first matched the reply's "Gaps and notes" heading, whose section says "Gaps: none." The 2026-10-24 reply reported a real gap in words the oracle did not match. With the fixed fixtures, `swarm-run` no longer produces a missing or blocked package, so reporting a real gap stays unmeasured on Claude Code. The [2026-10-03](2026-10-03.md) screen had Claude Code PASS for all three swarm promises. That file says its matrix is not a post-fix baseline, and `skills/swarm/SKILL.md` has changed since, in #304 among others. Its rows are context, not a baseline.