diff --git a/evals/promises/README.md b/evals/promises/README.md index e9d9bd16..963eec2f 100644 --- a/evals/promises/README.md +++ b/evals/promises/README.md @@ -240,7 +240,7 @@ For the Claude Code, Codex, and Grok adapters, `live.execute` creates each turn' Each adapter in `harnesses/` isolates its harness from the host's own skills and settings: -- **Claude Code.** An outer Seatbelt profile denies filesystem reads and writes except the fixture, private runtime state, and reviewed read dependencies. Support requires the measured Darwin `25.6.0` arm64 host and Claude Code's native install below the current account's home. The adapter reads the installer's `.local/bin/claude` symlink. It must hold the absolute path of a file directly inside `.local/share/claude/versions` whose name is a release number such as `2.1.304`. Neither that file nor any directory above it may be a link. `/usr/bin/codesign --verify --strict` must accept the file against Anthropic's Developer ID Application requirement, with identifier `com.anthropic.claude-code` and team `Q6L2SF6YDW`. An unsigned file, a file from another signer, a directory or other non-regular file, or a launcher that points anywhere else raises `IsolationUnavailable`. Admission digests the file before and after `codesign` runs and refuses a change between the two. Every later check holds the run to that digest. No release is pinned, so a Claude Code update needs no adapter change. Admission runs once per `live.py` process, so every run in a batch uses the same release. If the installer deletes that release during a `live.py run`, the next check raises `FileNotFoundError` and stops that process. A new `live.py` process admits the current release. `launch.json` in the run root records the admitted `path`, `version`, and `sha256`, and harvest copies that record to `x_binary`. Python and Node require the mise directories `.local/share/mise/installs/python/3.14.7` and `.local/share/mise/installs/node/24.20.0` below that same home. The adapter snapshots the operating system account database at import, so an exported `HOME` does not select these runtime paths. Within those directories, only the `bin/python3.14` and `bin/node` executable digests are pinned. Ripgrep requires `/Applications/ChatGPT.app/Contents/Resources/codex-cli/codex-path/rg`. It accepts the original fingerprint and the reviewed ripgrep `15.2.0` build with SHA-256 `e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9`. That build passed strict code-signature verification for OpenAI team `2DC432GLL2`. These are explicit digest admissions. An app update with other bytes still refuses. Git requires `/Library/Developer/CommandLineTools/usr/bin/git` with one of the two fingerprints listed above. Preparation records the measured executable digest. Later checks reject a different digest, even if it is another supported Git or ripgrep build. The fixed child `PATH` selects that native Git. Read grants include its `usr/libexec/git-core` and `usr/share/git-core/templates` directories and literal `usr/share/git-core/gitattributes` file. The adapter also grants named system tools, `/System/Library`, `/usr/lib`, literal locale and resolver files, and the user's `Library/Keychains` tree for reading. Literal read grants include `/`, `/private/etc/hosts`, `/dev/null`, `/dev/random`, and `/dev/urandom`. Writes also permit `/dev/null` and `/dev/fd`. It does not derive grants from the host `PATH`. Unsupported hosts, missing dependencies, changed executables, invalid policy, or substituted directories stop launch. `--setting-sources user,project` loads installed project skills and agents and a private user scope. Preparation writes that scope's `settings.json` in the run's `CLAUDE_CONFIG_DIR` with `syncClaudeAiSkills` and `syncClaudeAiPlugins` off, so the account's own skills and plugins stay out of the run. When the installed tree has `setup-pstack/scripts/skill-listing.py`, preparation runs its `install` against that file and the installed skills, as a user who ran setup-pstack would. It keeps the script's stdout in `skill-listing-install.json` and stops on a nonzero exit or a state other than `current`. A tree without the script installs nothing. `skill-listing.json` in the run root records the step (`installed` or `absent`) and the name-only skills, and harvest copies it to `x_skill_listing`. The Seatbelt policy grants no read of the host's `~/.claude`. Strict empty MCP configuration applies to initial and resumed turns. The boundary restricts filesystem access. Network, signals, and IPC remain unconfined. +- **Claude Code.** An outer Seatbelt profile denies filesystem reads and writes except the fixture, private runtime state, and reviewed read dependencies. Support requires the measured Darwin `25.6.0` arm64 host and Claude Code's native install below the current account's home. The adapter reads the installer's `.local/bin/claude` symlink. It must hold the absolute path of a file directly inside `.local/share/claude/versions` whose name is a release number such as `2.1.304`. Neither that file nor any directory above it may be a link. `/usr/bin/codesign --verify --strict` must accept the file against Anthropic's Developer ID Application requirement, with identifier `com.anthropic.claude-code` and team `Q6L2SF6YDW`. An unsigned file, a file from another signer, a directory or other non-regular file, or a launcher that points anywhere else raises `IsolationUnavailable`. Admission digests the file before and after `codesign` runs and refuses a change between the two. Every later check holds the run to that digest. No release is pinned, so a Claude Code update needs no adapter change. Admission runs once per `live.py` process, so every run in a batch uses the same release. If the installer deletes that release during a `live.py run`, the next check raises `FileNotFoundError` and stops that process. A new `live.py` process admits the current release. `launch.json` in the run root records the admitted `path`, `version`, and `sha256`, and harvest copies that record to `x_binary`. Python and Node require the mise directories `.local/share/mise/installs/python/3.14.7` and `.local/share/mise/installs/node/24.20.0` below that same home. The adapter snapshots the operating system account database at import, so an exported `HOME` does not select these runtime paths. Within those directories, only the `bin/python3.14` and `bin/node` executable digests are pinned. Ripgrep requires `/Applications/ChatGPT.app/Contents/Resources/codex-cli/codex-path/rg`. It accepts the original fingerprint and two reviewed ripgrep `15.2.0` builds with SHA-256 `e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9` and `1507d45d9b1f6e7a3b0daaa788554c327e95e040fa7048d93d19ba97f71fa073`, the second a re-signed copy of the same release. Both passed strict code-signature verification for OpenAI team `2DC432GLL2`. These are explicit digest admissions. An app update with other bytes still refuses. Git requires `/Library/Developer/CommandLineTools/usr/bin/git` with one of the two fingerprints listed above. Preparation records the measured executable digest. Later checks reject a different digest, even if it is another supported Git or ripgrep build. The fixed child `PATH` selects that native Git. Read grants include its `usr/libexec/git-core` and `usr/share/git-core/templates` directories and literal `usr/share/git-core/gitattributes` file. The adapter also grants named system tools, `/System/Library`, `/usr/lib`, literal locale and resolver files, and the user's `Library/Keychains` tree for reading. Literal read grants include `/`, `/private/etc/hosts`, `/dev/null`, `/dev/random`, and `/dev/urandom`. Writes also permit `/dev/null` and `/dev/fd`. It does not derive grants from the host `PATH`. Unsupported hosts, missing dependencies, changed executables, invalid policy, or substituted directories stop launch. `--setting-sources user,project` loads installed project skills and agents and a private user scope. Preparation writes that scope's `settings.json` in the run's `CLAUDE_CONFIG_DIR` with `syncClaudeAiSkills` and `syncClaudeAiPlugins` off, so the account's own skills and plugins stay out of the run. When the installed tree has `setup-pstack/scripts/skill-listing.py`, preparation runs its `install` against that file and the installed skills, as a user who ran setup-pstack would. It keeps the script's stdout in `skill-listing-install.json` and stops on a nonzero exit or a state other than `current`. A tree without the script installs nothing. `skill-listing.json` in the run root records the step (`installed` or `absent`) and the name-only skills, and harvest copies it to `x_skill_listing`. The Seatbelt policy grants no read of the host's `~/.claude`. Strict empty MCP configuration applies to initial and resumed turns. The boundary restricts filesystem access. Network, signals, and IPC remain unconfined. - **Codex.** A moved `HOME` and a private `CODEX_HOME` with an `auth.json` link preserve login without loading host settings. `--disable apps` excludes account apps and their `codex_apps` MCP tools on initial and resumed turns. Harvest reads each rollout once through no-follow descriptors, then retains and parses those same bytes. A linked rollout, dangling or not, raises `GradeRefused` before any of its bytes are read. A linked directory in the rollout store and a linked copy destination raise it too. The error stops the entire `live.py run` batch before the affected run gets `trace.json` or `verdict.json`. - **Hermes.** A throwaway `docker run` of the existing local gateway image, with a temp `HERMES_HOME` and `auth.json` mounted read-only. Preparation resolves the local image ID once. Every probe and turn uses that ID with `--pull=never`, including when `HERMES_IMAGE` names a mutable tag. - **Grok Build.** A temp `HOME` and `GROK_HOME`, with an auth provider that reads `~/.grok/auth.json` and never writes it. Harvest copies session files through no-follow descriptors as Codex harvest does, with the same `GradeRefused` batch stop on a link. It parses session evidence only from the bytes each copy read and wrote, so it never reopens a native or captured session file after the copy. A swapped link or rewrite on either side is never read. A session file that vanishes before its copy raises `GradeRefused` with the same batch stop. diff --git a/evals/promises/harnesses/claude_code.py b/evals/promises/harnesses/claude_code.py index c0a61c8f..b35683b0 100644 --- a/evals/promises/harnesses/claude_code.py +++ b/evals/promises/harnesses/claude_code.py @@ -106,7 +106,8 @@ class ConfinedCommand: (PYTHON_ROOT / "bin/python3.14", ("1bfa9a829d950ecd4870a3d7a6826eb57edb4aa93f69d07cd3bb21e9fcc6d439",)), (NODE_ROOT / "bin/node", ("9d050fd455b56426e25d4d603c7c501cbb2630348e836cf221dcce748e90588a",)), (RG_BINARY, ("268128f0b67b319ce00e659bc9130b28aff1edd7a184660a9930f797b135576c", - "e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9")), + "e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9", + "1507d45d9b1f6e7a3b0daaa788554c327e95e040fa7048d93d19ba97f71fa073")), (GIT_ROOT / "usr/bin/git", ("a73bf622a2e470d5d57a4b1d5aef1e8680e67278018d4858a2f93825b7d595c7", "be4afb2b003904725826250de9fb76567bbacf82323457b5a1ec26706b66bcae")), ) diff --git a/evals/promises/test_claude_code.py b/evals/promises/test_claude_code.py index 70a60820..77bd0a50 100644 --- a/evals/promises/test_claude_code.py +++ b/evals/promises/test_claude_code.py @@ -877,10 +877,11 @@ def test_both_supported_git_digests_become_the_selected_fingerprint(self): runtime = claude_code._host_runtime() self.assertEqual(dict(runtime.fingerprints)[git], digest) - def test_both_reviewed_ripgrep_digests_become_the_selected_fingerprint(self): + def test_every_reviewed_ripgrep_digest_becomes_the_selected_fingerprint(self): rg = Path("/Applications/ChatGPT.app/Contents/Resources/codex-cli/codex-path/rg") for digest in ("268128f0b67b319ce00e659bc9130b28aff1edd7a184660a9930f797b135576c", - "e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9"): + "e97d666f32c3e86b2bd0e047cb1398d4385ea1f05bfb791a8201d3fa98a91bb9", + "1507d45d9b1f6e7a3b0daaa788554c327e95e040fa7048d93d19ba97f71fa073"): with self.subTest(digest=digest): claude_code._host_runtime.cache_clear() self.fingerprints[rg] = digest