From 2b60e1b93f9a39424a334eb73e38030539c6dd2d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:57:58 -0500 Subject: [PATCH 1/6] test(code-tidying): install the comment tooling in the eval run's home The eval sandbox hides the operator's home, so its python3 falls back to a system interpreter with no tree-sitter, and it has no network to install one. change-shape.py then cannot certify a deletion and every remove-comments case could only propose deletions. seed.sh now installs the hash-pinned pygments and tree-sitter wheels into the run's user site, which the scaffold shares with the agent; probe-environment runs the same install and checks the import. Refs: #12703 Co-Authored-By: Claude Opus 5.5 --- .../evals/fixtures/comment-tooling-seed.sh | 15 ++++ .../fixtures/comment-tooling.requirements.txt | 82 +++++++++++++++++++ plugins/code-tidying/evals/fixtures/seed.sh | 4 +- .../evals/probe-environment/case.yaml | 4 +- .../evals/probe-environment/scaffold.sh | 2 + 5 files changed, 105 insertions(+), 2 deletions(-) create mode 100755 plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh create mode 100644 plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt create mode 100755 plugins/code-tidying/evals/probe-environment/scaffold.sh diff --git a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh new file mode 100755 index 00000000000..aef37cdd121 --- /dev/null +++ b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Installs pygments and tree-sitter with its grammars into the run's user site, so the agent's +# python3 imports them and change-shape.py can certify a deletion. The eval sandbox hides the +# operator's home, with any interpreter or package installed there, and has no network for an +# install of its own; the scaffold shares the run's HOME with the agent and has the network. +# Hash-pinned wheels only, so no build code runs. Exits non-zero when the import still fails, +# so no run is scored that could only propose deletions. +# Usage: comment-tooling-seed.sh +set -euo pipefail + +here="$(dirname "${BASH_SOURCE[0]}")" +python3 -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \ + --no-deps --no-cache-dir --disable-pip-version-check --no-warn-script-location --quiet \ + -r "$here/comment-tooling.requirements.txt" +python3 -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python' diff --git a/plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt b/plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt new file mode 100644 index 00000000000..024ebdcae4d --- /dev/null +++ b/plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt @@ -0,0 +1,82 @@ +# The pygments and tree-sitter pins of .github/requirements-ci.txt, which stays the reviewed +# surface: move these lines with it. +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c +tree-sitter==0.26.0 \ + --hash=sha256:1d6fe0e8fb4df77b5ee816228e2c4475a63d8cc1d4d3a7ffd7097b2b87fc3e95 \ + --hash=sha256:514a9bf8993e5210e7970736aaf6020d1759b670e195ef17b1c48f586aa30736 \ + --hash=sha256:10f0d4eb94aa7242dcb7f554bcd24dd7ba1c114f00d58759ba08c7a46c8ec51a \ + --hash=sha256:335294ce0504fcefde5245dff596778ffaf820205b98ae0b549c72e48855f1d8 \ + --hash=sha256:c56581ad256c4195a21bfe449fed5d44a02fe83a4a7d6e70e6ec302c881191c7 \ + --hash=sha256:0f8793fd18ad7eec276ed4b51c097b4bf2002b357259b66b0d75db1f3f41c754 \ + --hash=sha256:dea4b4e27d49e9ec5b785d4f994da000e6726882fcc6ad05ec98478500c71aef \ + --hash=sha256:5a3c93a352b7e6f70f73e121bbfa2d0117ba7478bd51114ed35c91b0b78814fa \ + --hash=sha256:5fc2f41bf246ff2f70a9cc3690be35ec7580a4923151873d898c8bcb1a4503d3 \ + --hash=sha256:b8ea92a255c91671a7ec4625aba3ab7bb5220c423630ffbf83c45d7312abe084 \ + --hash=sha256:f665510f0fcf4636fb9696f1f7853bed7a3bd764b7bb0cb8494e619c14ed5a0c \ + --hash=sha256:ff80d4833d330a73184a3ac5132abe93c575d2dea31975c6f15c0d21fef238aa \ + --hash=sha256:a4033fecc8f606c7f2e8b8014d0057b74668a7f0152763606f7bc25c5f9ec64c \ + --hash=sha256:823251c4b6725a7c03ed497a339135ede7ae4bdde75bb8be7ef5e305aeb4ff52 +tree-sitter-python==0.25.0 \ + --hash=sha256:14a79a47ddef72f987d5a2c122d148a812169d7484ff5c75a3db9609d419f361 \ + --hash=sha256:480c21dbd995b7fe44813e741d71fed10ba695e7caab627fb034e3828469d762 \ + --hash=sha256:86f118e5eecad616ecdb81d171a36dde9bef5a0b21ed71ea9c3e390813c3baf5 \ + --hash=sha256:be71650ca2b93b6e9649e5d65c6811aad87a7614c8c1003246b303f6b150f61b \ + --hash=sha256:e6d5b5799628cc0f24691ab2a172a8e676f668fe90dc60468bee14084a35c16d \ + --hash=sha256:71959832fc5d9642e52c11f2f7d79ae520b461e63334927e93ca46cd61cd9683 \ + --hash=sha256:9bcde33f18792de54ee579b00e1b4fe186b7926825444766f849bf7181793a76 \ + --hash=sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb +tree-sitter-c-sharp==0.23.5 \ + --hash=sha256:61e1981cf21b09ee547b9c4c68e64fb4394325f8fc8d5f6d50d41471eba923ea \ + --hash=sha256:a75994a11f6fed3f5b8c36ad6a00e5dc43205bd912c43af3a2a54fdf649664eb \ + --hash=sha256:aa88a780204cd153c4c1ae2d59c654cee1402212fa0d069823d6d34301587438 \ + --hash=sha256:3ea38fb095d85d360dc5a0bec2fa605e496228876f798c9e089d5f0e72bcef46 \ + --hash=sha256:05a9256415e7f24d4f133133794a9c224c60d19f677a04e2f6a94c25090b6d65 \ + --hash=sha256:8636dc70b5a373c35c1036ed5de98e801f2e4d105ae41e2e20b6804c36e3bf33 \ + --hash=sha256:41a28cfa3d9ea50f5629e44550a03188c8fbd5079803dfc03554b6fd594b33fa \ + --hash=sha256:2de4ebf95ddc2e92cd3105c8a8e0e7ec646bc82f52bfaf2f3acec0fa2401ec09 +tree-sitter-typescript==0.23.2 \ + --hash=sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478 \ + --hash=sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8 \ + --hash=sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31 \ + --hash=sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c \ + --hash=sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0 \ + --hash=sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9 \ + --hash=sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7 +tree-sitter-bash==0.25.1 \ + --hash=sha256:0e6235f59e366d220dde7d830196bed597d01e853e44d8ccd1a82c5dd2500acf \ + --hash=sha256:f4a34a6504c7c5b2a9b8c5c4065531dea19ca2c35026e706cf2eeeebe2c92512 \ + --hash=sha256:e76c4cfb20b076552406782b7f8c2a3946835993df0a44df006de54b7030c7dc \ + --hash=sha256:3f484c4bb8796cde7a87ca351e6116f09653edac0eb3c6d238566359dd28b117 \ + --hash=sha256:5e76af6df46d958c7f5b6d5884c9743218e3902a00ccb493ec92728b1084430b \ + --hash=sha256:a3332d71c7b7d5f78259b19d02d0ea111fcb82b72712ee4a93aaa5b226d3f0a8 \ + --hash=sha256:52a6802d9218f86278aa3e8b459c3abdad67eed0fde1f9f13aca5b6c634217a6 \ + --hash=sha256:59115057ec2bae319e8082ff29559861045002964c3431ccb0fc92aa4bc9bccb +tree-sitter-javascript==0.25.0 \ + --hash=sha256:b70f887fb269d6e58c349d683f59fa647140c410cfe2bee44a883b20ec92e3dc \ + --hash=sha256:8264a996b8845cfce06965152a013b5d9cbb7d199bc3503e12b5682e62bb1de1 \ + --hash=sha256:9dc04ba91fc8583344e57c1f1ed5b2c97ecaaf47480011b92fbeab8dda96db75 \ + --hash=sha256:199d09985190852e0912da2b8d26c932159be314bc04952cf917ed0e4c633e6b \ + --hash=sha256:dfcf789064c58dc13c0a4edb550acacfc6f0f280577f1e7a00de3e89fc7f8ddc \ + --hash=sha256:1b852d3aee8a36186dbcc32c798b11b4869f9b5041743b63b65c2ef793db7a54 \ + --hash=sha256:e5ed840f5bd4a3f0272e441d19429b26eedc257abe5574c8546da6b556865e3c \ + --hash=sha256:622a69d677aa7f6ee2931d8c77c981a33f0ebb6d275aa9d43d3397c879a9bb0b +tree-sitter-yaml==0.7.2 \ + --hash=sha256:7e269ddcfcab8edb14fbb1f1d34eed1e1e26888f78f94eedfe7cc98c60f8bc9f \ + --hash=sha256:0807b7966e23ddf7dddc4545216e28b5a58cdadedcecca86b8d8c74271a07870 \ + --hash=sha256:f1a5c60c98b6c4c037aae023569f020d0c489fad8dc26fdfd5510363c9c29a41 \ + --hash=sha256:88636d19d0654fd24f4f242eaaafa90f6f5ebdba8a62e4b32d251ed156c51a2a \ + --hash=sha256:1d2e8f0bb14aa4537320952d0f9607eef3021d5aada8383c34ebeece17db1e06 \ + --hash=sha256:74ca712c50fc9d7dbc68cb36b4a7811d6e67a5466b5a789f19bf8dd6084ef752 \ + --hash=sha256:7587b5ca00fc4f9a548eff649697a3b395370b2304b399ceefa2087d8a6c9186 \ + --hash=sha256:f63c227b18e7ce7587bce124578f0bbf1f890ac63d3e3cd027417574273642c4 +tree-sitter-toml==0.7.0 \ + --hash=sha256:b9ae5c3e7c5b6bb05299dd73452ceafa7fa0687d5af3012332afa7757653b676 \ + --hash=sha256:18be09538e9775cddc0290392c4e2739de2201260af361473ca60b5c21f7bd22 \ + --hash=sha256:a045e0acfcf91b7065066f7e51ea038ed7385c1e35e7e8fae18f252d3f8adb8c \ + --hash=sha256:7a2f8cf9d73f07b6628093b35e5c5fbac039247e32cb075eaa5289a5914e73af \ + --hash=sha256:860ffa4513b2dc3083d8e412bd815a350b0a9490624b37e7c8f6ed5c6f9ce63c \ + --hash=sha256:2760a04f06937b01b1562a2135cd7e8207e399e73ef75bbebc77e37b1ad3b15d \ + --hash=sha256:fd00fd8a51c65aa19c40539431cb1773d87c30af5757b4041fa6c229058420b4 \ + --hash=sha256:29e257612fa8f0c1fcbc4e7e08ddc561169f1725265302e64d81086354144a70 diff --git a/plugins/code-tidying/evals/fixtures/seed.sh b/plugins/code-tidying/evals/fixtures/seed.sh index 0375c60180b..52f6007686a 100755 --- a/plugins/code-tidying/evals/fixtures/seed.sh +++ b/plugins/code-tidying/evals/fixtures/seed.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash -# Seeds the eval workspace with fixtures/.txt copied to , committed once. +# Seeds the eval workspace with fixtures/.txt copied to , committed once, and the +# run's home with the comment tooling (comment-tooling-seed.sh). # Usage: seed.sh ... set -euo pipefail fixtures="$(dirname "${BASH_SOURCE[0]}")" +bash "$fixtures/comment-tooling-seed.sh" git init -q for name in "$@"; do mkdir -p "$(dirname "$name")" diff --git a/plugins/code-tidying/evals/probe-environment/case.yaml b/plugins/code-tidying/evals/probe-environment/case.yaml index a01316d6bd7..fa9a9fceac3 100644 --- a/plugins/code-tidying/evals/probe-environment/case.yaml +++ b/plugins/code-tidying/evals/probe-environment/case.yaml @@ -1,8 +1,10 @@ schema_version: "1.1" name: probe-environment -description: Confirms the eval run resolves a python3 with the comment-tooling wheels and a make binary +description: Confirms the sandbox's python3 imports the comment-tooling wheels the scaffold installs, and a make binary resolves tags: [probe] runs: 1 +context: + scaffold_script: scaffold.sh execution: prompt: "Run this exact command with the Bash tool and report its full output verbatim: command -v python3; python3 -c 'import pygments, tree_sitter, tree_sitter_bash' && echo imports-$((40+2)); command -v make; echo PATH=$PATH" allowed_tools: [Bash] diff --git a/plugins/code-tidying/evals/probe-environment/scaffold.sh b/plugins/code-tidying/evals/probe-environment/scaffold.sh new file mode 100755 index 00000000000..0d452e27702 --- /dev/null +++ b/plugins/code-tidying/evals/probe-environment/scaffold.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec bash "$(dirname "${BASH_SOURCE[0]}")/../fixtures/comment-tooling-seed.sh" From 62168e5804d1f65b799ecf5276cd0aacc8b78ade Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:00:00 -0500 Subject: [PATCH 2/6] chore(code-tidying): add the changelog fragment for the eval tooling seed Refs: #12703 Co-Authored-By: Claude Opus 5.5 --- .../test-12703-comment-rules-measurable-5b1e7c2d.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/code-tidying/test-12703-comment-rules-measurable-5b1e7c2d.md diff --git a/.changes/code-tidying/test-12703-comment-rules-measurable-5b1e7c2d.md b/.changes/code-tidying/test-12703-comment-rules-measurable-5b1e7c2d.md new file mode 100644 index 00000000000..8ad6fdada3a --- /dev/null +++ b/.changes/code-tidying/test-12703-comment-rules-measurable-5b1e7c2d.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +Eval fixtures only: the `remove-comments` eval cases now install the hash-pinned pygments and tree-sitter wheels into the run's home, so `change-shape.py` certifies deletions inside the eval sandbox and the cases measure applied deletions rather than proposals. The skills and everything else the plugin ships are unchanged. From 0581ad5bb6121e316b15cc86a178084d2687659c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:42:30 -0500 Subject: [PATCH 3/6] test(code-tidying): fail when the eval tooling pins drift from the CI pins comment-tooling-pins.test.sh compares every requirement in the eval seed's comment-tooling.requirements.txt with the same package in .github/requirements-ci.txt, version and hash set, and declares both files in its test-scope so a change to either selects it. Refs: #12703 Co-Authored-By: Claude Opus 5.5 --- .../scripts/comment-tooling-pins.test.sh | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 plugins/code-tidying/scripts/comment-tooling-pins.test.sh diff --git a/plugins/code-tidying/scripts/comment-tooling-pins.test.sh b/plugins/code-tidying/scripts/comment-tooling-pins.test.sh new file mode 100755 index 00000000000..7051b744e2e --- /dev/null +++ b/plugins/code-tidying/scripts/comment-tooling-pins.test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Contract: every requirement in the eval seed's comment-tooling.requirements.txt +# matches the same package's requirement in .github/requirements-ci.txt, version +# and hash set, so a pin bumped on the reviewed surface cannot leave the eval +# seed installing the old one. +# Usage: comment-tooling-pins.test.sh [ ] +# test-scope: plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt .github/requirements-ci.txt +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../../.." && pwd)" +eval_req="${1:-$root/plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt}" +ci_req="${2:-$root/.github/requirements-ci.txt}" + +# One line per requirement: continuations joined, comments dropped, hashes sorted. +requirements() { + awk ' + /^[[:space:]]*#/ { next } + { sub(/[[:space:]]*\\$/, ""); gsub(/^[[:space:]]+/, ""); if ($0 == "") next } + /^--hash=/ { req = req " " $0; next } + { if (req != "") print req; req = $0 } + END { if (req != "") print req } + ' "$1" | while read -r spec hashes; do + printf '%s %s\n' "$spec" "$(tr ' ' '\n' <<<"$hashes" | sort | paste -sd ' ')" + done +} + +ci="$(requirements "$ci_req")" +rc=0 +count=0 +while read -r spec rest; do + count=$((count + 1)) + name="${spec%%==*}" + match="$(grep -E "^${name//./\\.}==" <<<"$ci" || true)" + if [[ -z "$match" ]]; then + printf 'FAIL: %s is not pinned in %s\n' "$name" "$ci_req" >&2 + rc=1 + elif [[ "$match" != "$spec $rest" ]]; then + printf 'FAIL: %s differs from %s (version or hashes)\n' "$name" "$ci_req" >&2 + rc=1 + fi +done < <(requirements "$eval_req") +if [[ "$count" -eq 0 ]]; then + printf 'FAIL: no requirement read from %s\n' "$eval_req" >&2 + rc=1 +fi +[[ "$rc" -eq 0 ]] && printf 'ok: %d comment-tooling pins match %s\n' "$count" "$ci_req" +exit "$rc" From 2fe5cd58cb77be43173b0a2cafe00b39954a4930 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:04:54 -0500 Subject: [PATCH 4/6] test(code-tidying): install the comment tooling for each interpreter the sandbox may run pip --user writes to the invoking interpreter's own user site, so a host whose /usr/bin/python3 (the sandbox's fallback) differs in minor version from the scaffold's python3 got wheels the agent never imports. The seed now installs and checks the import for each distinct minor version of the two. Refs: #12703 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015te8CoSUhxDqvddaTSwBz8 --- .../evals/fixtures/comment-tooling-seed.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh index aef37cdd121..075698e6093 100755 --- a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh +++ b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh @@ -9,7 +9,18 @@ set -euo pipefail here="$(dirname "${BASH_SOURCE[0]}")" -python3 -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \ - --no-deps --no-cache-dir --disable-pip-version-check --no-warn-script-location --quiet \ - -r "$here/comment-tooling.requirements.txt" -python3 -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python' +# --user installs into the invoking interpreter's own user site, so install for each interpreter +# the agent may resolve: the scaffold's python3 and /usr/bin/python3, which the sandbox falls back +# to when the operator's interpreter is hidden. Two of the same minor version share one site. +seen="" +for py in python3 /usr/bin/python3; do + command -v "$py" >/dev/null 2>&1 || continue + ver="$("$py" -c 'import sys; print("%d.%d" % sys.version_info[:2])')" + [[ " $seen " == *" $ver "* ]] && continue + seen="$seen $ver" + "$py" -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \ + --no-deps --no-cache-dir --disable-pip-version-check --no-warn-script-location --quiet \ + -r "$here/comment-tooling.requirements.txt" + "$py" -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python' +done +[[ -n "$seen" ]] From 495dff5908824cdd2468d57b50602a4c340c0ea8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:19:42 -0500 Subject: [PATCH 5/6] test(code-tidying): require the tooling only for the scaffold's python3 The fallback /usr/bin/python3 can be a minor the pinned wheels do not cover (the Xcode 3.9 shim, Ubuntu's 3.12), and failing on it failed every seeded case on hosts where the agent never runs it. It is now best-effort with a warning that names the supported minors; probe-environment still catches a fallback that never got the tooling. Refs: #12703 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015te8CoSUhxDqvddaTSwBz8 --- .../evals/fixtures/comment-tooling-seed.sh | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh index 075698e6093..f1a41e75297 100755 --- a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh +++ b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh @@ -9,18 +9,24 @@ set -euo pipefail here="$(dirname "${BASH_SOURCE[0]}")" -# --user installs into the invoking interpreter's own user site, so install for each interpreter -# the agent may resolve: the scaffold's python3 and /usr/bin/python3, which the sandbox falls back -# to when the operator's interpreter is hidden. Two of the same minor version share one site. -seen="" -for py in python3 /usr/bin/python3; do - command -v "$py" >/dev/null 2>&1 || continue - ver="$("$py" -c 'import sys; print("%d.%d" % sys.version_info[:2])')" - [[ " $seen " == *" $ver "* ]] && continue - seen="$seen $ver" - "$py" -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \ +# --user installs into the invoking interpreter's own user site. The scaffold's python3 must take +# the tooling: a failure there fails the run. /usr/bin/python3, the sandbox's fallback when the +# operator's interpreter is hidden, is best-effort: a failure there only warns, and the in-sandbox +# probe-environment case catches a fallback that never got it. The pinned tree-sitter wheels cover +# CPython 3.11 and 3.14 only (.github/requirements-ci.txt), so another minor fails at the install. +install() { + "$1" -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \ --no-deps --no-cache-dir --disable-pip-version-check --no-warn-script-location --quiet \ - -r "$here/comment-tooling.requirements.txt" - "$py" -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python' -done -[[ -n "$seen" ]] + -r "$here/comment-tooling.requirements.txt" && + "$1" -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python' +} +minor() { "$1" -c 'import sys; print("%d.%d" % sys.version_info[:2])'; } + +install python3 +[[ -x /usr/bin/python3 ]] || exit 0 +own="$(minor python3)" +fallback="$(minor /usr/bin/python3)" +[[ "$fallback" != "$own" ]] || exit 0 +# shellcheck disable=SC2310 # best-effort by design: a failure only warns +install /usr/bin/python3 || + echo "comment-tooling-seed: /usr/bin/python3 $fallback did not take the tooling (wheels cover 3.11 and 3.14); continuing" >&2 From 4c9681f11d87140db5b2ad54d2294a9497561025 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:40:34 -0500 Subject: [PATCH 6/6] test(code-tidying): skip a fallback python3 that does not run The version probe on /usr/bin/python3 ran under set -e, so the macOS xcrun stub (executable, exits 1 without the Command Line Tools) still failed the seed. A fallback that does not run now warns and is skipped. Refs: #12703 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015te8CoSUhxDqvddaTSwBz8 --- plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh index f1a41e75297..8aad037f8ab 100755 --- a/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh +++ b/plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh @@ -25,7 +25,11 @@ minor() { "$1" -c 'import sys; print("%d.%d" % sys.version_info[:2])'; } install python3 [[ -x /usr/bin/python3 ]] || exit 0 own="$(minor python3)" -fallback="$(minor /usr/bin/python3)" +# shellcheck disable=SC2310 # best-effort by design: a fallback that does not run is skipped +fallback="$(minor /usr/bin/python3)" || { + echo "comment-tooling-seed: /usr/bin/python3 does not run; skipping the fallback" >&2 + exit 0 +} [[ "$fallback" != "$own" ]] || exit 0 # shellcheck disable=SC2310 # best-effort by design: a failure only warns install /usr/bin/python3 ||