test(code-tidying): install the comment tooling in eval runs so body rules are measurable

.changes/code-tidying/test-12703-comment-rules-measurable-5b1e7c2d.md

@@ -0,0 +1,5 @@
+---
+bump: none
+---
+
+Eval fixtures only: the `remove-comments` eval cases now install the hash-pinned pygments and tree-sitter wheels into the run's home, so `change-shape.py` certifies deletions inside the eval sandbox and the cases measure applied deletions rather than proposals. The skills and everything else the plugin ships are unchanged.

plugins/code-tidying/evals/fixtures/comment-tooling-seed.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Installs pygments and tree-sitter with its grammars into the run's user site, so the agent's
+# python3 imports them and change-shape.py can certify a deletion. The eval sandbox hides the
+# operator's home, with any interpreter or package installed there, and has no network for an
+# install of its own; the scaffold shares the run's HOME with the agent and has the network.
+# Hash-pinned wheels only, so no build code runs. Exits non-zero when the import still fails,
+# so no run is scored that could only propose deletions.
+# Usage: comment-tooling-seed.sh
+set -euo pipefail
+
+here="$(dirname "${BASH_SOURCE[0]}")"
+# --user installs into the invoking interpreter's own user site. The scaffold's python3 must take
+# the tooling: a failure there fails the run. /usr/bin/python3, the sandbox's fallback when the
+# operator's interpreter is hidden, is best-effort: a failure there only warns, and the in-sandbox
+# probe-environment case catches a fallback that never got it. The pinned tree-sitter wheels cover
+# CPython 3.11 and 3.14 only (.github/requirements-ci.txt), so another minor fails at the install.
+install() {
+  "$1" -m pip install --user --break-system-packages --require-hashes --only-binary=:all: \
+    --no-deps --no-cache-dir --disable-pip-version-check --no-warn-script-location --quiet \
+    -r "$here/comment-tooling.requirements.txt" &&
+    "$1" -c 'import pygments, tree_sitter, tree_sitter_bash, tree_sitter_python'
+}
+minor() { "$1" -c 'import sys; print("%d.%d" % sys.version_info[:2])'; }
+
+install python3
+[[ -x /usr/bin/python3 ]] || exit 0
+own="$(minor python3)"
+# shellcheck disable=SC2310 # best-effort by design: a fallback that does not run is skipped
+fallback="$(minor /usr/bin/python3)" || {
+  echo "comment-tooling-seed: /usr/bin/python3 does not run; skipping the fallback" >&2
+  exit 0
+}
+[[ "$fallback" != "$own" ]] || exit 0
+# shellcheck disable=SC2310 # best-effort by design: a failure only warns
+install /usr/bin/python3 ||
+  echo "comment-tooling-seed: /usr/bin/python3 $fallback did not take the tooling (wheels cover 3.11 and 3.14); continuing" >&2

plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt

@@ -0,0 +1,82 @@
+# The pygments and tree-sitter pins of .github/requirements-ci.txt, which stays the reviewed
+# surface: move these lines with it.
+pygments==2.21.0 \
+    --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \
+    --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c
+tree-sitter==0.26.0 \
+    --hash=sha256:1d6fe0e8fb4df77b5ee816228e2c4475a63d8cc1d4d3a7ffd7097b2b87fc3e95 \
+    --hash=sha256:514a9bf8993e5210e7970736aaf6020d1759b670e195ef17b1c48f586aa30736 \
+    --hash=sha256:10f0d4eb94aa7242dcb7f554bcd24dd7ba1c114f00d58759ba08c7a46c8ec51a \
+    --hash=sha256:335294ce0504fcefde5245dff596778ffaf820205b98ae0b549c72e48855f1d8 \
+    --hash=sha256:c56581ad256c4195a21bfe449fed5d44a02fe83a4a7d6e70e6ec302c881191c7 \
+    --hash=sha256:0f8793fd18ad7eec276ed4b51c097b4bf2002b357259b66b0d75db1f3f41c754 \
+    --hash=sha256:dea4b4e27d49e9ec5b785d4f994da000e6726882fcc6ad05ec98478500c71aef \
+    --hash=sha256:5a3c93a352b7e6f70f73e121bbfa2d0117ba7478bd51114ed35c91b0b78814fa \
+    --hash=sha256:5fc2f41bf246ff2f70a9cc3690be35ec7580a4923151873d898c8bcb1a4503d3 \
+    --hash=sha256:b8ea92a255c91671a7ec4625aba3ab7bb5220c423630ffbf83c45d7312abe084 \
+    --hash=sha256:f665510f0fcf4636fb9696f1f7853bed7a3bd764b7bb0cb8494e619c14ed5a0c \
+    --hash=sha256:ff80d4833d330a73184a3ac5132abe93c575d2dea31975c6f15c0d21fef238aa \
+    --hash=sha256:a4033fecc8f606c7f2e8b8014d0057b74668a7f0152763606f7bc25c5f9ec64c \
+    --hash=sha256:823251c4b6725a7c03ed497a339135ede7ae4bdde75bb8be7ef5e305aeb4ff52
+tree-sitter-python==0.25.0 \
+    --hash=sha256:14a79a47ddef72f987d5a2c122d148a812169d7484ff5c75a3db9609d419f361 \
+    --hash=sha256:480c21dbd995b7fe44813e741d71fed10ba695e7caab627fb034e3828469d762 \
+    --hash=sha256:86f118e5eecad616ecdb81d171a36dde9bef5a0b21ed71ea9c3e390813c3baf5 \
+    --hash=sha256:be71650ca2b93b6e9649e5d65c6811aad87a7614c8c1003246b303f6b150f61b \
+    --hash=sha256:e6d5b5799628cc0f24691ab2a172a8e676f668fe90dc60468bee14084a35c16d \
+    --hash=sha256:71959832fc5d9642e52c11f2f7d79ae520b461e63334927e93ca46cd61cd9683 \
+    --hash=sha256:9bcde33f18792de54ee579b00e1b4fe186b7926825444766f849bf7181793a76 \
+    --hash=sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb
+tree-sitter-c-sharp==0.23.5 \
+    --hash=sha256:61e1981cf21b09ee547b9c4c68e64fb4394325f8fc8d5f6d50d41471eba923ea \
+    --hash=sha256:a75994a11f6fed3f5b8c36ad6a00e5dc43205bd912c43af3a2a54fdf649664eb \
+    --hash=sha256:aa88a780204cd153c4c1ae2d59c654cee1402212fa0d069823d6d34301587438 \
+    --hash=sha256:3ea38fb095d85d360dc5a0bec2fa605e496228876f798c9e089d5f0e72bcef46 \
+    --hash=sha256:05a9256415e7f24d4f133133794a9c224c60d19f677a04e2f6a94c25090b6d65 \
+    --hash=sha256:8636dc70b5a373c35c1036ed5de98e801f2e4d105ae41e2e20b6804c36e3bf33 \
+    --hash=sha256:41a28cfa3d9ea50f5629e44550a03188c8fbd5079803dfc03554b6fd594b33fa \
+    --hash=sha256:2de4ebf95ddc2e92cd3105c8a8e0e7ec646bc82f52bfaf2f3acec0fa2401ec09
+tree-sitter-typescript==0.23.2 \
+    --hash=sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478 \
+    --hash=sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8 \
+    --hash=sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31 \
+    --hash=sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c \
+    --hash=sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0 \
+    --hash=sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9 \
+    --hash=sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7
+tree-sitter-bash==0.25.1 \
+    --hash=sha256:0e6235f59e366d220dde7d830196bed597d01e853e44d8ccd1a82c5dd2500acf \
+    --hash=sha256:f4a34a6504c7c5b2a9b8c5c4065531dea19ca2c35026e706cf2eeeebe2c92512 \
+    --hash=sha256:e76c4cfb20b076552406782b7f8c2a3946835993df0a44df006de54b7030c7dc \
+    --hash=sha256:3f484c4bb8796cde7a87ca351e6116f09653edac0eb3c6d238566359dd28b117 \
+    --hash=sha256:5e76af6df46d958c7f5b6d5884c9743218e3902a00ccb493ec92728b1084430b \
+    --hash=sha256:a3332d71c7b7d5f78259b19d02d0ea111fcb82b72712ee4a93aaa5b226d3f0a8 \
+    --hash=sha256:52a6802d9218f86278aa3e8b459c3abdad67eed0fde1f9f13aca5b6c634217a6 \
+    --hash=sha256:59115057ec2bae319e8082ff29559861045002964c3431ccb0fc92aa4bc9bccb
+tree-sitter-javascript==0.25.0 \
+    --hash=sha256:b70f887fb269d6e58c349d683f59fa647140c410cfe2bee44a883b20ec92e3dc \
+    --hash=sha256:8264a996b8845cfce06965152a013b5d9cbb7d199bc3503e12b5682e62bb1de1 \
+    --hash=sha256:9dc04ba91fc8583344e57c1f1ed5b2c97ecaaf47480011b92fbeab8dda96db75 \
+    --hash=sha256:199d09985190852e0912da2b8d26c932159be314bc04952cf917ed0e4c633e6b \
+    --hash=sha256:dfcf789064c58dc13c0a4edb550acacfc6f0f280577f1e7a00de3e89fc7f8ddc \
+    --hash=sha256:1b852d3aee8a36186dbcc32c798b11b4869f9b5041743b63b65c2ef793db7a54 \
+    --hash=sha256:e5ed840f5bd4a3f0272e441d19429b26eedc257abe5574c8546da6b556865e3c \
+    --hash=sha256:622a69d677aa7f6ee2931d8c77c981a33f0ebb6d275aa9d43d3397c879a9bb0b
+tree-sitter-yaml==0.7.2 \
+    --hash=sha256:7e269ddcfcab8edb14fbb1f1d34eed1e1e26888f78f94eedfe7cc98c60f8bc9f \
+    --hash=sha256:0807b7966e23ddf7dddc4545216e28b5a58cdadedcecca86b8d8c74271a07870 \
+    --hash=sha256:f1a5c60c98b6c4c037aae023569f020d0c489fad8dc26fdfd5510363c9c29a41 \
+    --hash=sha256:88636d19d0654fd24f4f242eaaafa90f6f5ebdba8a62e4b32d251ed156c51a2a \
+    --hash=sha256:1d2e8f0bb14aa4537320952d0f9607eef3021d5aada8383c34ebeece17db1e06 \
+    --hash=sha256:74ca712c50fc9d7dbc68cb36b4a7811d6e67a5466b5a789f19bf8dd6084ef752 \
+    --hash=sha256:7587b5ca00fc4f9a548eff649697a3b395370b2304b399ceefa2087d8a6c9186 \
+    --hash=sha256:f63c227b18e7ce7587bce124578f0bbf1f890ac63d3e3cd027417574273642c4
+tree-sitter-toml==0.7.0 \
+    --hash=sha256:b9ae5c3e7c5b6bb05299dd73452ceafa7fa0687d5af3012332afa7757653b676 \
+    --hash=sha256:18be09538e9775cddc0290392c4e2739de2201260af361473ca60b5c21f7bd22 \
+    --hash=sha256:a045e0acfcf91b7065066f7e51ea038ed7385c1e35e7e8fae18f252d3f8adb8c \
+    --hash=sha256:7a2f8cf9d73f07b6628093b35e5c5fbac039247e32cb075eaa5289a5914e73af \
+    --hash=sha256:860ffa4513b2dc3083d8e412bd815a350b0a9490624b37e7c8f6ed5c6f9ce63c \
+    --hash=sha256:2760a04f06937b01b1562a2135cd7e8207e399e73ef75bbebc77e37b1ad3b15d \
+    --hash=sha256:fd00fd8a51c65aa19c40539431cb1773d87c30af5757b4041fa6c229058420b4 \
+    --hash=sha256:29e257612fa8f0c1fcbc4e7e08ddc561169f1725265302e64d81086354144a70

plugins/code-tidying/evals/fixtures/seed.sh

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
-# Seeds the eval workspace with fixtures/<name>.txt copied to <name>, committed once.
+# Seeds the eval workspace with fixtures/<name>.txt copied to <name>, committed once, and the
+# run's home with the comment tooling (comment-tooling-seed.sh).
 # Usage: seed.sh <name>...
 set -euo pipefail
 
 fixtures="$(dirname "${BASH_SOURCE[0]}")"
+bash "$fixtures/comment-tooling-seed.sh"
 git init -q
 for name in "$@"; do
   mkdir -p "$(dirname "$name")"

plugins/code-tidying/evals/probe-environment/case.yaml

@@ -1,8 +1,10 @@
 schema_version: "1.1"
 name: probe-environment
-description: Confirms the eval run resolves a python3 with the comment-tooling wheels and a make binary
+description: Confirms the sandbox's python3 imports the comment-tooling wheels the scaffold installs, and a make binary resolves
 tags: [probe]
 runs: 1
+context:
+  scaffold_script: scaffold.sh
 execution:
   prompt: "Run this exact command with the Bash tool and report its full output verbatim: command -v python3; python3 -c 'import pygments, tree_sitter, tree_sitter_bash' && echo imports-$((40+2)); command -v make; echo PATH=$PATH"
   allowed_tools: [Bash]

plugins/code-tidying/evals/probe-environment/scaffold.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exec bash "$(dirname "${BASH_SOURCE[0]}")/../fixtures/comment-tooling-seed.sh"

plugins/code-tidying/scripts/comment-tooling-pins.test.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Contract: every requirement in the eval seed's comment-tooling.requirements.txt
+# matches the same package's requirement in .github/requirements-ci.txt, version
+# and hash set, so a pin bumped on the reviewed surface cannot leave the eval
+# seed installing the old one.
+# Usage: comment-tooling-pins.test.sh [<eval requirements> <ci requirements>]
+# test-scope: plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt .github/requirements-ci.txt
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+root="$(cd "$here/../../.." && pwd)"
+eval_req="${1:-$root/plugins/code-tidying/evals/fixtures/comment-tooling.requirements.txt}"
+ci_req="${2:-$root/.github/requirements-ci.txt}"
+
+# One line per requirement: continuations joined, comments dropped, hashes sorted.
+requirements() {
+  awk '
+    /^[[:space:]]*#/ { next }
+    { sub(/[[:space:]]*\\$/, ""); gsub(/^[[:space:]]+/, ""); if ($0 == "") next }
+    /^--hash=/ { req = req " " $0; next }
+    { if (req != "") print req; req = $0 }
+    END { if (req != "") print req }
+  ' "$1" | while read -r spec hashes; do
+    printf '%s %s\n' "$spec" "$(tr ' ' '\n' <<<"$hashes" | sort | paste -sd ' ')"
+  done
+}
+
+ci="$(requirements "$ci_req")"
+rc=0
+count=0
+while read -r spec rest; do
+  count=$((count + 1))
+  name="${spec%%==*}"
+  match="$(grep -E "^${name//./\\.}==" <<<"$ci" || true)"
+  if [[ -z "$match" ]]; then
+    printf 'FAIL: %s is not pinned in %s\n' "$name" "$ci_req" >&2
+    rc=1
+  elif [[ "$match" != "$spec $rest" ]]; then
+    printf 'FAIL: %s differs from %s (version or hashes)\n' "$name" "$ci_req" >&2
+    rc=1
+  fi
+done < <(requirements "$eval_req")
+if [[ "$count" -eq 0 ]]; then
+  printf 'FAIL: no requirement read from %s\n' "$eval_req" >&2
+  rc=1
+fi
+[[ "$rc" -eq 0 ]] && printf 'ok: %d comment-tooling pins match %s\n' "$count" "$ci_req"
+exit "$rc"