From efe5adb2220e729e9225f0d0d2e3f5160b8f21cf Mon Sep 17 00:00:00 2001 From: Chris Henesy Date: Mon, 5 Oct 2026 13:53:01 -0400 Subject: [PATCH 1/4] test(failing): 0064 model roles observations Add tests/model-roles.test.sh: executes every scripts/lib/models.sh function in a clean env -i shell against fixture TDDs, a Claude transcript and a Grok summary.json (TDD 0064 Verification plan observations 1-15), and pins the fresh-worker independence wording in the design-reviewer, /tdd-author 7b and plugin.json. Register it in the implement-gate aggregator. build-tdds-skill [D] now asserts the no-arg resolve line build=inherit review=inherit verify=inherit. Red against the unchanged models.sh (prior-gen pairing). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ki3TDnP2bNqqpEkFoZaEXq --- tests/build-tdds-skill.test.sh | 13 +- tests/implement-gate.test.sh | 8 +- tests/model-roles.test.sh | 248 +++++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 8 deletions(-) create mode 100755 tests/model-roles.test.sh diff --git a/tests/build-tdds-skill.test.sh b/tests/build-tdds-skill.test.sh index 500458f..411ccd8 100755 --- a/tests/build-tdds-skill.test.sh +++ b/tests/build-tdds-skill.test.sh @@ -50,18 +50,21 @@ echo "[C] skill names require_flip, next_gate, tokens, no nested spawn" || bad "missing 'must not spawn children'" ) || true -# --- [D] models.sh is the ADR 0009 binding site ------------------------------ -echo "[D] models.sh binds tl_resolve_models" +# --- [D] models.sh is the ADR 0015 binding site (TDD 0064) ------------------- +# No pins (incl. CLAUDE_CODE_SUBAGENT_MODEL), no session id, no args → every +# judgment slot inherits the parent session's model. +echo "[D] models.sh binds tl_resolve_models (inherit by default)" ( bash -n "$MODELS" 2>"$ROOT/D.err" \ && ok "models.sh parses" \ || bad "models.sh bash -n failed: $(cat "$ROOT/D.err" 2>/dev/null)" if env -u GROK_PLUGIN_ROOT -u THROUGHLINE_BUILD_MODEL \ -u THROUGHLINE_REVIEW_MODEL -u THROUGHLINE_RUNTIME_VERIFY_MODEL \ + -u CLAUDE_CODE_SUBAGENT_MODEL -u CLAUDE_CODE_SESSION_ID \ bash -c "source \"$MODELS\"; tl_resolve_models" >"$ROOT/D.out" 2>"$ROOT/D2.err"; then out="$(cat "$ROOT/D.out")" - printf '%s' "$out" | grep -qE '^build=.+ review=.+ verify=.+$' \ - && ok "prints build= review= verify= ($out)" \ - || bad "unexpected resolve line: $out" + [ "$out" = "build=inherit review=inherit verify=inherit" ] \ + && ok "no-arg line is build=inherit review=inherit verify=inherit" \ + || bad "unexpected resolve line: '$out' want 'build=inherit review=inherit verify=inherit'" else bad "tl_resolve_models failed: $(cat "$ROOT/D2.err")" fi diff --git a/tests/implement-gate.test.sh b/tests/implement-gate.test.sh index 7ba2bca..05b45c34 100755 --- a/tests/implement-gate.test.sh +++ b/tests/implement-gate.test.sh @@ -6,7 +6,8 @@ # # Coprocess/runner evals were removed with the implement.sh body (TDD 0063+). # This aggregator keeps the 0063 stub/status observations and the live -# 0060–0062 helpers: plugin-root, verdicts, run-record, build-tdds-skill. +# 0060–0062 helpers: plugin-root, verdicts, run-record, build-tdds-skill, +# and the 0064 model-roles eval. # # Run: bash tests/implement-gate.test.sh set -uo pipefail @@ -189,9 +190,10 @@ PASS="$(grep -c '^ok$' "$RESULTS" 2>/dev/null)"; PASS="${PASS:-0}" FAIL="$(grep -c '^fail$' "$RESULTS" 2>/dev/null)"; FAIL="${FAIL:-0}" echo "=== 0063 observations: $PASS passed, $FAIL failed ===" -# 0060–0062 evals (kept). Each file is invoked only when it exists. +# 0060–0062 evals (kept) + 0064 model-roles. Each file is invoked only when +# it exists. EVAL_FAIL=0 -for ev in plugin-root verdicts run-record build-tdds-skill; do +for ev in plugin-root verdicts run-record build-tdds-skill model-roles; do f="$(dirname "$0")/${ev}.test.sh" if [ -f "$f" ]; then echo diff --git a/tests/model-roles.test.sh b/tests/model-roles.test.sh new file mode 100755 index 0000000..a56444b --- /dev/null +++ b/tests/model-roles.test.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# model-roles.test.sh — eval for TDD 0064 / NFR-3, FR-52, FR-87, FR-10, +# FR-15(d), FR-50: EXECUTES every scripts/lib/models.sh function in a clean +# `env -i HOME= PATH="$PATH" bash -c '. models.sh; …'` shell against +# fixture TDDs, Claude transcripts and a Grok summary.json (no network), and +# pins the fresh-worker independence wording (design-reviewer, /tdd-author 7b, +# plugin.json). Observation numbers [1]–[15] are the TDD's Verification plan. +# +# Written red-first. Run: bash tests/model-roles.test.sh +set -uo pipefail +REPO="$(cd "$(dirname "$0")/.." && pwd)" +MODELS="$REPO/scripts/lib/models.sh" +DR="$REPO/agents/design-reviewer.md" +TA="$REPO/skills/tdd-author/SKILL.md" +PJ="$REPO/.claude-plugin/plugin.json" +RESULTS="$(mktemp)"; export RESULTS +ok() { printf 'ok\n' >>"$RESULTS"; printf ' ok — %s\n' "$1"; } +bad() { printf 'fail\n' >>"$RESULTS"; printf ' FAIL — %s\n' "$1"; } + +ROOT="$(mktemp -d)"; trap 'rm -rf "$ROOT" "$RESULTS"' EXIT +H="$ROOT/home"; mkdir -p "$H" +q() { printf '%q' "$1"; } + +# m [VAR=val ...] '' — source models.sh then run in a CLEAN shell +# (env -i: no caller variable or function leaks in). A later PATH=/HOME= in the +# VAR list overrides the default. Sets OUT, ERR, RC, NL (stdout line count). +m() { + local cmd="${!#}" + local -a envs=("${@:1:$#-1}") + env -i HOME="$H" PATH="$PATH" ${envs[@]+"${envs[@]}"} \ + bash -c ". $(q "$MODELS") || exit 97; $cmd" >"$ROOT/out" 2>"$ROOT/err" + RC=$? + OUT="$(cat "$ROOT/out")"; ERR="$(cat "$ROOT/err")" + NL="$(wc -l <"$ROOT/out" | tr -d ' ')" +} +# eq