diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 02ccdfd..680b3e7 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "throughline", - "version": "3.41.0", + "version": "3.42.0", "description": "Throughline plugin: a PRD/TDD/ADR design-doc pipeline with phase-gate PRs, layered as a thin governance overlay on top of the official superpowers/pr-review-toolkit plugins (it owns governance; they own discovery + engineering). /prd-author requires an observable acceptance criterion per new requirement. /tdd-author decides how many TDDs a PRD change needs (git-diff + coverage), recommends ADR actions, requires an alternatives analysis for every new dependency, requires a verification plan (observable surface → observation point(s) → expected observations) per TDD, reads BLOCKERS.md so implementation-time design blockers feed back into design, self-reviews, then runs an independent design-critique gate (different model, fresh context) before the design PR. /implement builds every TDD merged to the integration branch unattended in detached processes on the latest top-tier model but does not trust self-reported success: the flip to implemented is gated on failing-test-first discipline, a mechanical ci-checks gate (tests+typecheck+lint — CI's job), a runtime-verify gate that drives the built artifact at its observable surface (PASS/FAIL/BLOCKED/SKIP kept distinct; mechanism delegated, no harness vendored — ADR 0004), and an independent review on a DIFFERENT model (the prior generation's top tier) for reviewer diversity — one stacked PR per TDD, downstream halt-on-failure, never merges. Includes a toolchain bootstrap skill and a format+lint hook for JS/TS, Python, Rust, and Go.", "author": { "name": "Chris Henesy" diff --git a/docs/tdd/0061-file-backed-gate-verdicts.md b/docs/tdd/0061-file-backed-gate-verdicts.md index f064bf6..7568d16 100644 --- a/docs/tdd/0061-file-backed-gate-verdicts.md +++ b/docs/tdd/0061-file-backed-gate-verdicts.md @@ -1,6 +1,7 @@ # TDD 0061: File-backed four-gate verdicts and durable run record -Status: draft +Status: implemented +build-engine: bootstrap PRD refs: FR-15, FR-16, FR-25, FR-27, FR-39, FR-40, FR-41, FR-42, FR-43, FR-44, FR-45, FR-63, FR-64, FR-67, FR-70, FR-82 PRD-rev: 373cd89 ADR constraints: 0004, 0005, 0006, 0011, 0013 diff --git a/scripts/lib/run-record.sh b/scripts/lib/run-record.sh new file mode 100644 index 0000000..ac443bd --- /dev/null +++ b/scripts/lib/run-record.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# run-record.sh — durable /build-tdds run record (TDD 0061 / FR-27, FR-39, +# FR-40, FR-43, FR-18). Queue, status, halt, lock, next-gate. Does not +# read a harness transcript. No top-level side effects. +# +# Every tl_run_* takes first (absolute). Never uses cwd as +# the logs root. + +_TL_RUN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib/verdicts.sh +{ [ -r "${_TL_RUN_DIR}/verdicts.sh" ] && . "${_TL_RUN_DIR}/verdicts.sh"; } || { + echo "FATAL: cannot source ${_TL_RUN_DIR}/verdicts.sh" >&2 + return 1 2>/dev/null || exit 1 +} + +_tl_run_abs() { + case "${1:-}" in + /*) return 0 ;; + *) echo "run-record: must be an absolute path" >&2; return 2 ;; + esac +} + +_tl_run_logs() { # + _tl_run_abs "$1" || return $? + printf '%s/docs/tdd/.implement-logs\n' "$1" +} + +_tl_run_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# Atomic write: printf to mktemp then mv. A kill -9 before mv leaves the +# previous file intact (FR-44). +_tl_run_atomic() { # + local dest="$1" body="$2" tmp dir + dir="$(dirname "$dest")" + mkdir -p "$dir" || return 1 + tmp="$(mktemp "$dir/.$(basename "$dest").XXXXXX")" || return 1 + if ! printf '%s\n' "$body" >"$tmp"; then rm -f "$tmp"; return 1; fi + if ! mv "$tmp" "$dest"; then rm -f "$tmp"; return 1; fi + return 0 +} + +_tl_run_json() { # + printf '{"run_id":"%s","status":"%s","started_at":"%s","updated_at":"%s","tdd_total":"%s","tdd_done":"%s"}' \ + "$(tl_json_escape "$1")" "$(tl_json_escape "$2")" \ + "$(tl_json_escape "$3")" "$(tl_json_escape "$4")" \ + "$(tl_json_escape "$5")" "$(tl_json_escape "$6")" +} + +tl_run_init() { # + local root="${1:-}" run="${2:-}" logs dir now + logs="$(_tl_run_logs "$root")" || return $? + _tl_valid_run "$run" || { echo "run-record: invalid run-id" >&2; return 2; } + dir="$logs/$run" + mkdir -p "$dir" || return 1 + now="$(_tl_run_now)" + _tl_run_atomic "$dir/run.json" "$(_tl_run_json "$run" running "$now" "$now" 0 0)" || return 1 + ln -sfn "$dir" "$logs/latest" || return 1 +} + +_tl_run_read_run() { # + [ -f "$1" ] || return 1 + tl_json_field "$2" <"$1" +} + +tl_run_set_tdd() { # [halt_cause] [current_gate] + local root="${1:-}" run="${2:-}" slug="${3:-}" status="${4:-}" + local cause="${5:-}" gate="${6:-}" logs rfile sfile now + local qidx started pr logp total done prev + logs="$(_tl_run_logs "$root")" || return $? + _tl_valid_run "$run" || { echo "run-record: invalid run-id" >&2; return 2; } + _tl_valid_slug "$slug" || { echo "run-record: invalid slug '$slug'" >&2; return 2; } + case "$status" in + pending|building|verifying|reviewing|done|failed|blocked|skipped|paused) ;; + *) echo "run-record: bad status '$status'" >&2; return 2 ;; + esac + if [ -n "$cause" ]; then + case "$cause" in + ratelimit|usage-limit|transient|resume-blocked-build-state-missing|resume-blocked-branch-missing|resume-blocked-branch-divergence|structural-finding|design-escalation|external-blocker|gate-fail) ;; + *) echo "run-record: bad halt_cause '$cause'" >&2; return 2 ;; + esac + fi + rfile="$logs/$run/run.json" + [ -f "$rfile" ] || { echo "run-record: run $run not initialized" >&2; return 1; } + sfile="$logs/$run/${slug}.json" + now="$(_tl_run_now)" + logp="$(tl_verdict_dir "$root" "$run" "$slug")" || return $? + total="$(_tl_run_read_run "$rfile" tdd_total)"; total="${total:-0}" + done="$(_tl_run_read_run "$rfile" tdd_done)"; done="${done:-0}" + if [ -f "$sfile" ]; then + qidx="$(tl_json_field queue_index <"$sfile")" + started="$(tl_json_field started_at <"$sfile")" + pr="$(tl_json_field pr_url <"$sfile")" + prev="$(tl_json_field status <"$sfile")" + else + qidx="$total" + started="$now" + pr="" + prev="" + total=$((total + 1)) + fi + if [ "$status" = "done" ] && [ "$prev" != "done" ]; then + done=$((done + 1)) + fi + _tl_run_atomic "$sfile" "$(printf \ + '{"slug":"%s","status":"%s","halt_cause":"%s","queue_index":"%s","current_gate":"%s","started_at":"%s","updated_at":"%s","pr_url":"%s","log_path":"%s"}' \ + "$(tl_json_escape "$slug")" "$(tl_json_escape "$status")" \ + "$(tl_json_escape "$cause")" "$(tl_json_escape "$qidx")" \ + "$(tl_json_escape "$gate")" "$(tl_json_escape "$started")" \ + "$(tl_json_escape "$now")" "$(tl_json_escape "$pr")" \ + "$(tl_json_escape "$logp")")" || return 1 + _tl_run_atomic "$rfile" "$(_tl_run_json "$run" \ + "$(_tl_run_read_run "$rfile" status || printf running)" \ + "$(_tl_run_read_run "$rfile" started_at || printf '%s' "$now")" \ + "$now" "$total" "$done")" || return 1 +} + +tl_run_set_pr() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" url="${4:-}" + local logs sfile now qidx started status cause gate logp + logs="$(_tl_run_logs "$root")" || return $? + sfile="$logs/$run/${slug}.json" + [ -f "$sfile" ] || { echo "run-record: no fragment for $slug" >&2; return 1; } + now="$(_tl_run_now)" + status="$(tl_json_field status <"$sfile")" + cause="$(tl_json_field halt_cause <"$sfile")" + qidx="$(tl_json_field queue_index <"$sfile")" + gate="$(tl_json_field current_gate <"$sfile")" + started="$(tl_json_field started_at <"$sfile")" + logp="$(tl_json_field log_path <"$sfile")" + _tl_run_atomic "$sfile" "$(printf \ + '{"slug":"%s","status":"%s","halt_cause":"%s","queue_index":"%s","current_gate":"%s","started_at":"%s","updated_at":"%s","pr_url":"%s","log_path":"%s"}' \ + "$(tl_json_escape "$slug")" "$(tl_json_escape "$status")" \ + "$(tl_json_escape "$cause")" "$(tl_json_escape "$qidx")" \ + "$(tl_json_escape "$gate")" "$(tl_json_escape "$started")" \ + "$(tl_json_escape "$now")" "$(tl_json_escape "$url")" \ + "$(tl_json_escape "$logp")")" +} + +_tl_lock_path() { printf '%s/.run.lock\n' "$(_tl_run_logs "$1")" || return $?; } + +tl_run_lock() { # + local root="${1:-}" lock pid logs + logs="$(_tl_run_logs "$root")" || return $? + mkdir -p "$logs" || return 1 + lock="$logs/.run.lock" + if [ -f "$lock" ]; then + pid="$(tr -d ' \n' <"$lock")" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + echo "run-record: lock held by live PID $pid" >&2 + return 1 + fi + fi + printf '%s\n' "$$" >"$lock" +} + +tl_run_unlock() { # + local lock + lock="$(_tl_lock_path "$1")" || return $? + rm -f "$lock" +} + +tl_run_lock_reclaim() { # + local root="${1:-}" lock pid logs + logs="$(_tl_run_logs "$root")" || return $? + mkdir -p "$logs" || return 1 + lock="$logs/.run.lock" + if [ -f "$lock" ]; then + pid="$(tr -d ' \n' <"$lock")" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + echo "run-record: cannot reclaim; PID $pid is live" >&2 + return 1 + fi + fi + printf '%s\n' "$$" >"$lock" +} + +tl_run_next_gate() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" g json st + if tl_verdict_require_flip "$root" "$run" "$slug" 2>/dev/null; then + printf 'flip\n' + return 0 + fi + for g in test-first ci-checks runtime-verify review; do + json="$(tl_verdict_read "$root" "$run" "$slug" "$g" 2>/dev/null)" || { printf '%s\n' "$g"; return 0; } + st="$(printf '%s' "$json" | tl_json_field status)" + case "$st" in + PASS|SKIP) ;; + *) printf '%s\n' "$g"; return 0 ;; + esac + done + # Files look complete but require_flip failed (e.g. SKIP without evidence). + printf 'runtime-verify\n' +} diff --git a/scripts/lib/verdicts.sh b/scripts/lib/verdicts.sh new file mode 100644 index 0000000..e5c6190 --- /dev/null +++ b/scripts/lib/verdicts.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# verdicts.sh — file-backed four-gate verdicts (TDD 0061 / FR-15, FR-25, FR-82, +# ADR 0011). Flip authority is four JSON files under +# /docs/tdd/.implement-logs///.json +# Never reads a harness transcript. No top-level side effects. +# +# Every tl_verdict_* / tl_test_first_observe takes an explicit path; none +# consult the process cwd as . + +_TL_VERDICTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_jlib="${_TL_VERDICTS_DIR}/json.sh" +# shellcheck source=scripts/lib/json.sh +{ [ -r "$_jlib" ] && . "$_jlib"; } || { + echo "FATAL: cannot source $_jlib (partial install or perms)" >&2 + return 1 2>/dev/null || exit 1 +} +unset _jlib + +_tl_verdict_gates='test-first|ci-checks|runtime-verify|review' +_tl_verdict_statuses='PASS|FAIL|BLOCKED|SKIP' + +# _tl_abs_root — rc 0 iff is a non-empty absolute directory +# path (need not exist yet). Rejects empty and relative so callers cannot +# silently write under cwd. +_tl_abs_root() { + case "${1:-}" in + /*) return 0 ;; + *) echo "verdicts: must be an absolute path" >&2; return 2 ;; + esac +} + +# _tl_valid_slug — ^[a-z0-9][a-z0-9.-]*$ +_tl_valid_slug() { + case "${1:-}" in + ''|*[!a-z0-9.-]*|.*|-*) return 1 ;; + esac + case "${1}" in + [a-z0-9]*) return 0 ;; + *) return 1 ;; + esac +} + +# _tl_valid_run — no slashes; non-empty path segment. +_tl_valid_run() { + case "${1:-}" in + ''|*/*|*..*) return 1 ;; + *) return 0 ;; + esac +} + +tl_verdict_dir() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" + _tl_abs_root "$root" || return $? + _tl_valid_run "$run" || { echo "verdicts: invalid run-id" >&2; return 2; } + _tl_valid_slug "$slug" || { echo "verdicts: invalid slug '$slug'" >&2; return 2; } + printf '%s/docs/tdd/.implement-logs/%s/%s\n' "$root" "$run" "$slug" +} + +tl_verdict_write() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" gate="${4:-}" status="${5:-}" evidence="${6:-}" + local dir f tmp + dir="$(tl_verdict_dir "$root" "$run" "$slug")" || return $? + case "$gate" in + test-first|ci-checks|runtime-verify|review) ;; + *) echo "verdicts: bad gate token '$gate' (want ${_tl_verdict_gates})" >&2; return 2 ;; + esac + case "$status" in + PASS|FAIL|BLOCKED|SKIP) ;; + *) echo "verdicts: bad status token '$status' (want ${_tl_verdict_statuses})" >&2; return 2 ;; + esac + mkdir -p "$dir" || { echo "verdicts: cannot mkdir $dir" >&2; return 1; } + f="$dir/${gate}.json" + tmp="$(mktemp "$dir/.${gate}.XXXXXX")" || { echo "verdicts: mktemp failed" >&2; return 1; } + if ! printf '{"gate":"%s","status":"%s","evidence":"%s"}\n' \ + "$(tl_json_escape "$gate")" \ + "$(tl_json_escape "$status")" \ + "$(tl_json_escape "$evidence")" >"$tmp"; then + rm -f "$tmp" + echo "verdicts: write failed for $f" >&2 + return 1 + fi + if ! mv "$tmp" "$f"; then + rm -f "$tmp" + echo "verdicts: mv failed for $f" >&2 + return 1 + fi + return 0 +} + +tl_verdict_read() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" gate="${4:-}" dir f + dir="$(tl_verdict_dir "$root" "$run" "$slug")" || return $? + case "$gate" in + test-first|ci-checks|runtime-verify|review) ;; + *) echo "verdicts: bad gate token '$gate'" >&2; return 2 ;; + esac + f="$dir/${gate}.json" + [ -f "$f" ] || return 1 + cat "$f" +} + +# tl_verdict_require_flip — 0 iff test-first, ci-checks, review are PASS +# and runtime-verify is PASS or SKIP with non-empty evidence. +tl_verdict_require_flip() { # + local root="${1:-}" run="${2:-}" slug="${3:-}" g json st ev + for g in test-first ci-checks review; do + json="$(tl_verdict_read "$root" "$run" "$slug" "$g")" || return 1 + st="$(printf '%s' "$json" | tl_json_field status)" + [ "$st" = "PASS" ] || return 1 + done + json="$(tl_verdict_read "$root" "$run" "$slug" runtime-verify)" || return 1 + st="$(printf '%s' "$json" | tl_json_field status)" + ev="$(printf '%s' "$json" | tl_json_field evidence)" + case "$st" in + PASS) return 0 ;; + SKIP) [ -n "$ev" ] && return 0; return 1 ;; + *) return 1 ;; + esac +} + +# _tl_integration_ref — print a resolvable integration ref, or rc 1. +_tl_integration_ref() { + local d="$1" cand + if [ -n "${THROUGHLINE_INTEGRATION_BRANCH:-}" ]; then + git -C "$d" rev-parse -q --verify "${THROUGHLINE_INTEGRATION_BRANCH}^{commit}" >/dev/null 2>&1 \ + && { printf '%s\n' "$THROUGHLINE_INTEGRATION_BRANCH"; return 0; } + fi + if git -C "$d" symbolic-ref -q refs/remotes/origin/HEAD >/dev/null 2>&1; then + cand="$(git -C "$d" symbolic-ref --short refs/remotes/origin/HEAD)" + git -C "$d" rev-parse -q --verify "${cand}^{commit}" >/dev/null 2>&1 \ + && { printf '%s\n' "$cand"; return 0; } + fi + for cand in main master; do + git -C "$d" rev-parse -q --verify "${cand}^{commit}" >/dev/null 2>&1 \ + && { printf '%s\n' "$cand"; return 0; } + done + return 1 +} + +# tl_test_first_observe — 0 iff unique commits vs the integration +# merge-base contain a test(failing): subject before the first feat: / step( +# subject. Empty range / feat-only / reversed order → rc 1. Git-history only. +tl_test_first_observe() { + local d="${1:-}" integ mb subj seen_tf=0 + [ -n "$d" ] && [ -d "$d" ] || { echo "verdicts: tl_test_first_observe needs a git dir" >&2; return 1; } + integ="$(_tl_integration_ref "$d")" || return 1 + mb="$(git -C "$d" merge-base "$integ" HEAD 2>/dev/null)" || return 1 + while IFS= read -r subj; do + case "$subj" in + 'test(failing):'*) seen_tf=1 ;; + feat:*|step\(*) + [ "$seen_tf" -eq 1 ] || return 1 + ;; + esac + done < <(git -C "$d" log --format=%s --reverse "${mb}..HEAD" 2>/dev/null) + [ "$seen_tf" -eq 1 ] +} diff --git a/tests/run-record.test.sh b/tests/run-record.test.sh new file mode 100755 index 0000000..df4b836 --- /dev/null +++ b/tests/run-record.test.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# run-record.test.sh — eval for TDD 0061 / FR-27, FR-39, FR-40, FR-43, FR-18: +# pins scripts/lib/run-record.sh (init, set_tdd, lock reclaim, next_gate, +# atomic write). +# +# Written red-first: before TDD 0061 lands, run-record.sh does not exist. +# Run: bash tests/run-record.test.sh +set -uo pipefail +REPO="$(cd "$(dirname "$0")/.." && pwd)" +LIB="$REPO/scripts/lib/run-record.sh" +JLIB="$REPO/scripts/lib/json.sh" +RESULTS="$(mktemp)"; export RESULTS +ok() { printf 'ok\n' >>"$RESULTS"; printf ' ok — %s\n' "$1"; } +bad() { printf 'fail\n' >>"$RESULTS"; printf ' FAIL — %s\n' "$1"; } + +ROOT="$(mktemp -d)"; trap 'chmod -R u+rwX "$ROOT" 2>/dev/null; rm -rf "$ROOT" "$RESULTS"' EXIT + +src() { bash -c "set -uo pipefail; source \"$LIB\"; $*"; } +field() { # + bash -c "set -uo pipefail; source \"$JLIB\"; tl_json_field \"$2\"" <"$1" +} + +# --- [A] parses + sources ---------------------------------------------------- +echo "[A] run-record.sh parses + sources in isolation" +( bash -n "$LIB" 2>"$ROOT/A.err" \ + && ok "run-record.sh parses (bash -n)" \ + || bad "run-record.sh failed bash -n: $(cat "$ROOT/A.err" 2>/dev/null)" + if bash -c "set -uo pipefail; source \"$LIB\"; type -t tl_run_init && type -t tl_run_set_tdd && type -t tl_run_set_pr && type -t tl_run_lock && type -t tl_run_unlock && type -t tl_run_lock_reclaim && type -t tl_run_next_gate" >/dev/null 2>"$ROOT/A2.err"; then + ok "binds init/set_tdd/set_pr/lock/unlock/reclaim/next_gate" + else + bad "failed to source/bind: $(cat "$ROOT/A2.err" 2>/dev/null)" + fi +) || true + +# --- [B] tl_run_init writes run.json and points latest ----------------------- +echo "[B] tl_run_init writes run.json + latest symlink" +( R="$ROOT/b"; mkdir -p "$R" + src "tl_run_init \"$R\" run1" + f="$R/docs/tdd/.implement-logs/run1/run.json" + [ -f "$f" ] && ok "run.json exists" || bad "run.json missing" + [ "$(field "$f" status)" = "running" ] && ok "status=running" || bad "status=$(field "$f" status)" + [ "$(field "$f" run_id)" = "run1" ] && ok "run_id=run1" || bad "run_id=$(field "$f" run_id)" + link="$R/docs/tdd/.implement-logs/latest" + if [ -L "$link" ]; then + tgt="$(readlink "$link")" + case "$tgt" in + *run1) ok "latest points at run1 ($tgt)" ;; + *) bad "latest target unexpected: $tgt" ;; + esac + else + bad "latest is not a symlink" + fi +) || true + +# --- [C] set_tdd upsert + queue_index + observation 5 ------------------------ +echo "[C] tl_run_set_tdd upserts slug.json; first insert assigns queue_index" +( R="$ROOT/c"; mkdir -p "$R" + src "tl_run_init \"$R\" r + tl_run_set_tdd \"$R\" r slug paused ratelimit" + f="$R/docs/tdd/.implement-logs/r/slug.json" + [ -f "$f" ] && ok "slug.json exists" || bad "slug.json missing" + [ "$(field "$f" status)" = "paused" ] && ok "status=paused (obs 5)" \ + || bad "status=$(field "$f" status) want paused" + [ "$(field "$f" halt_cause)" = "ratelimit" ] && ok "halt_cause=ratelimit" \ + || bad "halt_cause=$(field "$f" halt_cause)" + [ "$(field "$f" queue_index)" = "0" ] && ok "first queue_index=0" \ + || bad "queue_index=$(field "$f" queue_index) want 0" + rf="$R/docs/tdd/.implement-logs/r/run.json" + # tdd_total incremented after first insert + tot="$(field "$rf" tdd_total)" + [ "$tot" = "1" ] && ok "run.json tdd_total=1 after first insert" \ + || bad "tdd_total=$tot want 1" + + src "tl_run_set_tdd \"$R\" r slug building" + [ "$(field "$f" queue_index)" = "0" ] && ok "later call keeps queue_index" \ + || bad "queue_index mutated to $(field "$f" queue_index)" + [ "$(field "$f" status)" = "building" ] && ok "later call refreshes status" \ + || bad "status after upsert=$(field "$f" status)" + + src "tl_run_set_tdd \"$R\" r other pending" + [ "$(field "$R/docs/tdd/.implement-logs/r/other.json" queue_index)" = "1" ] \ + && ok "second slug queue_index=1" || bad "second queue_index wrong" +) || true + +# --- [D] unknown halt_cause is rc 2 ------------------------------------------ +echo "[D] unknown halt_cause rejected" +( R="$ROOT/d"; mkdir -p "$R" + src "tl_run_init \"$R\" r" + set +e + src "tl_run_set_tdd \"$R\" r slug paused not-a-cause" >/dev/null 2>"$ROOT/D.err" + rc=$? + set -e + [ "$rc" -eq 2 ] && ok "unknown cause → rc 2" || bad "unknown cause rc=$rc want 2" +) || true + +# --- [E] lock: live PID holds; dead PID reclaimable -------------------------- +echo "[E] tl_run_lock reclaim when PID is dead (FR-43)" +( R="$ROOT/e"; mkdir -p "$R" + src "tl_run_init \"$R\" r" + lock="$R/docs/tdd/.implement-logs/.run.lock" + # Stale lock: PID that is not alive. + printf '1\n' >"$lock" # PID 1 is init — usually alive. Use a high unused pid. + printf '999999\n' >"$lock" + set +e + src "tl_run_lock_reclaim \"$R\""; rc=$? + set -e + [ "$rc" -eq 0 ] && ok "reclaim succeeds when PID is dead" \ + || bad "reclaim rc=$rc want 0" + # After reclaim, lock should hold this (sub)shell's... actually src is a + # new bash, so the lock PID is that child's (now dead). Re-read: reclaim + # writes $$ of the src bash, which has exited. That's OK for this case — + # we only asserted reclaim rc 0. + + # Held by a live background PID. + sleep 30 & + live=$! + printf '%s\n' "$live" >"$lock" + set +e + src "tl_run_lock \"$R\""; rc=$? + set -e + kill "$live" 2>/dev/null || true + wait "$live" 2>/dev/null || true + [ "$rc" -ne 0 ] && ok "lock held by live PID is refused" \ + || bad "lock accepted while live PID held it" +) || true + +# --- [F] next_gate order + flip ---------------------------------------------- +echo "[F] tl_run_next_gate walks test-first → … → flip" +( R="$ROOT/f"; mkdir -p "$R" + src "tl_run_init \"$R\" r + tl_run_set_tdd \"$R\" r s pending" + g="$(src "tl_run_next_gate \"$R\" r s")" + [ "$g" = "test-first" ] && ok "no verdicts → next_gate=test-first" \ + || bad "next_gate='$g' want test-first" + + src "source \"$REPO/scripts/lib/verdicts.sh\"; tl_verdict_write \"$R\" r s test-first PASS e" + g="$(src "tl_run_next_gate \"$R\" r s")" + [ "$g" = "ci-checks" ] && ok "after test-first PASS → ci-checks" \ + || bad "next_gate='$g' want ci-checks" + + src "source \"$REPO/scripts/lib/verdicts.sh\" + tl_verdict_write \"$R\" r s ci-checks PASS e + tl_verdict_write \"$R\" r s runtime-verify PASS e + tl_verdict_write \"$R\" r s review PASS e" + g="$(src "tl_run_next_gate \"$R\" r s")" + [ "$g" = "flip" ] && ok "all four PASS → flip" || bad "next_gate='$g' want flip" +) || true + +# --- [G] atomic write: a mid-write tmp does not corrupt the live file -------- +echo "[G] kill-mid-write fixture still parses as prior or new (FR-44)" +( R="$ROOT/g"; mkdir -p "$R" + src "tl_run_init \"$R\" r + tl_run_set_tdd \"$R\" r slug paused ratelimit" + f="$R/docs/tdd/.implement-logs/r/slug.json" + dir="$(dirname "$f")" + # Plant a truncated tmp as if kill -9 hit during printf, before mv. + printf '{"slug":"slug","status":"buil' >"$dir/.slug.json.tmp" + # Live file must still parse as the prior (paused) state. + python3 -m json.tool "$f" >/dev/null 2>&1 \ + && ok "live slug.json still valid JSON after orphan tmp" \ + || bad "live file unparseable" + [ "$(field "$f" status)" = "paused" ] && ok "live file still prior status=paused" \ + || bad "live status=$(field "$f" status)" + # Completing a new write replaces atomically. + src "tl_run_set_tdd \"$R\" r slug done" + python3 -m json.tool "$f" >/dev/null 2>&1 \ + && [ "$(field "$f" status)" = "done" ] \ + && ok "post-write file is the new state (done)" \ + || bad "post-write status=$(field "$f" status)" +) || true + +# --- [H] pr_url only via tl_run_set_pr --------------------------------------- +echo "[H] pr_url is set only by tl_run_set_pr" +( R="$ROOT/h"; mkdir -p "$R" + src "tl_run_init \"$R\" r + tl_run_set_tdd \"$R\" r slug building + tl_run_set_pr \"$R\" r slug https://example.test/pr/1" + [ "$(field "$R/docs/tdd/.implement-logs/r/slug.json" pr_url)" = "https://example.test/pr/1" ] \ + && ok "tl_run_set_pr stored pr_url" || bad "pr_url not stored" +) || true + +echo +PASS="$(grep -c '^ok$' "$RESULTS" 2>/dev/null)"; PASS="${PASS:-0}" +FAIL="$(grep -c '^fail$' "$RESULTS" 2>/dev/null)"; FAIL="${FAIL:-0}" +echo "=== run-record eval: $PASS passed, $FAIL failed ===" +[ "$FAIL" -eq 0 ] diff --git a/tests/verdicts.test.sh b/tests/verdicts.test.sh new file mode 100755 index 0000000..1567594 --- /dev/null +++ b/tests/verdicts.test.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# verdicts.test.sh — eval for TDD 0061 / FR-15, FR-25, FR-82: +# pins scripts/lib/verdicts.sh (write/read/require + tl_test_first_observe). +# +# Written red-first: before TDD 0061 lands, verdicts.sh does not exist. +# Run: bash tests/verdicts.test.sh +set -uo pipefail +REPO="$(cd "$(dirname "$0")/.." && pwd)" +LIB="$REPO/scripts/lib/verdicts.sh" +RESULTS="$(mktemp)"; export RESULTS +ok() { printf 'ok\n' >>"$RESULTS"; printf ' ok — %s\n' "$1"; } +bad() { printf 'fail\n' >>"$RESULTS"; printf ' FAIL — %s\n' "$1"; } + +ROOT="$(mktemp -d)"; trap 'chmod -R u+rwX "$ROOT" 2>/dev/null; rm -rf "$ROOT" "$RESULTS"' EXIT + +src() { bash -c "set -uo pipefail; source \"$LIB\"; $*"; } + +# --- [A] parses + sources + binds the contract ------------------------------- +echo "[A] verdicts.sh parses + sources in isolation" +( bash -n "$LIB" 2>"$ROOT/A.err" \ + && ok "verdicts.sh parses (bash -n)" \ + || bad "verdicts.sh failed bash -n: $(cat "$ROOT/A.err" 2>/dev/null)" + if bash -c "set -uo pipefail; source \"$LIB\"; type -t tl_verdict_dir && type -t tl_verdict_write && type -t tl_verdict_read && type -t tl_verdict_require_flip && type -t tl_test_first_observe" >/dev/null 2>"$ROOT/A2.err"; then + ok "binds dir/write/read/require_flip/test_first_observe" + else + bad "failed to source/bind: $(cat "$ROOT/A2.err" 2>/dev/null)" + fi +) || true + +# --- [B] write creates /docs/tdd/.implement-logs///.json +echo "[B] tl_verdict_write creates the ADR 0011 artifact" +( R="$ROOT/b"; mkdir -p "$R" + set +e + src "tl_verdict_write \"$R\" r s test-first PASS 'git log …'" >/dev/null 2>"$ROOT/B.err" + rc=$? + set -e + f="$R/docs/tdd/.implement-logs/r/s/test-first.json" + [ "$rc" -eq 0 ] && [ -f "$f" ] && ok "wrote $f (rc=0)" \ + || bad "write rc=$rc file-exists=$([ -f "$f" ] && echo y || echo n) err=$(cat "$ROOT/B.err")" + if [ -f "$f" ]; then + st="$(tl_json_field() { :; }; bash -c "source \"$REPO/scripts/lib/json.sh\"; tl_json_field status" <"$f")" + [ "$st" = "PASS" ] && ok "status field is PASS" || bad "status='$st' want PASS" + gt="$(bash -c "source \"$REPO/scripts/lib/json.sh\"; tl_json_field gate" <"$f")" + [ "$gt" = "test-first" ] && ok "gate field is test-first" || bad "gate='$gt'" + fi +) || true + +# --- [C] require_flip is rc 1 until all four gates satisfy the flip rule ------ +echo "[C] tl_verdict_require_flip stays rc 1 until all four are written" +( R="$ROOT/c"; mkdir -p "$R" + src "tl_verdict_require_flip \"$R\" r s"; rc=$? + [ "$rc" -eq 1 ] && ok "missing files → rc 1" || bad "empty require rc=$rc want 1" + + src "tl_verdict_write \"$R\" r s test-first PASS e1; tl_verdict_write \"$R\" r s ci-checks PASS e2; tl_verdict_write \"$R\" r s review PASS e3" + src "tl_verdict_require_flip \"$R\" r s"; rc=$? + [ "$rc" -eq 1 ] && ok "three PASS, runtime-verify missing → rc 1" \ + || bad "three-of-four require rc=$rc want 1" + + src "tl_verdict_write \"$R\" r s runtime-verify PASS e4" + src "tl_verdict_require_flip \"$R\" r s"; rc=$? + [ "$rc" -eq 0 ] && ok "four PASS → rc 0" || bad "four PASS require rc=$rc want 0" +) || true + +# --- [D] SKIP without evidence fails require; SKIP with evidence on rv ok ----- +echo "[D] runtime-verify SKIP requires non-empty evidence" +( R="$ROOT/d"; mkdir -p "$R" + src "tl_verdict_write \"$R\" r s test-first PASS a + tl_verdict_write \"$R\" r s ci-checks PASS b + tl_verdict_write \"$R\" r s review PASS c + tl_verdict_write \"$R\" r s runtime-verify SKIP ''" + src "tl_verdict_require_flip \"$R\" r s"; rc=$? + [ "$rc" -eq 1 ] && ok "SKIP with empty evidence → rc 1" || bad "empty-SKIP require rc=$rc want 1" + + src "tl_verdict_write \"$R\" r s runtime-verify SKIP 'no observable surface'" + src "tl_verdict_require_flip \"$R\" r s"; rc=$? + [ "$rc" -eq 0 ] && ok "SKIP with evidence → rc 0" || bad "justified SKIP require rc=$rc want 0" +) || true + +# --- [E] reject unknown gate/status with rc 2 -------------------------------- +echo "[E] unknown gate/status rejected (rc 2)" +( R="$ROOT/e"; mkdir -p "$R" + set +e + src "tl_verdict_write \"$R\" r s review WHATEVER x" >/dev/null 2>"$ROOT/E.err" + rc=$? + set -e + [ "$rc" -eq 2 ] && ok "bad status WHATEVER → rc 2" || bad "bad status rc=$rc want 2" + grep -q 'WHATEVER' "$ROOT/E.err" && ok "stderr names the bad token" \ + || bad "stderr did not name WHATEVER ($(cat "$ROOT/E.err"))" + [ ! -f "$R/docs/tdd/.implement-logs/r/s/review.json" ] \ + && ok "bad write created no file" || bad "bad write left a file" + + set +e + src "tl_verdict_write \"$R\" r s not-a-gate PASS x" >/dev/null 2>"$ROOT/E2.err" + rc=$? + set -e + [ "$rc" -eq 2 ] && ok "bad gate → rc 2" || bad "bad gate rc=$rc want 2" +) || true + +# --- [F] missing read is rc 1, not 0-with-empty ------------------------------ +echo "[F] tl_verdict_read missing file is rc 1" +( R="$ROOT/f"; mkdir -p "$R" + set +e + out="$(src "tl_verdict_read \"$R\" r s test-first" 2>/dev/null)" + rc=$? + set -e + [ "$rc" -eq 1 ] && ok "missing read → rc 1" || bad "missing read rc=$rc want 1" + [ -z "$out" ] && ok "missing read stdout empty" || bad "missing read out='$out'" +) || true + +# --- [G] slug with slash rejected -------------------------------------------- +echo "[G] slug containing / is rejected" +( R="$ROOT/g"; mkdir -p "$R" + set +e + src "tl_verdict_write \"$R\" r 'evil/slug' test-first PASS x" >/dev/null 2>"$ROOT/G.err" + rc=$? + set -e + [ "$rc" -ne 0 ] && ok "slash slug rejected (rc=$rc)" || bad "slash slug accepted" + [ ! -d "$R/docs/tdd/.implement-logs/r/evil" ] \ + && ok "slash slug did not create a nested dir" || bad "slash slug escaped into a path" +) || true + +# --- [H] functions do not use process cwd ------------------------------------ +echo "[H] write honors even when cwd is elsewhere" +( R="$ROOT/h"; mkdir -p "$R" "$ROOT/h-cwd" + ( cd "$ROOT/h-cwd" && src "tl_verdict_write \"$R\" r s ci-checks PASS ev" ) + [ -f "$R/docs/tdd/.implement-logs/r/s/ci-checks.json" ] \ + && ok "file landed under repo-root, not cwd" \ + || bad "file missing under repo-root" + [ ! -e "$ROOT/h-cwd/docs" ] && ok "cwd was not used as repo-root" \ + || bad "wrote under cwd" +) || true + +# --- [I] tl_test_first_observe: feat-only fail; test-then-feat pass ----------- +echo "[I] tl_test_first_observe on a fixture repo" +mkfix() { # + local d="$1" + mkdir -p "$d" + git -C "$d" init -q + git -C "$d" checkout -q -b master 2>/dev/null || git -C "$d" branch -M master + git -C "$d" config user.email t@e; git -C "$d" config user.name t + printf 'init\n' >"$d/f" + git -C "$d" add f; git -C "$d" commit -qm init +} +( D="$ROOT/i-feat"; mkfix "$D" + git -C "$D" checkout -q -b build + printf 'x\n' >>"$D/f"; git -C "$D" add f; git -C "$D" commit -qm 'feat: x' + set +e + src "tl_test_first_observe \"$D\""; rc=$? + set -e + [ "$rc" -eq 1 ] && ok "feat-only → rc 1" || bad "feat-only rc=$rc want 1" +) +( D="$ROOT/i-ok"; mkfix "$D" + git -C "$D" checkout -q -b build + printf 't\n' >>"$D/f"; git -C "$D" add f; git -C "$D" commit -qm 'test(failing): x' + printf 'x\n' >>"$D/f"; git -C "$D" add f; git -C "$D" commit -qm 'feat: x' + set +e + src "tl_test_first_observe \"$D\""; rc=$? + set -e + [ "$rc" -eq 0 ] && ok "test(failing): then feat: → rc 0" || bad "ordered pair rc=$rc want 0" +) +( D="$ROOT/i-empty"; mkdir -p "$D"; git -C "$D" init -q + git -C "$D" config user.email t@e; git -C "$D" config user.name t + set +e + src "tl_test_first_observe \"$D\""; rc=$? + set -e + [ "$rc" -eq 1 ] && ok "empty repo → rc 1" || bad "empty repo rc=$rc want 1" +) +( D="$ROOT/i-late"; mkfix "$D" + git -C "$D" checkout -q -b build + printf 'x\n' >>"$D/f"; git -C "$D" add f; git -C "$D" commit -qm 'feat: x' + printf 't\n' >>"$D/f"; git -C "$D" add f; git -C "$D" commit -qm 'test(failing): late' + set +e + src "tl_test_first_observe \"$D\""; rc=$? + set -e + [ "$rc" -eq 1 ] && ok "feat: before test(failing): → rc 1" \ + || bad "reversed order rc=$rc want 1" +) || true + +echo +PASS="$(grep -c '^ok$' "$RESULTS" 2>/dev/null)"; PASS="${PASS:-0}" +FAIL="$(grep -c '^fail$' "$RESULTS" 2>/dev/null)"; FAIL="${FAIL:-0}" +echo "=== verdicts eval: $PASS passed, $FAIL failed ===" +[ "$FAIL" -eq 0 ]