diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7337054..3363fe6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,10 @@ jobs: run: ./test/test-version-classifier.sh - name: Plugin-status unit tests run: ./test/test-plugin-status.sh + - name: Doctor behavioral tests + run: ./test/test-doctor.sh + - name: Status / readiness behavioral tests + run: ./test/test-status.sh - name: Preamble embedded-python tests run: ./test/test-preamble-python.sh - name: Setup behavioral tests diff --git a/test/integration-test.sh b/test/integration-test.sh index c26f704..35f0735 100755 --- a/test/integration-test.sh +++ b/test/integration-test.sh @@ -2,9 +2,7 @@ # idstack integration tests — behavioral tests for bin scripts set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" IDSTACK_DIR="$(cd "$(dirname "$0")/.." && pwd)" TEST_DIR=$(mktemp -d) @@ -16,21 +14,6 @@ trap 'rm -rf "$TEST_DIR"' EXIT # uncommitted work doesn't trip the check. TREE_BEFORE=$(git -C "$IDSTACK_DIR" status --porcelain 2>/dev/null || true) -check() { - TOTAL=$((TOTAL + 1)) - local _out - if _out=$(eval "$2" 2>&1); then - echo " PASS: $1" - PASS=$((PASS + 1)) - else - echo " FAIL: $1" - if [ -n "$_out" ]; then - printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' - fi - FAIL=$((FAIL + 1)) - fi -} - echo "idstack integration tests" echo " test dir: $TEST_DIR" echo "" diff --git a/test/mutation-test.sh b/test/mutation-test.sh index 5ef7228..257f4dd 100755 --- a/test/mutation-test.sh +++ b/test/mutation-test.sh @@ -149,6 +149,49 @@ open(p,'w').write(s) PY expect_fail "plugin-status fixed-window regression" "$WORK/r/test/test-plugin-status.sh" +# 7b. doctor stops flagging a disabled install -> test-doctor must fail. +# doctor exists to explain why /idstack: is missing; a branch that stops +# reporting turns it into a script that always says everything is fine. +fresh +sed -i.bak 's|echo " PROBLEM: idstack@idstack is installed but not enabled."|echo " OK: installed"|' \ + "$WORK/r/bin/idstack-doctor" +expect_fail "doctor silently passes a disabled install" "$WORK/r/test/test-doctor.sh" + +# 7c. readiness threshold drifts by one -> test-status must fail. +# A fixture failing every threshold at once cannot catch this: some other +# unmet condition keeps the verdict NOT-READY either way. The boundary cases +# in test-status.sh hold the other two dimensions passing so the constant is +# the only thing that can flip the verdict. +fresh +sed -i.bak 's|^READY_QUALITY_MIN=70|READY_QUALITY_MIN=69|' "$WORK/r/bin/idstack-status" +expect_fail "readiness quality threshold off by one" "$WORK/r/test/test-status.sh" + +# 7d. WCAG Level-A override removed from the verdict -> test-status must fail. +# The override exists in access_tier() and again in verdict(); deleting either +# alone leaves the other reporting, so both are pinned. +fresh +python3 - "$WORK/r/bin/idstack-status" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +s = s.replace(" if level_a_violations:\n issues.append(f'{len(level_a_violations)} WCAG Level-A violation(s)')\n", "") +open(p, 'w').write(s) +PY +expect_fail "WCAG Level-A override dropped from verdict" "$WORK/r/test/test-status.sh" + +# 7e. a suite regrows its own counters -> smoke-test must fail. +# This is how the nine copies of check() drifted apart in the first place: each +# suite kept a private PASS=0 and its own body, and one of them ended up +# swallowing failure output entirely. +fresh +python3 - "$WORK/r/test/test-doctor.sh" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +s = s.replace('. "$(dirname "$0")/test-helper.sh"', + 'PASS=0\nFAIL=0\nTOTAL=0\ncheck() { TOTAL=$((TOTAL+1)); eval "$2" >/dev/null 2>&1 && PASS=$((PASS+1)) || FAIL=$((FAIL+1)); }', 1) +open(p, 'w').write(s) +PY +expect_fail "suite regrows private test counters" "$WORK/r/test/smoke-test.sh" "$WORK/r" + # 8. version disagreement -> smoke-test must fail fresh printf '9.9.9.9\n' > "$WORK/r/VERSION" diff --git a/test/smoke-test.sh b/test/smoke-test.sh index d6f8d8f..b8bfdaf 100755 --- a/test/smoke-test.sh +++ b/test/smoke-test.sh @@ -2,9 +2,7 @@ # idstack smoke test — verifies installation is correct set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" # Verify the repo this script lives in (test/smoke-test.sh -> repo root is ".."). # Override with $1 to point at a different checkout (CI fixtures, etc.). @@ -15,21 +13,6 @@ IDSTACK_DIR="${1:-$(cd "$(dirname "$0")/.." && pwd -P)}" VER="$(tr -d '[:space:]' < "$IDSTACK_DIR/VERSION" 2>/dev/null || true)" VER3="${VER%.*}" # 4-component 3.2.0.0 -> 3-component 3.2.0 (JSON-LD softwareVersion) -check() { - TOTAL=$((TOTAL + 1)) - local _out - if _out=$(eval "$2" 2>&1); then - echo " PASS: $1" - PASS=$((PASS + 1)) - else - echo " FAIL: $1" - if [ -n "$_out" ]; then - printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' - fi - FAIL=$((FAIL + 1)) - fi -} - echo "idstack smoke test" echo " idstack dir: $IDSTACK_DIR" echo "" @@ -73,10 +56,23 @@ for script in idstack-migrate idstack-timeline-log idstack-learnings-log idstack done # Bash syntax gate for the shell entry points (idstack-manifest-merge is python). -for script in setup bin/idstack-doctor bin/idstack-gen-skills bin/idstack-status bin/idstack-migrate bin/idstack-slugify bin/idstack-update-check bin/lib/version-classify.sh bin/lib/plugin-status.sh; do +for script in setup bin/idstack-doctor bin/idstack-gen-skills bin/idstack-status bin/idstack-migrate bin/idstack-slugify bin/idstack-update-check bin/lib/version-classify.sh bin/lib/plugin-status.sh test/test-helper.sh; do check "$script passes bash -n" "bash -n '$IDSTACK_DIR/$script'" done +# Every suite takes its counters from test/test-helper.sh. Nine suites had +# each grown their own copy, and they had already drifted — two spelled the +# helper `assert`, and one swallowed failure output entirely, so CI reported a +# bare FAIL with no diagnostics. A local `PASS=0` is how that regrows. +check "test-helper.sh exists" "[ -f '$IDSTACK_DIR/test/test-helper.sh' ]" +for suite in smoke-test integration-test test-manifest-merge test-version-classifier \ + test-plugin-status test-preamble-python test-setup test-doctor test-status; do + check "test/$suite.sh sources the shared helper" \ + "grep -q 'test-helper.sh' '$IDSTACK_DIR/test/$suite.sh'" + check "test/$suite.sh defines no local counters" \ + "! grep -qE '^(PASS|FAIL|TOTAL)=0' '$IDSTACK_DIR/test/$suite.sh'" +done + # Check template system check "templates/preamble.md exists" "[ -f '$IDSTACK_DIR/templates/preamble.md' ]" check "templates/report-format.md exists" "[ -f '$IDSTACK_DIR/templates/report-format.md' ]" @@ -274,6 +270,15 @@ check "idstack-doctor no longer uses a fixed -A4 window" "! grep -q 'grep -A4' ' if [ -x "$IDSTACK_DIR/test/test-plugin-status.sh" ]; then check "plugin-status unit tests pass" "'$IDSTACK_DIR/test/test-plugin-status.sh'" fi +# Hooked in here as well as in CI because release.yml's gate runs only +# smoke-test.sh — doctor's branches would otherwise be unverified at release. +if [ -x "$IDSTACK_DIR/test/test-doctor.sh" ]; then + check "doctor behavioral tests pass" "'$IDSTACK_DIR/test/test-doctor.sh'" +fi +# --readiness is the pre-export gate; same reasoning as doctor above. +if [ -x "$IDSTACK_DIR/test/test-status.sh" ]; then + check "idstack-status behavioral tests pass" "'$IDSTACK_DIR/test/test-status.sh'" +fi # ./setup is what a new user runs first; it is exercised against a repo copy # with a fake $HOME and a stub `claude`, never the real install. diff --git a/test/test-doctor.sh b/test/test-doctor.sh new file mode 100755 index 0000000..034c12e --- /dev/null +++ b/test/test-doctor.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Unit tests for bin/idstack-doctor. +# +# doctor's whole job is to tell a user why /idstack: isn't working, so +# every branch it can print is a branch someone will act on. Before this suite +# it had no execution coverage at all — smoke-test only checked the file exists +# and parses. +# +# Two things this harness has to get right, both of which are easy to get +# wrong: +# +# 1. doctor sources bin/lib/version-classify.sh and bin/lib/plugin-status.sh +# (see bin/idstack-doctor:21-23), so the fixture copies the whole bin/ +# tree. Copying just the one script gives a "No such file" on the source +# line and every assertion below fails for the wrong reason. +# +# 2. PATH is replaced, not prepended. The "claude not found" case is only +# meaningful if `claude` is genuinely unreachable — with the real PATH +# still appended, that test passes on CI (no claude) and fails on any +# developer machine that has Claude Code installed. + +set -e + +. "$(dirname "$0")/test-helper.sh" + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT + +export HOME="$TEST_ROOT/home" +mkdir -p "$HOME" + +# Fixture repo — whole bin/ tree, so the sourced libs resolve. +MOCK_IDSTACK_DIR="$TEST_ROOT/idstack" +mkdir -p "$MOCK_IDSTACK_DIR/bin" +cp -R "$REPO_DIR/bin/." "$MOCK_IDSTACK_DIR/bin/" +DOCTOR_CMD="$MOCK_IDSTACK_DIR/bin/idstack-doctor" + +# Pinned PATH: only what doctor and its libs actually invoke, plus whatever +# mock we drop in. `claude` is absent unless a test installs it. +MOCK_BIN="$TEST_ROOT/bin" +mkdir -p "$MOCK_BIN" +for _c in bash sh env command dirname echo grep head python3 readlink sed tr awk cat ls rm mkdir ln chmod mktemp printf sort wc basename find touch cp; do + _p=$(command -v "$_c" 2>/dev/null) && ln -sf "$_p" "$MOCK_BIN/$_c" +done +export PATH="$MOCK_BIN" + +SKILLS="needs-analysis learning-objectives course-quality-review course-import \ +assessment-design course-builder course-export accessibility-review red-team \ +pipeline learn" + +setup_happy_path() { + rm -rf "$MOCK_IDSTACK_DIR/.claude-plugin" "$MOCK_IDSTACK_DIR/skills" "$HOME/.claude" + mkdir -p "$MOCK_IDSTACK_DIR/.claude-plugin" + # Version must parse as JSON — doctor reads it with python3, not a regex. + echo '{"name": "idstack", "version": "9.9.9.9"}' > "$MOCK_IDSTACK_DIR/.claude-plugin/plugin.json" + echo '{"name": "idstack"}' > "$MOCK_IDSTACK_DIR/.claude-plugin/marketplace.json" + for skill in $SKILLS; do + mkdir -p "$MOCK_IDSTACK_DIR/skills/$skill" + touch "$MOCK_IDSTACK_DIR/skills/$skill/SKILL.md" + done +} + +# Stub `claude` whose `plugin list` prints $1 verbatim. +mock_claude_output() { + { + echo '#!/bin/sh' + echo 'if [ "$1 $2" = "plugin list" ]; then' + echo "cat << 'INNER_EOF'" + echo "$1" + echo 'INNER_EOF' + echo 'fi' + } > "$MOCK_BIN/claude" + chmod +x "$MOCK_BIN/claude" +} + +remove_mock_claude() { rm -f "$MOCK_BIN/claude"; } + +echo "idstack doctor tests" +echo "" + +# --- healthy install --- +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +check "healthy install reports OK and exits 0" "'$DOCTOR_CMD'" 0 "OK" + +# --- manifest problems --- +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +rm "$MOCK_IDSTACK_DIR/.claude-plugin/plugin.json" +check "missing plugin.json is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: missing.*plugin\.json" + +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +rm "$MOCK_IDSTACK_DIR/.claude-plugin/marketplace.json" +check "missing marketplace.json is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: missing.*marketplace\.json" + +# plugin.json is parsed as JSON, not grepped — malformed must be caught. +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +echo 'not json at all' > "$MOCK_IDSTACK_DIR/.claude-plugin/plugin.json" +check "unparseable plugin.json is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: could not parse version" + +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +rm "$MOCK_IDSTACK_DIR/skills/needs-analysis/SKILL.md" +check "missing SKILL.md is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: 1 SKILL\.md file" + +# --- claude install state --- +setup_happy_path +remove_mock_claude +check "absent claude is a WARNING, not a failure" "'$DOCTOR_CMD'" 0 "WARNING: 'claude' not found on PATH" + +setup_happy_path +mock_claude_output "idstack@idstack +status: disabled" +check "installed but disabled is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: idstack@idstack is installed but not enabled" + +setup_happy_path +mock_claude_output "some-other-plugin +status: enabled" +check "not installed is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: idstack@idstack is not installed" + +# The v3.3.0.0 regression: a neighbouring plugin's "enabled" line was read as +# idstack's, so a disabled idstack was diagnosed healthy. Pinned here too, at +# the doctor level rather than only in test-plugin-status.sh. +setup_happy_path +mock_claude_output "idstack@idstack +status: disabled +other-plugin@other +status: enabled" +check "neighbour's enabled line is not attributed to idstack" "'$DOCTOR_CMD'" 1 "PROBLEM: idstack@idstack is installed but not enabled" + +# --- legacy install conflicts --- +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +mkdir -p "$HOME/.claude/skills" +ln -s "$TEST_ROOT/dummy" "$HOME/.claude/skills/idstack" +check "legacy dispatcher symlink is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: legacy symlink at" + +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +mkdir -p "$HOME/.claude/skills/idstack" +echo "2.0.0" > "$HOME/.claude/skills/idstack/VERSION" +check "pre-v2.0.1.0 install dir is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: pre-v2\.0\.1\.0 install at" + +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +mkdir -p "$HOME/.claude/skills" +ln -s "$TEST_ROOT/dummy-idstack/some-skill" "$HOME/.claude/skills/needs-analysis" +check "pre-v2 per-skill symlink is a PROBLEM" "'$DOCTOR_CMD'" 1 "PROBLEM: pre-v2 skill symlink" + +# An unrelated directory must not be claimed as a broken idstack install. +setup_happy_path +mock_claude_output "idstack@idstack +status: enabled" +mkdir -p "$HOME/.claude/skills/idstack" +echo "someone else's thing" > "$HOME/.claude/skills/idstack/README.md" +check "unrecognized dir warns rather than claiming ownership" "'$DOCTOR_CMD'" 0 "WARNING: directory at" + +echo "" +echo " $PASS/$TOTAL passed" +[ "$FAIL" = "0" ] || exit 1 diff --git a/test/test-helper.sh b/test/test-helper.sh new file mode 100644 index 0000000..ec5c454 --- /dev/null +++ b/test/test-helper.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Shared counters and assertion helper for the idstack test suites. +# Sourced, never executed: `. "$(dirname "$0")/test-helper.sh"`. +# +# Nine suites each carried their own PASS/FAIL/TOTAL and their own copy of this +# function. They had already drifted: some printed failure diagnostics, some +# swallowed them with 2>/dev/null, and two spelled the helper `assert` instead +# of `check`. A suite that prints nothing on failure is the worst version — +# CI tells you something broke and not what. +# +# Suites with a genuinely different assertion shape keep their own wrapper and +# use only the counters from here: test-version-classifier.sh compares a +# version string to an expected classification rather than running a command. + +PASS=0 +FAIL=0 +TOTAL=0 + +# check [expected_exit] [expected_output_regex] +# +# Runs through eval with stderr folded into stdout. With no optional +# arguments this is the plain "command must succeed" form every suite already +# used. Pass expected_exit to assert a specific non-zero code (exit codes are +# part of the contract for bin/idstack-manifest-merge and bin/idstack-doctor), +# and expected_output_regex to also require matching output — an ERE, matched +# with grep -qE. +# +# On failure the first 5 lines of output are printed, indented. Never silence +# this; a bare FAIL is the thing this helper exists to stop. +check() { + TOTAL=$((TOTAL + 1)) + local _desc="$1" _cmd="$2" _want_exit="${3:-0}" _want_out="${4:-}" + local _out _code=0 + + _out=$(eval "$_cmd" 2>&1) || _code=$? + + if [ "$_code" -ne "$_want_exit" ]; then + FAIL=$((FAIL + 1)) + echo " FAIL: $_desc (expected exit $_want_exit, got $_code)" + [ -n "$_out" ] && printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' + return 0 + fi + + if [ -n "$_want_out" ] && ! printf '%s\n' "$_out" | grep -qE "$_want_out"; then + FAIL=$((FAIL + 1)) + echo " FAIL: $_desc (exit $_code correct, output missing /$_want_out/)" + [ -n "$_out" ] && printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' + return 0 + fi + + PASS=$((PASS + 1)) + echo " PASS: $_desc" + return 0 +} diff --git a/test/test-manifest-merge.sh b/test/test-manifest-merge.sh index b20ec09..6d39d3b 100755 --- a/test/test-manifest-merge.sh +++ b/test/test-manifest-merge.sh @@ -5,24 +5,11 @@ set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" MERGE="$REPO_ROOT/bin/idstack-manifest-merge" -assert() { - TOTAL=$((TOTAL + 1)) - if eval "$2" >/dev/null 2>&1; then - PASS=$((PASS + 1)) - echo " PASS: $1" - else - FAIL=$((FAIL + 1)) - echo " FAIL: $1" - fi -} - # Skip the suite if python3 is missing (the tool requires python3). if ! command -v python3 >/dev/null 2>&1; then echo "test-manifest-merge: python3 not available, skipping" @@ -59,25 +46,25 @@ echo "" seed_manifest echo '{"confidence_score": 75}' > "$WORK/payload.json" "$MERGE" --section red_team_audit --payload "$WORK/payload.json" --manifest "$WORK/project.json" --quiet -assert "section replaced" \ +check "section replaced" \ "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"red_team_audit\"]=={\"confidence_score\": 75}'" # --- Test 2: foreign sections preserved --- -assert "context preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"context\"]=={\"modality\": \"online\"}'" -assert "needs_analysis preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"needs_analysis\"]=={\"existing\": \"data\"}'" +check "context preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"context\"]=={\"modality\": \"online\"}'" +check "needs_analysis preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"needs_analysis\"]=={\"existing\": \"data\"}'" # --- Test 3: top-level fields preserved (project_name, created) --- -assert "project_name preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"project_name\"]==\"test-project\"'" -assert "created preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"created\"]==\"2026-01-01T00:00:00Z\"'" -assert "version preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"version\"]==\"1.4\"'" +check "project_name preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"project_name\"]==\"test-project\"'" +check "created preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"created\"]==\"2026-01-01T00:00:00Z\"'" +check "version preserved" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"version\"]==\"1.4\"'" # --- Test 4: updated timestamp bumped (not equal to seeded value) --- -assert "updated timestamp bumped" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"updated\"]!=\"2026-01-01T00:00:00Z\"'" +check "updated timestamp bumped" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"updated\"]!=\"2026-01-01T00:00:00Z\"'" # --- Test 5: stdin payload --- seed_manifest echo '{"score": 99}' | "$MERGE" --section quality_review --payload - --manifest "$WORK/project.json" --quiet -assert "stdin payload accepted" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"quality_review\"]=={\"score\": 99}'" +check "stdin payload accepted" "python3 -c 'import json; d=json.load(open(\"$WORK/project.json\")); assert d[\"quality_review\"]=={\"score\": 99}'" # --- Test 6: unknown section rejected with exit 3 --- seed_manifest @@ -85,7 +72,7 @@ set +e "$MERGE" --section bogus_section --payload "$WORK/payload.json" --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "unknown section exits 3" "[ $EC -eq 3 ]" +check "unknown section exits 3" "[ $EC -eq 3 ]" # --- Test 7: malformed payload rejected with exit 1 --- seed_manifest @@ -94,7 +81,7 @@ set +e "$MERGE" --section red_team_audit --payload "$WORK/bad.json" --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "malformed payload exits 1" "[ $EC -eq 1 ]" +check "malformed payload exits 1" "[ $EC -eq 1 ]" # --- Test 8: malformed manifest rejected with exit 2 --- echo 'not json' > "$WORK/project.json" @@ -102,7 +89,7 @@ set +e echo '{}' | "$MERGE" --section red_team_audit --payload - --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "malformed manifest exits 2" "[ $EC -eq 2 ]" +check "malformed manifest exits 2" "[ $EC -eq 2 ]" # --- Test 9: missing manifest rejected with exit 4 --- rm -f "$WORK/project.json" @@ -110,7 +97,7 @@ set +e echo '{}' | "$MERGE" --section red_team_audit --payload - --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "missing manifest exits 4" "[ $EC -eq 4 ]" +check "missing manifest exits 4" "[ $EC -eq 4 ]" # --- Test 10: missing payload file rejected with exit 5 --- seed_manifest @@ -118,7 +105,7 @@ set +e "$MERGE" --section red_team_audit --payload "$WORK/nonexistent.json" --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "missing payload file exits 5" "[ $EC -eq 5 ]" +check "missing payload file exits 5" "[ $EC -eq 5 ]" # --- Test 11: manifest with non-dict root (list) rejected with exit 2 --- echo '[]' > "$WORK/project.json" @@ -126,7 +113,7 @@ set +e echo '{}' | "$MERGE" --section red_team_audit --payload - --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "manifest with list root exits 2" "[ $EC -eq 2 ]" +check "manifest with list root exits 2" "[ $EC -eq 2 ]" # --- Test 12: manifest with non-dict root (string) rejected with exit 2 --- echo '"a string"' > "$WORK/project.json" @@ -134,7 +121,7 @@ set +e echo '{}' | "$MERGE" --section red_team_audit --payload - --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "manifest with string root exits 2" "[ $EC -eq 2 ]" +check "manifest with string root exits 2" "[ $EC -eq 2 ]" # --- idstack-migrate --init: the manifest bootstrap the merge tool depends on --- # Skills running standalone create the manifest with --init before merging; @@ -144,14 +131,14 @@ MIGRATE="$(cd "$(dirname "$0")/.." && pwd -P)/bin/idstack-migrate" INIT_DIR="$WORK/initcase" mkdir -p "$INIT_DIR/.idstack" ( cd "$INIT_DIR" && "$MIGRATE" --init .idstack/project.json >/dev/null 2>&1 ) -assert "--init creates a manifest" "[ -f '$INIT_DIR/.idstack/project.json' ]" -assert "--init manifest is at the latest schema version" \ +check "--init creates a manifest" "[ -f '$INIT_DIR/.idstack/project.json' ]" +check "--init manifest is at the latest schema version" \ "python3 -c \"import json; assert json.load(open('$INIT_DIR/.idstack/project.json'))['version']=='1.4'\"" # Every section the merge tool accepts must exist in a freshly initialized # manifest — otherwise a standalone skill's first merge lands in a document # missing the sections downstream skills read. -assert "--init manifest carries every mergeable section" \ +check "--init manifest carries every mergeable section" \ "python3 -c \" import json, re, sys d = json.load(open('$INIT_DIR/.idstack/project.json')) @@ -163,19 +150,19 @@ assert not missing, 'missing from --init manifest: ' + repr(missing) # A merge into a freshly initialized manifest must succeed. ( cd "$INIT_DIR" && echo '{"overall_score": 77}' | "$MERGE" --section quality_review --payload - --quiet >/dev/null 2>&1 ) -assert "merge into an --init manifest succeeds" \ +check "merge into an --init manifest succeeds" \ "python3 -c \"import json; assert json.load(open('$INIT_DIR/.idstack/project.json'))['quality_review']['overall_score']==77\"" # --init must never clobber an existing manifest. ( cd "$INIT_DIR" && "$MIGRATE" --init .idstack/project.json >/dev/null 2>&1 ) -assert "--init preserves an existing manifest" \ +check "--init preserves an existing manifest" \ "python3 -c \"import json; assert json.load(open('$INIT_DIR/.idstack/project.json'))['quality_review']['overall_score']==77\"" # Without --init the old behavior holds: silent no-op on a missing file. NOINIT_DIR="$WORK/noinit" mkdir -p "$NOINIT_DIR/.idstack" ( cd "$NOINIT_DIR" && "$MIGRATE" .idstack/project.json >/dev/null 2>&1 ) -assert "plain migrate does not create a manifest" "[ ! -f '$NOINIT_DIR/.idstack/project.json' ]" +check "plain migrate does not create a manifest" "[ ! -f '$NOINIT_DIR/.idstack/project.json' ]" echo "" @@ -186,7 +173,7 @@ set +e "$MERGE" --section red_team_audit --payload "$WORK/payload.json" --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "payload with list root exits 1" "[ $EC -eq 1 ]" +check "payload with list root exits 1" "[ $EC -eq 1 ]" # --- Test 14: payload with non-dict root (string) rejected with exit 1 --- seed_manifest @@ -195,7 +182,7 @@ set +e "$MERGE" --section red_team_audit --payload "$WORK/payload.json" --manifest "$WORK/project.json" --quiet 2>/dev/null EC=$? set -e -assert "payload with string root exits 1" "[ $EC -eq 1 ]" +check "payload with string root exits 1" "[ $EC -eq 1 ]" echo "" echo "manifest-merge: $PASS/$TOTAL passed, $FAIL failed" diff --git a/test/test-plugin-status.sh b/test/test-plugin-status.sh index 8812b47..f85e589 100755 --- a/test/test-plugin-status.sh +++ b/test/test-plugin-status.sh @@ -12,15 +12,16 @@ # Run from the repo root or via smoke-test.sh. set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" IDSTACK_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" . "$IDSTACK_DIR/bin/lib/plugin-status.sh" -# check -check() { +# Domain-specific shape: feeds a `claude plugin list` listing to +# plugin_is_enabled and compares the result. Takes only the counters from +# test-helper.sh, and is named check_listing so it cannot shadow check(). +# check_listing +check_listing() { TOTAL=$((TOTAL + 1)) local name="$1" expected="$2" listing="$3" got if printf '%s\n' "$listing" | plugin_is_enabled "idstack@idstack"; then @@ -40,25 +41,25 @@ check() { echo "test-plugin-status" echo "" -check "enabled, only plugin installed" enabled \ +check_listing "enabled, only plugin installed" enabled \ 'idstack@idstack Status: enabled Version: 3.3.0.0' -check "disabled, only plugin installed" disabled \ +check_listing "disabled, only plugin installed" disabled \ 'idstack@idstack Status: disabled Version: 3.3.0.0' # The regression the fixed window caused: the neighbour's "enabled" was inside # the -A4 window, so a disabled idstack read as enabled. -check "disabled, next plugin enabled two lines later" disabled \ +check_listing "disabled, next plugin enabled two lines later" disabled \ 'idstack@idstack Status: disabled superpowers@marketplace Status: enabled' -check "disabled, next plugin enabled after a blank line" disabled \ +check_listing "disabled, next plugin enabled after a blank line" disabled \ 'idstack@idstack Status: disabled Version: 3.3.0.0 @@ -66,14 +67,14 @@ check "disabled, next plugin enabled after a blank line" disabled \ superpowers@marketplace Status: enabled' -check "disabled, a PRECEDING plugin is enabled" disabled \ +check_listing "disabled, a PRECEDING plugin is enabled" disabled \ 'superpowers@marketplace Status: enabled idstack@idstack Status: disabled' -check "enabled, listed between two disabled plugins" enabled \ +check_listing "enabled, listed between two disabled plugins" enabled \ 'airtable@marketplace Status: disabled @@ -84,14 +85,14 @@ superpowers@marketplace Status: disabled' # Single-line listing formats. -check "single-line format, enabled" enabled 'idstack@idstack (enabled)' -check "single-line format, disabled with enabled neighbour" disabled \ +check_listing "single-line format, enabled" enabled 'idstack@idstack (enabled)' +check_listing "single-line format, disabled with enabled neighbour" disabled \ 'idstack@idstack (disabled) superpowers@marketplace (enabled)' # Not installed at all — doctor gates on a separate grep, but the parser must # not invent an enabled verdict from someone else's entry. -check "idstack absent, another plugin enabled" disabled \ +check_listing "idstack absent, another plugin enabled" disabled \ 'superpowers@marketplace Status: enabled' diff --git a/test/test-preamble-python.sh b/test/test-preamble-python.sh index 77f24a1..358828a 100755 --- a/test/test-preamble-python.sh +++ b/test/test-preamble-python.sh @@ -9,27 +9,10 @@ # Run from the repo root or via smoke-test.sh; CI runs it on Python 3.9 and 3.12. set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" IDSTACK_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" -check() { - TOTAL=$((TOTAL + 1)) - local _out - if _out=$(eval "$2" 2>&1); then - echo " PASS: $1" - PASS=$((PASS + 1)) - else - echo " FAIL: $1" - if [ -n "$_out" ]; then - printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' - fi - FAIL=$((FAIL + 1)) - fi -} - echo "test-preamble-python" echo " python3: $(python3 --version 2>&1 || echo 'not found')" echo "" diff --git a/test/test-setup.sh b/test/test-setup.sh index fd7e3fc..ec4df0d 100755 --- a/test/test-setup.sh +++ b/test/test-setup.sh @@ -12,27 +12,12 @@ # Usage: test/test-setup.sh [path-to-repo] set -u -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" SRC="${1:-$(cd "$(dirname "$0")/.." && pwd -P)}" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT -check() { - TOTAL=$((TOTAL + 1)) - local name="$1" cmd="$2" _out - if _out=$(eval "$cmd" 2>&1); then - echo " PASS: $name" - PASS=$((PASS + 1)) - else - echo " FAIL: $name" - [ -n "$_out" ] && printf '%s\n' "$_out" | head -5 | sed 's/^/ | /' - FAIL=$((FAIL + 1)) - fi -} - # Build an isolated environment: repo copy + fake HOME + stub claude/codex that # log their arguments instead of touching the real install. setup_env() { diff --git a/test/test-status.sh b/test/test-status.sh new file mode 100755 index 0000000..7e5fb51 --- /dev/null +++ b/test/test-status.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Behavioral tests for bin/idstack-status. +# +# --readiness (bin/idstack-status:193-294) is the pre-export gate: it decides +# whether a course is fit to ship. It had no coverage at all. +# +# Verdict-state tests alone are not enough here. A fixture that sits below +# every threshold at once still passes when a single threshold constant is +# wrong, because some other unmet condition keeps the verdict NOT-READY. So +# each threshold is probed at its own boundary with the other two held +# passing, and the WCAG Level-A override — which forces NOT-READY regardless +# of the overall accessibility score — is exercised separately. + +set -e + +. "$(dirname "$0")/test-helper.sh" + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +STATUS="$REPO_ROOT/bin/idstack-status" + +if ! command -v python3 >/dev/null 2>&1; then + echo "test-status: python3 not available, skipping" + exit 0 +fi +if [ ! -x "$STATUS" ]; then + echo "test-status: $STATUS missing or not executable" >&2 + exit 1 +fi + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" + +echo "idstack-status tests" +echo "" + +# --- dashboard --- +check "no timeline shows the empty state" \ + "$STATUS | grep -q 'No course data yet'" + +mkdir -p .idstack +printf '%s\n' '{"project_name": "Test Course 123"}' > .idstack/project.json +printf '%s\n' '{"skill":"needs-analysis","event":"completed","training_justified":true}' > .idstack/timeline.jsonl + +check "project name comes from the manifest" \ + "$STATUS | grep -q 'Project: Test Course 123'" + +cat > .idstack/timeline.jsonl <<'JSON' +{"skill":"needs-analysis","event":"completed","training_justified":true} +{"skill":"course-quality-review","event":"completed","score":65,"dimensions":{"teaching_presence":7,"social_presence":3,"cognitive_presence":5}} +{"skill":"course-quality-review","event":"completed","score":85,"dimensions":{"teaching_presence":8,"social_presence":7,"cognitive_presence":8}} +JSON + +# Namespaced forms — v3.3.0.0 banned the bare /skill spelling because it +# resolves in neither CLI. A test asserting the bare form would re-admit it. +check "completed skills render as namespaced checkboxes" \ + "$STATUS | grep -q '\[x\] /idstack:needs-analysis'" + +check "quality trend shows the progression" \ + "$STATUS | grep -q 'Quality trend: 65 -> 85'" + +check "next step is suggested, namespaced" \ + "$STATUS | grep -q 'Suggested next: /idstack:learning-objectives'" + +cat > .idstack/learnings.jsonl <<'JSON' +{"skill":"needs-analysis","type":"operational","insight":"Learned X"} +{"skill":"needs-analysis","type":"pattern","insight":"Learned Y"} +JSON +check "learnings are counted" \ + "$STATUS | grep -q 'Learnings: 2 discoveries stored'" + +mkdir -p .idstack/exports/test-course-123 +touch .idstack/exports/test-course-123/index.html +touch .idstack/exports/test-course-123/other-report.html +check "reports section is present" \ + "$STATUS | grep -q 'Reports'" +check "dashboard index.html is listed" \ + "$STATUS | grep -q 'test-course-123/index.html'" +check "per-skill report is listed" \ + "$STATUS | grep -q 'test-course-123/other-report.html'" + +echo "" + +# --- readiness gate --- +# Helper: write a manifest with explicit scores. $4 is optional extra JSON for +# the accessibility section (used for the WCAG override case). +write_manifest() { + cat > .idstack/project.json <> .idstack/timeline.jsonl <<'JSON' +{"skill":"red-team","event":"completed"} +{"skill":"accessibility-review","event":"completed"} +JSON + +# All three reviews have now run, so verdicts turn on the scores alone. +write_manifest 80 0 90 +check "all thresholds met gives READY TO EXPORT" \ + "$STATUS --readiness | grep -q 'READY TO EXPORT'" + +# --- threshold boundaries, one at a time --- +# Each case holds the other two dimensions passing, so the verdict can only +# flip because of the constant under test. + +write_manifest 70 0 90 +check "quality exactly at the threshold (70) is READY" \ + "$STATUS --readiness | grep -q 'READY TO EXPORT'" + +write_manifest 69 0 90 +check "quality one below the threshold (69) is NOT-READY" \ + "$STATUS --readiness | grep -q 'NOT-READY'" +check "quality failure is itemized with both numbers" \ + "$STATUS --readiness | grep -q 'quality score 69 < 70'" + +write_manifest 80 0 80 +check "accessibility exactly at the threshold (80) is READY" \ + "$STATUS --readiness | grep -q 'READY TO EXPORT'" + +write_manifest 80 0 79 +check "accessibility one below the threshold (79) is NOT-READY" \ + "$STATUS --readiness | grep -q 'accessibility score 79 < 80'" + +write_manifest 80 1 90 +check "a single critical red-team finding is NOT-READY" \ + "$STATUS --readiness | grep -q '1 critical red-team finding'" + +# --- WCAG Level-A override --- +# A Level-A violation forces NOT-READY even when the overall score passes; +# nothing exercised this path, so the score check could have swallowed it. +# +# The override is implemented TWICE — once in access_tier() for the per-skill +# row and once in verdict() for the overall line. Removing either one alone +# leaves the other still reporting, so both are asserted; a single assertion +# here would let half the override rot silently. +write_manifest 80 0 95 '"wcag_violations": [{"level": "A", "criterion": "1.1.1"}]' +check "a Level-A violation overrides a passing accessibility score" \ + "$STATUS --readiness | grep -q '1 WCAG Level-A violation'" +check "Level-A violation forces NOT-READY overall" \ + "$STATUS --readiness | grep -q 'NOT-READY'" +check "Level-A violation also marks the accessibility row NEEDS-WORK" \ + "$STATUS --readiness | grep -qE 'Accessibility Review: +NEEDS-WORK'" + +# Level-AA is not the override; it must not block export on its own. +write_manifest 80 0 95 '"wcag_violations": [{"level": "AA", "criterion": "1.4.3"}]' +check "a Level-AA violation alone does not block export" \ + "$STATUS --readiness | grep -q 'READY TO EXPORT'" + +echo "" +echo "idstack-status: $PASS/$TOTAL passed, $FAIL failed" +[ "$FAIL" -eq 0 ] && exit 0 || exit 1 diff --git a/test/test-version-classifier.sh b/test/test-version-classifier.sh index ea4c45b..0686b08 100755 --- a/test/test-version-classifier.sh +++ b/test/test-version-classifier.sh @@ -20,14 +20,16 @@ set -e -PASS=0 -FAIL=0 -TOTAL=0 +. "$(dirname "$0")/test-helper.sh" IDSTACK_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" . "$IDSTACK_DIR/bin/lib/version-classify.sh" -check() { +# Domain-specific shape: this compares a version string to an expected +# classification rather than running a command, so it takes only the counters +# from test-helper.sh. Named check_version so it cannot shadow the shared +# check() — a same-named local wrapper is how the two would drift apart again. +check_version() { TOTAL=$((TOTAL + 1)) local version="$1" local expected="$2" @@ -46,41 +48,41 @@ echo "test-version-classifier" echo "" # Legacy versions that ever shipped. -check "0.1.0" legacy -check "0.5.0" legacy -check "1.0.0" legacy -check "1.9.0" legacy -check "2.0.0" legacy -check "2.0.0.0" legacy -check "2.0.0.1" legacy +check_version "0.1.0" legacy +check_version "0.5.0" legacy +check_version "1.0.0" legacy +check_version "1.9.0" legacy +check_version "2.0.0" legacy +check_version "2.0.0.0" legacy +check_version "2.0.0.1" legacy # Modern versions — including multi-digit components that broke the # previous patterns. -check "2.0.1.0" skip -check "2.0.1.5" skip -check "2.0.10.0" skip -check "2.0.99.0" skip -check "2.1.0.0" skip -check "2.4.0.0" skip -check "2.10.0.0" skip -check "2.99.0.0" skip -check "3.0.0.0" skip -check "9.0.0.0" skip -check "10.0.0.0" skip -check "19.0.0.0" skip -check "100.0.0.0" skip +check_version "2.0.1.0" skip +check_version "2.0.1.5" skip +check_version "2.0.10.0" skip +check_version "2.0.99.0" skip +check_version "2.1.0.0" skip +check_version "2.4.0.0" skip +check_version "2.10.0.0" skip +check_version "2.99.0.0" skip +check_version "3.0.0.0" skip +check_version "9.0.0.0" skip +check_version "10.0.0.0" skip +check_version "19.0.0.0" skip +check_version "100.0.0.0" skip # Future major versions where the major itself is multi-digit but does not # start with 1. Caught by Gemini on PR #21 — the previous 1[0-9]* arm # missed these and silently fell through to "unknown". Now classified by # [1-9][0-9]*. -check "20.0.0.0" skip -check "21.5.0" skip -check "25.99.0" skip -check "29.0.0" skip -check "30.0.0" skip -check "200.0.0" skip -check "999.0.0" skip +check_version "20.0.0.0" skip +check_version "21.5.0" skip +check_version "25.99.0" skip +check_version "29.0.0" skip +check_version "30.0.0" skip +check_version "200.0.0" skip +check_version "999.0.0" skip echo "" echo " $PASS/$TOTAL passed"