From 976adc6ab8ec33a33c780d6d7392c174d1048aa4 Mon Sep 17 00:00:00 2001 From: Florian DAVID Date: Tue, 6 Oct 2026 18:37:00 +0200 Subject: [PATCH 1/2] fix: tolerate a tied timing print in test-run-all-timing.sh (#482) test-b-slow.sh now sleeps 2s instead of 1s, widening the margin against a loaded CI runner crossing a second boundary on the near-instant fixture. The ordering assertion also compares the two suites' printed times: when they differ, the strict slow-before-fast line-position check still applies (the positive control against a real sort regression); when they tie at 1s resolution, the order is genuinely undefined, so the check reports it as an inconclusive skip rather than a flaky failure. Co-Authored-By: Claude Sonnet 5 --- changelog.d/482.fixed.run-all-timing-tie.md | 1 + tests/test-run-all-timing.sh | 25 ++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 changelog.d/482.fixed.run-all-timing-tie.md diff --git a/changelog.d/482.fixed.run-all-timing-tie.md b/changelog.d/482.fixed.run-all-timing-tie.md new file mode 100644 index 00000000..4de69f90 --- /dev/null +++ b/changelog.d/482.fixed.run-all-timing-tie.md @@ -0,0 +1 @@ +- **A 1s tie in `tests/run-all.sh`'s timing block sorted by name, failing `tests/test-run-all-timing.sh` on a loaded Windows runner** (#482). `test-b-slow.sh` sleeps long enough that it almost never ties with the near-instant fixture, and the ordering check only fails the test when the two printed times actually differ -- a genuine tie is reported as inconclusive rather than as a flake. diff --git a/tests/test-run-all-timing.sh b/tests/test-run-all-timing.sh index 63f9593a..bb39c390 100755 --- a/tests/test-run-all-timing.sh +++ b/tests/test-run-all-timing.sh @@ -61,7 +61,7 @@ exit 0 EOF cat > "$dir/test-b-slow.sh" << 'EOF' #!/bin/bash -sleep 1 +sleep 2 exit 0 EOF chmod +x "$dir"/test-*.sh @@ -103,13 +103,26 @@ else bad "the timing block comes after the tally, not before it" "tally=$TALLY_LINE timing=$TIMING_LINE" fi -# Slowest first: test-b-slow.sh (sleep 1) must be listed before test-a-fast.sh. +# Slowest first: test-b-slow.sh (sleep 2) must be listed before test-a-fast.sh -- +# unless the two land on the same whole second, which a loaded runner can do even +# with the 2s margin (#482). A tie makes the ORDER undefined at 1s resolution, not +# wrong, so it is reported as inconclusive rather than failed. A real sort +# regression still has to fail here: when the two printed times differ, the strict +# line-position check below still applies -- this is the positive control a +# tie-tolerant assertion needs. SLOW_LINE=$(grep -n 'test-b-slow\.sh' "$OUT_PASS" | tail -1 | cut -d: -f1) FAST_LINE=$(grep -n 'test-a-fast\.sh' "$OUT_PASS" | tail -1 | cut -d: -f1) -if [ -n "$SLOW_LINE" ] && [ -n "$FAST_LINE" ] && [ "$SLOW_LINE" -lt "$FAST_LINE" ]; then +SLOW_SECS=$(grep 'test-b-slow\.sh' "$OUT_PASS" | tail -1 | sed -E 's/^[[:space:]]*([0-9]+)s.*/\1/') +FAST_SECS=$(grep 'test-a-fast\.sh' "$OUT_PASS" | tail -1 | sed -E 's/^[[:space:]]*([0-9]+)s.*/\1/') +if [ -z "$SLOW_LINE" ] || [ -z "$FAST_LINE" ] || [ -z "$SLOW_SECS" ] || [ -z "$FAST_SECS" ]; then + bad "the slower suite is listed before the faster one" "could not read a printed line/time for one or both suites" + cat "$OUT_PASS" +elif [ "$SLOW_SECS" -eq "$FAST_SECS" ]; then + echo " SKIPPED: test-a-fast.sh and test-b-slow.sh both printed ${SLOW_SECS}s -- a tie at 1s resolution, order is undefined, not wrong" +elif [ "$SLOW_LINE" -lt "$FAST_LINE" ]; then ok "the slower suite is listed before the faster one" else - bad "the slower suite is listed before the faster one" "slow_line=$SLOW_LINE fast_line=$FAST_LINE" + bad "the slower suite is listed before the faster one" "slow_line=$SLOW_LINE fast_line=$FAST_LINE slow_secs=$SLOW_SECS fast_secs=$FAST_SECS" cat "$OUT_PASS" fi @@ -127,9 +140,9 @@ fi # assertion above except this one. SLOW_TIME_LINE=$(grep 'test-b-slow\.sh' "$OUT_PASS" | tail -1) if grep -qE '^ *[1-9][0-9]*s' <<< "$SLOW_TIME_LINE"; then - ok "the slow suite (sleep 1) is timed at 1 second or more, not 0" + ok "the slow suite (sleep 2) is timed at 1 second or more, not 0" else - bad "the slow suite (sleep 1) is timed at 1 second or more, not 0" "$SLOW_TIME_LINE" + bad "the slow suite (sleep 2) is timed at 1 second or more, not 0" "$SLOW_TIME_LINE" fi # --- Failing fixture: the exit code and the tally text must not change shape --- From 3b606cd4fa463835e15493ebfeb332a8324a5f19 Mon Sep 17 00:00:00 2001 From: Florian DAVID Date: Tue, 6 Oct 2026 18:55:02 +0200 Subject: [PATCH 2/2] fix: close two review findings in the #482 tie-tolerant assertion Review (Explore) found the tie-detection added for #482 was too permissive: comparing only the two suites' parsed printed times meant a stub that always printed the same constant (e.g. "1s" for every suite, a broken $SECONDS measurement) would be read as a legitimate tie and silently SKIPPED rather than failed. test-b-slow.sh sleeps a full 2 seconds, so a correct measurement can never read below 2 for it -- that is now asserted unconditionally, before the tie check runs, so a stuck/stubbed timer fails hard instead of sliding into the tie-tolerant branch. The same review also noted the closing tally line (PASS/TOTAL passed, FAIL failed) silently excluded a tie from its denominator with no indication anything was skipped. Added a SKIP counter and a skip() helper alongside ok()/bad(), and the closing line now names the skip count when one occurred. Co-Authored-By: Claude Sonnet 5 --- tests/test-run-all-timing.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/test-run-all-timing.sh b/tests/test-run-all-timing.sh index bb39c390..e02f83f9 100755 --- a/tests/test-run-all-timing.sh +++ b/tests/test-run-all-timing.sh @@ -28,6 +28,7 @@ RUN_ALL="$REPO/tests/run-all.sh" PASS=0 FAIL=0 +SKIP=0 TMP="$(mktemp -d 2> /dev/null)" || TMP="" if [ -z "$TMP" ] || [ ! -d "$TMP" ]; then @@ -47,6 +48,13 @@ bad() { [ $# -gt 0 ] && echo " $*" return 0 } +# A third outcome, distinct from both -- an assertion that genuinely cannot be +# evaluated this run (the two suites tied at whole-second resolution) is neither +# a pass nor a failure, and counting it as either would misreport what ran. +skip() { + SKIP=$((SKIP + 1)) + echo " SKIPPED: $1" +} # A minimal fixture directory: run-all.sh cds to its own dirname and globs test-*.sh # there, so a copy of it plus a couple of stub suites is a complete sandbox -- nothing @@ -110,6 +118,13 @@ fi # regression still has to fail here: when the two printed times differ, the strict # line-position check below still applies -- this is the positive control a # tie-tolerant assertion needs. +# +# A tie is only legitimate at 2s or more. test-b-slow.sh sleeps a full 2 seconds, +# so a correct measurement can never read below 2 for it -- if a broken timer +# (or one stuck printing the same constant for every suite, as a stub would) +# reports "1s" for both, that is not a resolution tie, it is the measurement +# itself being wrong, and it must fail rather than slide into SKIPPED (review +# finding on #482: the first draft let SLOW_SECS==FAST_SECS==1 through as a tie). SLOW_LINE=$(grep -n 'test-b-slow\.sh' "$OUT_PASS" | tail -1 | cut -d: -f1) FAST_LINE=$(grep -n 'test-a-fast\.sh' "$OUT_PASS" | tail -1 | cut -d: -f1) SLOW_SECS=$(grep 'test-b-slow\.sh' "$OUT_PASS" | tail -1 | sed -E 's/^[[:space:]]*([0-9]+)s.*/\1/') @@ -117,8 +132,11 @@ FAST_SECS=$(grep 'test-a-fast\.sh' "$OUT_PASS" | tail -1 | sed -E 's/^[[:space:] if [ -z "$SLOW_LINE" ] || [ -z "$FAST_LINE" ] || [ -z "$SLOW_SECS" ] || [ -z "$FAST_SECS" ]; then bad "the slower suite is listed before the faster one" "could not read a printed line/time for one or both suites" cat "$OUT_PASS" +elif [ "$SLOW_SECS" -lt 2 ]; then + bad "the slower suite is listed before the faster one" "test-b-slow.sh (sleep 2) printed ${SLOW_SECS}s, which a correct measurement cannot produce -- not a tie, a broken timer" + cat "$OUT_PASS" elif [ "$SLOW_SECS" -eq "$FAST_SECS" ]; then - echo " SKIPPED: test-a-fast.sh and test-b-slow.sh both printed ${SLOW_SECS}s -- a tie at 1s resolution, order is undefined, not wrong" + skip "test-a-fast.sh and test-b-slow.sh both printed ${SLOW_SECS}s -- a tie at 1s resolution, order is undefined, not wrong" elif [ "$SLOW_LINE" -lt "$FAST_LINE" ]; then ok "the slower suite is listed before the faster one" else @@ -134,10 +152,11 @@ else cat "$OUT_PASS" fi -# Positive control for the timing itself: the slow suite's own line names a number of -# seconds greater than 0 -- this is the case "would this pass if the code did nothing" -# is aimed at: a stub that always printed "1s" for every suite would pass every -# assertion above except this one. +# A second, independent positive control for the timing itself: the slow suite's +# own line names a number of seconds greater than 0 -- this is the case "would +# this pass if the code did nothing" is aimed at. The stronger form of this check +# (SLOW_SECS -lt 2, above) already fails a stub that always prints "1s" for every +# suite; this one is the last line of defense if that check is ever weakened. SLOW_TIME_LINE=$(grep 'test-b-slow\.sh' "$OUT_PASS" | tail -1) if grep -qE '^ *[1-9][0-9]*s' <<< "$SLOW_TIME_LINE"; then ok "the slow suite (sleep 2) is timed at 1 second or more, not 0" @@ -210,7 +229,11 @@ fi echo "" echo "========================" TOTAL=$((PASS + FAIL)) -echo " $PASS/$TOTAL passed, $FAIL failed" +if [ "$SKIP" -gt 0 ]; then + echo " $PASS/$TOTAL passed, $FAIL failed, $SKIP skipped (inconclusive, not counted above)" +else + echo " $PASS/$TOTAL passed, $FAIL failed" +fi echo "========================" [ "$FAIL" -eq 0 ] && exit 0 || exit 1