diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b055d430ff..aa16efe2b56 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -871,6 +871,7 @@ jobs: run: | python3 scripts/check_test_registration.py --self-test python3 scripts/check_test_registration.py + python3 scripts/node_suite_run.py --self-test # #7672: the GC test guards CLEAR ~20 process-global side tables from # whatever libtest thread constructs a guard, and nothing requires a diff --git a/run_parity_tests.sh b/run_parity_tests.sh index 6855040c1b5..7816294a446 100755 --- a/run_parity_tests.sh +++ b/run_parity_tests.sh @@ -1308,7 +1308,7 @@ case "$TEST_SUITE" in if [[ -d "$node_suite_search_root" ]]; then while IFS= read -r test_file; do TEST_FILES+=("$test_file") - done < <(find "$node_suite_search_root" -type f -name '*.ts' | sort) + done < <(find "$node_suite_search_root" -type f -name '*.ts' -not -path '*/fixtures/*' | sort) fi ;; *) diff --git a/scripts/check_test_registration.py b/scripts/check_test_registration.py index 06b6d794125..55b4b0618ed 100644 --- a/scripts/check_test_registration.py +++ b/scripts/check_test_registration.py @@ -368,7 +368,8 @@ def _stem(rel: str) -> str: # absence from MECHANISMS is exactly the silence this gate exists to remove. GLOB_DRIVEN = ( ("run_parity_tests.sh", "find test-files -maxdepth 1 -name '*.ts'; " - "find test-parity/node-suite -name '*.ts'"), + "find test-parity/node-suite -name '*.ts' -not -path '*/fixtures/*' " + "(scripts/node_suite_run.py applies the same rule; its --self-test pins it)"), ("scripts/gc_root_dominance_corpus.sh", "PATTERNS globs over test-files/; " "a pattern that matches nothing is already loud, and MIN_COMPILED floors " "the corpus size"), diff --git a/scripts/node_suite_run.py b/scripts/node_suite_run.py index bb14a83f789..da599229fd6 100644 --- a/scripts/node_suite_run.py +++ b/scripts/node_suite_run.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 """Differential runner for the print-and-diff node-suite (test-parity/node-suite). -For every `test-parity/node-suite//**/*.ts`, run `node ` and +For every `test-parity/node-suite//**/*.ts` outside a `fixtures/` +directory, run `node ` and `perry -o out && out`, then compare stdout (trailing whitespace ignored). Prints a per-module pass/total table plus an overall figure. @@ -20,7 +21,11 @@ run STRICTLY SEQUENTIALLY so their numbers are trustworthy; everything else stays parallel. +A `fixtures/` directory holds modules that tests import, never tests: its `.ts` +files print nothing, so counting them would only add trivial passes. + Usage: node_suite_run.py [comma-separated-modules] + node_suite_run.py --self-test """ import os, re, subprocess, sys, tempfile from concurrent.futures import ThreadPoolExecutor @@ -71,6 +76,43 @@ def normalize(text: str) -> str: return "\n".join(out) +FIXTURE_DIR = "fixtures" + + +def discover(base, mods=None): + """(module, path) for every test entry under `base`, skipping `fixtures/`.""" + found = [] + for mod in (mods or sorted(os.listdir(base))): + md = os.path.join(base, mod) + if not os.path.isdir(md): + continue + for dp, dirs, files in os.walk(md): + dirs[:] = [d for d in dirs if d != FIXTURE_DIR] + for f in files: + if f.endswith(".ts") and not f.endswith(".d.ts"): + found.append((mod, os.path.join(dp, f))) + return found + + +def self_test(): + with tempfile.TemporaryDirectory() as td: + for rel in ("m/a.ts", "m/sub/b.ts", "m/sub/fixtures/dep.ts", + "m/fixtures/nested/dep.ts", "m/types.d.ts", "n/c.ts"): + path = os.path.join(td, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w").close() + got = sorted(os.path.relpath(p, td) for _, p in discover(td)) + want = ["m/a.ts", "m/sub/b.ts", "n/c.ts"] + if got != want: + print(f"FAIL: discovered {got}, want {want}") + return 1 + print("OK: node-suite discovery skips fixtures/ and .d.ts") + return 0 + + +if len(sys.argv) > 1 and sys.argv[1] == "--self-test": + sys.exit(self_test()) + PERRY = sys.argv[1] ROOT = sys.argv[2] MODS = sys.argv[3].split(",") if len(sys.argv) > 3 and sys.argv[3] else None @@ -87,16 +129,7 @@ def normalize(text: str) -> str: "trace_events", } -tests = [] -base = os.path.join(ROOT, "test-parity", "node-suite") -for mod in (MODS or sorted(os.listdir(base))): - md = os.path.join(base, mod) - if not os.path.isdir(md): - continue - for dp, _, files in os.walk(md): - for f in files: - if f.endswith(".ts") and not f.endswith(".d.ts"): - tests.append((mod, os.path.join(dp, f))) +tests = discover(os.path.join(ROOT, "test-parity", "node-suite"), MODS) def run_one(args): diff --git a/test-parity/node-suite/module/README.md b/test-parity/node-suite/module/README.md index cd443764df1..71d01892f7d 100644 --- a/test-parity/node-suite/module/README.md +++ b/test-parity/node-suite/module/README.md @@ -21,10 +21,11 @@ to 69**. The 69 entries are grouped by directory as follows: | `require` | 13 | 15 | overloads, resolution, cache, cycles, metadata, packages | | `source-map` | 7 | 7 | payloads, getters, mappings, origins, validation | -`loader/fixtures/**/*.ts` contains ten pre-existing executable fixture modules. -The generic runner intentionally counts every `.ts` file recursively, so they -remain represented in the total and baseline rather than being silently -subtracted in this document. +`loader/fixtures/**/*.ts` contains ten fixture modules that the loader tests +import. The runners skip every `fixtures/` directory, so these modules are not +counted as tests: the runner-visible lane is **59** entries (69 minus those +ten). The historical figures below were measured when the fixtures were still +counted. ## Primary evidence diff --git a/test-parity/node_suite_baseline.json b/test-parity/node_suite_baseline.json index 2ea471f5ac9..8a904fa38bf 100644 --- a/test-parity/node_suite_baseline.json +++ b/test-parity/node_suite_baseline.json @@ -2,7 +2,7 @@ "_schema": { "description": "Floor baseline for scripts/node_suite_regression_check.py. Each module's run must produce pass >= floor.pass and total >= floor.total; dropping below either is a regression (exit 1). Improvements are always accepted and reported as ratchet candidates. Captured in the node-26 environment with scripts/node_suite_run.py (pre-warm + fast/slow lanes).", "oracle": "node v26.5.0 (pinned by .node-version)", - "note": "Full pass sets the floor for deterministic modules. Timing/racy modules (net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9). node_suite_run.normalize() scrubs environment-variant tokens (console.time hrtime durations, stack-trace frame lines) symmetrically before the stdout compare, so console uses a full floor (119) for its deterministic content. Three clean focused Perry runs against Node 26.5.0 set the expanded HTTP matched floor at 22; #5891 adds one current unmatched regression and raises the total floor to 53. The expanded net floor is stable at 16/47. The granular http2 matched floor is 32; #5891 adds one current unmatched regression and raises its total floor to 60. The https suite is stable at 6/47 with no margin. The deterministic readline suite is stable at 20/95 with no margin. A clean node-26 run at 2810/2863 (98.1%) refreshed the floors, then an independent Node 26.5.0 run set the deterministic child_process floor at 43/53." + "note": "Full pass sets the floor for deterministic modules. Timing/racy modules (net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9). node_suite_run.normalize() scrubs environment-variant tokens (console.time hrtime durations, stack-trace frame lines) symmetrically before the stdout compare, so console uses a full floor (119) for its deterministic content. Three clean focused Perry runs against Node 26.5.0 set the expanded HTTP matched floor at 22; #5891 adds one current unmatched regression and raises the total floor to 53. The expanded net floor is stable at 16/47. The granular http2 matched floor is 32; #5891 adds one current unmatched regression and raises its total floor to 60. The https suite is stable at 6/47 with no margin. The deterministic readline suite is stable at 20/95 with no margin. A clean node-26 run at 2810/2863 (98.1%) refreshed the floors, then an independent Node 26.5.0 run set the deterministic child_process floor at 43/53. Files under a fixtures/ directory are imported modules, not tests, and are no longer discovered; module's total floor drops from 69 to 59 for the ten loader/fixtures modules (its pass floor of 29 stays below the 57/59 measured without them)." }, "modules": { "assert": { @@ -117,7 +117,7 @@ }, "module": { "pass": 29, - "total": 69 + "total": 59 }, "net": { "pass": 16,