From 26fcc7f16f455f9bfc3d86bebab8a87660157ed9 Mon Sep 17 00:00:00 2001 From: Sergi Gonzalez <31130069+TheHypnoo@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:10:11 +0200 Subject: [PATCH] test(node-suite): stop counting fixtures/ modules as tests Both node-suite discoverers (scripts/node_suite_run.py and run_parity_tests.sh --suite node-suite) treated every .ts file as a test, including modules under fixtures/ that tests only import. Those files print nothing, so each one added a trivial pass and inflated the totals: module/loader/fixtures contributed ten. A fixtures/ directory now never contains test entries. The runner gets a --self-test that pins the rule, wired into the lint job next to the test registration check. The module lane goes from 69 to 59 entries, so its total floor in node_suite_baseline.json drops accordingly. --- .github/workflows/test.yml | 1 + run_parity_tests.sh | 2 +- scripts/check_test_registration.py | 3 +- scripts/node_suite_run.py | 55 ++++++++++++++++++++----- test-parity/node-suite/module/README.md | 9 ++-- test-parity/node_suite_baseline.json | 4 +- 6 files changed, 55 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b055d430ff..aa16efe2b56 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -871,6 +871,7 @@ jobs: run: | python3 scripts/check_test_registration.py --self-test python3 scripts/check_test_registration.py + python3 scripts/node_suite_run.py --self-test # #7672: the GC test guards CLEAR ~20 process-global side tables from # whatever libtest thread constructs a guard, and nothing requires a diff --git a/run_parity_tests.sh b/run_parity_tests.sh index 6855040c1b5..7816294a446 100755 --- a/run_parity_tests.sh +++ b/run_parity_tests.sh @@ -1308,7 +1308,7 @@ case "$TEST_SUITE" in if [[ -d "$node_suite_search_root" ]]; then while IFS= read -r test_file; do TEST_FILES+=("$test_file") - done < <(find "$node_suite_search_root" -type f -name '*.ts' | sort) + done < <(find "$node_suite_search_root" -type f -name '*.ts' -not -path '*/fixtures/*' | sort) fi ;; *) diff --git a/scripts/check_test_registration.py b/scripts/check_test_registration.py index 06b6d794125..55b4b0618ed 100644 --- a/scripts/check_test_registration.py +++ b/scripts/check_test_registration.py @@ -368,7 +368,8 @@ def _stem(rel: str) -> str: # absence from MECHANISMS is exactly the silence this gate exists to remove. GLOB_DRIVEN = ( ("run_parity_tests.sh", "find test-files -maxdepth 1 -name '*.ts'; " - "find test-parity/node-suite -name '*.ts'"), + "find test-parity/node-suite -name '*.ts' -not -path '*/fixtures/*' " + "(scripts/node_suite_run.py applies the same rule; its --self-test pins it)"), ("scripts/gc_root_dominance_corpus.sh", "PATTERNS globs over test-files/; " "a pattern that matches nothing is already loud, and MIN_COMPILED floors " "the corpus size"), diff --git a/scripts/node_suite_run.py b/scripts/node_suite_run.py index bb14a83f789..da599229fd6 100644 --- a/scripts/node_suite_run.py +++ b/scripts/node_suite_run.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 """Differential runner for the print-and-diff node-suite (test-parity/node-suite). -For every `test-parity/node-suite//**/*.ts`, run `node ` and +For every `test-parity/node-suite//**/*.ts` outside a `fixtures/` +directory, run `node ` and `perry -o out && out`, then compare stdout (trailing whitespace ignored). Prints a per-module pass/total table plus an overall figure. @@ -20,7 +21,11 @@ run STRICTLY SEQUENTIALLY so their numbers are trustworthy; everything else stays parallel. +A `fixtures/` directory holds modules that tests import, never tests: its `.ts` +files print nothing, so counting them would only add trivial passes. + Usage: node_suite_run.py [comma-separated-modules] + node_suite_run.py --self-test """ import os, re, subprocess, sys, tempfile from concurrent.futures import ThreadPoolExecutor @@ -71,6 +76,43 @@ def normalize(text: str) -> str: return "\n".join(out) +FIXTURE_DIR = "fixtures" + + +def discover(base, mods=None): + """(module, path) for every test entry under `base`, skipping `fixtures/`.""" + found = [] + for mod in (mods or sorted(os.listdir(base))): + md = os.path.join(base, mod) + if not os.path.isdir(md): + continue + for dp, dirs, files in os.walk(md): + dirs[:] = [d for d in dirs if d != FIXTURE_DIR] + for f in files: + if f.endswith(".ts") and not f.endswith(".d.ts"): + found.append((mod, os.path.join(dp, f))) + return found + + +def self_test(): + with tempfile.TemporaryDirectory() as td: + for rel in ("m/a.ts", "m/sub/b.ts", "m/sub/fixtures/dep.ts", + "m/fixtures/nested/dep.ts", "m/types.d.ts", "n/c.ts"): + path = os.path.join(td, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w").close() + got = sorted(os.path.relpath(p, td) for _, p in discover(td)) + want = ["m/a.ts", "m/sub/b.ts", "n/c.ts"] + if got != want: + print(f"FAIL: discovered {got}, want {want}") + return 1 + print("OK: node-suite discovery skips fixtures/ and .d.ts") + return 0 + + +if len(sys.argv) > 1 and sys.argv[1] == "--self-test": + sys.exit(self_test()) + PERRY = sys.argv[1] ROOT = sys.argv[2] MODS = sys.argv[3].split(",") if len(sys.argv) > 3 and sys.argv[3] else None @@ -87,16 +129,7 @@ def normalize(text: str) -> str: "trace_events", } -tests = [] -base = os.path.join(ROOT, "test-parity", "node-suite") -for mod in (MODS or sorted(os.listdir(base))): - md = os.path.join(base, mod) - if not os.path.isdir(md): - continue - for dp, _, files in os.walk(md): - for f in files: - if f.endswith(".ts") and not f.endswith(".d.ts"): - tests.append((mod, os.path.join(dp, f))) +tests = discover(os.path.join(ROOT, "test-parity", "node-suite"), MODS) def run_one(args): diff --git a/test-parity/node-suite/module/README.md b/test-parity/node-suite/module/README.md index cd443764df1..71d01892f7d 100644 --- a/test-parity/node-suite/module/README.md +++ b/test-parity/node-suite/module/README.md @@ -21,10 +21,11 @@ to 69**. The 69 entries are grouped by directory as follows: | `require` | 13 | 15 | overloads, resolution, cache, cycles, metadata, packages | | `source-map` | 7 | 7 | payloads, getters, mappings, origins, validation | -`loader/fixtures/**/*.ts` contains ten pre-existing executable fixture modules. -The generic runner intentionally counts every `.ts` file recursively, so they -remain represented in the total and baseline rather than being silently -subtracted in this document. +`loader/fixtures/**/*.ts` contains ten fixture modules that the loader tests +import. The runners skip every `fixtures/` directory, so these modules are not +counted as tests: the runner-visible lane is **59** entries (69 minus those +ten). The historical figures below were measured when the fixtures were still +counted. ## Primary evidence diff --git a/test-parity/node_suite_baseline.json b/test-parity/node_suite_baseline.json index 2ea471f5ac9..8a904fa38bf 100644 --- a/test-parity/node_suite_baseline.json +++ b/test-parity/node_suite_baseline.json @@ -2,7 +2,7 @@ "_schema": { "description": "Floor baseline for scripts/node_suite_regression_check.py. Each module's run must produce pass >= floor.pass and total >= floor.total; dropping below either is a regression (exit 1). Improvements are always accepted and reported as ratchet candidates. Captured in the node-26 environment with scripts/node_suite_run.py (pre-warm + fast/slow lanes).", "oracle": "node v26.5.0 (pinned by .node-version)", - "note": "Full pass sets the floor for deterministic modules. Timing/racy modules (net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9). node_suite_run.normalize() scrubs environment-variant tokens (console.time hrtime durations, stack-trace frame lines) symmetrically before the stdout compare, so console uses a full floor (119) for its deterministic content. Three clean focused Perry runs against Node 26.5.0 set the expanded HTTP matched floor at 22; #5891 adds one current unmatched regression and raises the total floor to 53. The expanded net floor is stable at 16/47. The granular http2 matched floor is 32; #5891 adds one current unmatched regression and raises its total floor to 60. The https suite is stable at 6/47 with no margin. The deterministic readline suite is stable at 20/95 with no margin. A clean node-26 run at 2810/2863 (98.1%) refreshed the floors, then an independent Node 26.5.0 run set the deterministic child_process floor at 43/53." + "note": "Full pass sets the floor for deterministic modules. Timing/racy modules (net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9). node_suite_run.normalize() scrubs environment-variant tokens (console.time hrtime durations, stack-trace frame lines) symmetrically before the stdout compare, so console uses a full floor (119) for its deterministic content. Three clean focused Perry runs against Node 26.5.0 set the expanded HTTP matched floor at 22; #5891 adds one current unmatched regression and raises the total floor to 53. The expanded net floor is stable at 16/47. The granular http2 matched floor is 32; #5891 adds one current unmatched regression and raises its total floor to 60. The https suite is stable at 6/47 with no margin. The deterministic readline suite is stable at 20/95 with no margin. A clean node-26 run at 2810/2863 (98.1%) refreshed the floors, then an independent Node 26.5.0 run set the deterministic child_process floor at 43/53. Files under a fixtures/ directory are imported modules, not tests, and are no longer discovered; module's total floor drops from 69 to 59 for the ten loader/fixtures modules (its pass floor of 29 stays below the 57/59 measured without them)." }, "modules": { "assert": { @@ -117,7 +117,7 @@ }, "module": { "pass": 29, - "total": 69 + "total": 59 }, "net": { "pass": 16,