From e5d81c16bd037b09036e5f92df6174a60009b81f Mon Sep 17 00:00:00 2001 From: thomas Date: Sun, 15 Nov 2026 09:56:11 +0200 Subject: [PATCH 1/3] Run the checks this repository already owns Two of them ran in no workflow, and both had the shape the verification gate had before 845a4f2: a check nobody runs does not report a regression late, it reports nothing. `verification/tests` is the suite guarding the gate itself -- the comparison logic, the record selector, the inventory readers, the GIL-region and thread-count scans, the move bijection. 114 tests, run by hand only. Two of them are hardcoded tripwires, the `ALL` set in test_record_selector and the Makefile target spelling in test_wrap_link_guard, and a tripwire nobody runs is a comment. The other is the thread environment. Every CI run of the C suite has used the runner's default of two threads, and the Python legs run with OMP_NUM_THREADS unset, which tests/conftest.py already records as the condition that lets every test claim every core and makes several thrash. Two real defects in the parallelism work appeared only under a non-default cap: an assertion whose two outcomes collided on the same number, and a policy clause reading omp_get_num_procs() where it needed omp_get_max_threads(), which planned sixteen threads under OMP_NUM_THREADS=2. One environment cannot find that class. So the C suite now also runs at one thread and at three. One is where a reduction or a per-thread buffer can be skipped entirely; three is odd, divides no chunk count evenly, and -- unlike another power of two -- cannot collide with the plan fallback that made the test above pass by luck. At one thread the suite reports 438 passed and 1 skipped: test_plan.c:607 needs two threads to tell "the worker inherited the plan" from "the worker fell back", and skipping is the right answer there. The new tests assert the property rather than the YAML spelling, and each was mutation-tested: commenting out the tooling step, moving --threads 1 to 2, moving --threads 3 to 4, and dropping the OMP_NUM_THREADS pin each fail exactly one test and leave the other six passing. They prove a command is present outside a comment, not that the job is reached or that it passed -- the same limit the thread-plan wiring line turned out to have, stated in the file rather than discovered later. --- .github/workflows/tests.yml | 77 ++++++++++ verification/README.md | 8 ++ verification/tests/test_ci_runs_the_checks.py | 132 ++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 verification/tests/test_ci_runs_the_checks.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12fb806..42cf448 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -146,6 +146,83 @@ jobs: - name: Verification gate run: python verification/check.py + tooling: + # The tests that guard the gate itself -- the comparison logic, the record + # selector, the C inventory, the GIL-region and thread-count scans, the + # move bijection. They live outside `tests/` deliberately: CI copies + # `tests/` elsewhere and runs it against the INSTALLED package, so a test + # that imports this repository's tooling cannot live under it. + # + # Until now no workflow ran them, which is the same shape as the gate + # before 845a4f2. Two of them are hardcoded tripwires -- the `ALL` set in + # test_record_selector, the Makefile target spelling in + # test_wrap_link_guard -- and a tripwire nobody runs is a comment. + name: Verification tooling + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.11" + - name: Install BEAM in place with test dependencies + run: python -m pip install -e ".[dev]" + - name: Build the C test binaries the inventory tests read + # tests-only, not test: these tests ask the binaries what they contain. + # Running the C suite is the `native` job's business. + run: make -j"$(nproc)" tests-only + - name: Tooling test suite + run: python -m pytest verification/tests -q + + threads: + # One thread environment cannot find a defect that appears only in + # another, and two real ones in the parallelism work appeared only under a + # non-default cap: an assertion whose two outcomes collided on the same + # number, and a policy clause reading `omp_get_num_procs()` where it + # needed `omp_get_max_threads()` -- which planned 16 threads under + # OMP_NUM_THREADS=2. Running a test binary directly finds neither. + # + # The C runner pins OMP_NUM_THREADS=2 for every suite, so every CI run of + # it so far has used one environment. One is the case where a reduction or + # a per-thread buffer can be skipped entirely; three is odd and divides no + # chunk count evenly, which is the other half of the class. + # + # The Python leg is pinned to three for a second reason: unset is what the + # six matrix legs use today, and tests/conftest.py records that an unset + # cap lets every test claim every core and several then thrash rather than + # run. One environment here was chosen by the runner's hardware rather + # than by this repository. + name: Thread environments + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.11" + - name: Install BEAM in place with test dependencies + run: python -m pip install -e ".[dev]" + - name: Build the C test binaries once for both runs + run: make -j"$(nproc)" tests-only + - name: C suite, one thread per suite + run: tools/qa/run c --bin-dir build/tests/native --threads 1 + - name: C suite, three threads per suite + run: tools/qa/run c --bin-dir build/tests/native --threads 3 + - name: Python suite, three threads + # From the source tree against the editable install, unlike the six + # matrix legs, which run from a copy against the INSTALLED package to + # catch a file missing from the distribution. That is their job; this + # one varies the thread environment and nothing else, and running in + # place keeps the difference to one variable. + # + # faulthandler_timeout for the same reason the matrix legs carry it, + # only more so: a thread environment nobody has run before is where a + # deadlock would appear, and a hang should print its stacks rather + # than burn the 45-minute budget in silence. + env: + OMP_NUM_THREADS: "3" + run: python -m pytest tests -q -o faulthandler_timeout=300 + packaging: name: Build and check distributions runs-on: ubuntu-latest diff --git a/verification/README.md b/verification/README.md index 039612d..9dc20a1 100644 --- a/verification/README.md +++ b/verification/README.md @@ -50,8 +50,16 @@ it went wrong both times. make clean && make test # 331 C tests .venv/bin/python -m pytest tests -q # Python suite .venv/bin/python verification/check.py # the contract below +.venv/bin/python -m pytest verification/tests -q # the tests guarding this gate ``` +The last line is the suite that guards the checks themselves -- the comparison +logic, the record selector, the inventory readers, the GIL-region and +thread-count scans. It lives outside `tests/` because CI copies `tests/` +elsewhere and runs it against the installed package, so a test that imports +this repository's tooling cannot live under it. It runs in CI as +`Verification tooling`; until that job existed it ran only by hand. + `check.py` compares, against `baseline/`: | Check | What it catches | diff --git a/verification/tests/test_ci_runs_the_checks.py b/verification/tests/test_ci_runs_the_checks.py new file mode 100644 index 0000000..460b515 --- /dev/null +++ b/verification/tests/test_ci_runs_the_checks.py @@ -0,0 +1,132 @@ +"""The checks this repository owns are of no use unless CI runs them. + +Two of them were not run by any workflow, and both had the same shape as the +verification gate before `845a4f2`: a check nobody runs does not report a +regression late, it reports nothing. + + * `verification/tests` -- the suite guarding the gate itself, including two + hardcoded tripwires (the `ALL` set in `test_record_selector`, the Makefile + target spelling in `test_wrap_link_guard`) that only pay off if something + runs them. + * More than one thread environment. Two real defects in the parallelism work + appeared ONLY under a non-default `OMP_NUM_THREADS`: an assertion whose two + outcomes collided on the same number, and a policy clause reading + `omp_get_num_procs()` where it needed `omp_get_max_threads()`. The C runner + pins `OMP_NUM_THREADS=2` for every suite, so a single run of it cannot see + that class at all. + +WHAT THESE TESTS PROVE, AND WHAT THEY DO NOT. They read the workflow text and +prove a command is PRESENT in it, outside a comment. They do not prove the job +is reached, that its `if:` lets it run, or that it passed -- the same +limitation the thread-plan wiring check turned out to have, and worth stating +here rather than discovering later. What they are is a tripwire: delete the +job and a test fails, which is the property that was missing. + +Deliberately text-based. Parsing the YAML properly would mean adding pyyaml to +the dev dependencies for one tripwire, and these assertions are about commands +appearing in a file, which does not need a parser. +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows" + +# The C runner's own default, which every CI run of the C suite has used. A +# second environment has to differ from THIS, not merely exist. +RUNNER_DEFAULT_THREADS = 2 + + +def workflow_lines() -> list[tuple[str, str]]: + """Every (file, line) in the workflows, with comment-only lines dropped. + + A `#` inside a quoted string is not a comment, but no workflow here has + one, and the alternative is a YAML parser. Lines whose first non-space + character is `#` are the ones that matter: commenting a job out is how a + check stops running without anyone deleting it. + """ + out = [] + for path in sorted(WORKFLOWS.glob("*.yml")): + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("#"): + continue + out.append((path.name, line)) + return out + + +def test_some_workflow_runs_the_tooling_suite(): + """verification/tests runs in CI, not only by hand.""" + hits = [(f, l) for (f, l) in workflow_lines() + if "verification/tests" in l and "pytest" in l] + assert hits, ( + "No workflow runs `pytest verification/tests`. These are the tests " + "that guard the gate; running them by hand means they guard it only " + "on the days someone remembers." + ) + + +def test_the_c_suite_runs_at_more_than_one_thread_count(): + """Some workflow runs the C suite at a thread count other than the default.""" + counts = set() + for (_f, line) in workflow_lines(): + for m in re.finditer(r"--threads[= ]+(\d+)", line): + counts.add(int(m.group(1))) + extra = counts - {RUNNER_DEFAULT_THREADS} + assert extra, ( + f"Every CI run of the C suite uses the runner default of " + f"{RUNNER_DEFAULT_THREADS} threads (found: {sorted(counts) or 'none'}). " + "Two real defects in this work were visible only at a different " + "thread count, so one environment cannot find that class." + ) + + +def test_a_thread_count_of_one_is_among_them(): + """One thread is the environment that catches 'assumes a team'.""" + counts = {int(m.group(1)) + for (_f, line) in workflow_lines() + for m in re.finditer(r"--threads[= ]+(\d+)", line)} + assert 1 in counts, ( + f"No workflow runs the C suite single-threaded (found: {sorted(counts)}). " + "A team of one is the case where a reduction, a barrier or a " + "per-thread buffer can be skipped entirely." + ) + + +def test_an_odd_thread_count_is_among_them(): + """An odd count catches 'assumes the work divides evenly'.""" + counts = {int(m.group(1)) + for (_f, line) in workflow_lines() + for m in re.finditer(r"--threads[= ]+(\d+)", line)} + odd = {c for c in counts if c > 1 and c % 2 == 1} + assert odd, ( + f"No workflow runs the C suite at an odd thread count above one " + f"(found: {sorted(counts)}). Powers of two hide off-by-one chunking: " + "every count CI has ever used divides the chunk counts evenly." + ) + + +def test_the_python_suite_sees_a_pinned_thread_environment(): + """Some workflow sets OMP_NUM_THREADS for Python tests.""" + hits = [(f, l) for (f, l) in workflow_lines() if "OMP_NUM_THREADS" in l] + assert hits, ( + "No workflow sets OMP_NUM_THREADS. The Python legs therefore all run " + "at whatever the runner's core count implies, which is one " + "environment chosen by the hardware rather than by this repository." + ) + + +@pytest.mark.parametrize("command", [ + "verification/check.py", # the gate + "make -j", # the C suite +]) +def test_the_checks_that_already_ran_are_still_wired(command): + """A regression guard for the jobs that were already there. + + Without this, a later edit could satisfy the tests above while quietly + dropping the checks CI did run. + """ + assert any(command in l for (_f, l) in workflow_lines()), \ + f"No workflow runs `{command}` any more." From 0d3926a3e5c6a0f972efcce54c221868e59edae8 Mon Sep 17 00:00:00 2001 From: thomas Date: Sun, 15 Nov 2026 09:56:19 +0200 Subject: [PATCH 2/3] Say what the thread-counts check catches, and correct two counts The gate grew a line in 7f3717e and the table here never gained the row, so the one check that would have caught a region running at one thread per core regardless of the plan was undocumented in the file that explains the gate. Two counts beside it had drifted with the suites they describe: the C half is 439 tests, not 331, and `exact` holds 416 quantities, not 351. Neither number is recorded anywhere the gate reads, which is why they could drift in the first place. --- verification/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/verification/README.md b/verification/README.md index 9dc20a1..e78df08 100644 --- a/verification/README.md +++ b/verification/README.md @@ -47,7 +47,7 @@ it went wrong both times. ## Running the gate ```bash -make clean && make test # 331 C tests +make clean && make test # 439 C tests .venv/bin/python -m pytest tests -q # Python suite .venv/bin/python verification/check.py # the contract below .venv/bin/python -m pytest verification/tests -q # the tests guarding this gate @@ -70,10 +70,11 @@ this repository's tooling cannot live under it. It runs in CI as | pytest inventory | a test that stopped being collected -- a subset of a passing suite also passes | | C test inventory | the same for `RUN_TEST` cases | | thread plan wiring | a region that drops the GIL and resolves no thread plan, an unbalanced plan macro, or a path that escapes one | +| thread counts | a parallel region naming no thread count -- so it runs at one thread per core whatever the plan resolved -- or naming one outside the permitted forms | | numeric | 78 quantities, order 2/3/4 x default/uneven prior, as hex floats | | saved-model bytes | the sha256 of a written model | | thread independence | results at 1, 4 and 16 threads | -| exact | 351 quantities against brute-force enumeration of the whole support | +| exact | 416 quantities against brute-force enumeration of the whole support | Every row above `exact` pins REPRODUCIBILITY: each quantity is compared with a recording of ITSELF, so a quantity that has been wrong since the day it was From 757b1223b3f2940248e0806065c0a997b97ad13a Mon Sep 17 00:00:00 2001 From: thomas Date: Sun, 15 Nov 2026 10:13:54 +0200 Subject: [PATCH 3/3] Say which clause two plan tests are about The new thread-environment job failed on its first run, which is the job working: under OMP_NUM_THREADS=3 two tests in test_runtime.py failed, one asserting the plan never exceeds availability and one asserting a pinned BEAM_AVAILABLE_CORES drives the plan. Neither is a defect in BEAM. Clause 2 of the policy (src/native/core/plan.c) treats OMP_NUM_THREADS as the caller deciding, exactly as set_threads() does, so the runtime's own answer stands and availability does not revise it -- a plan of 3 against an availability of 2 is the documented behaviour, and under a cap of 16 the plan is 16, wider than max_width, for the same reason. Both tests describe clause 3, the path that reads the free cores, and neither said so. They passed for years because no run had ever set the variable: a test whose precondition is "the environment nobody varied" is not wrong, it is silently narrow, and the fix is to state the precondition rather than to weaken the assertion. So each skips when the plan reports omp_num_threads, which is the value the policy actually read rather than a second guess at os.environ. This is how the C side already does it: test_the_default_scope_stands_aside_for_omp_ num_threads skips unless the variable is set at process start, and pins availability to 64 to prove clause 2 ignores it. Between them the two clauses are covered in the environments where each applies -- clause 3 on the six matrix legs where the variable is unset, clause 2 on the C side and in the new job. The pinned-availability test keeps its first assertion unconditional: that BEAM_AVAILABLE_CORES is observable in the plan holds under every clause, and only its effect on the width is clause 3's business. Verified both ways: under OMP_NUM_THREADS=3 the two skip with their reasons and the file's other 12 tests pass; with the variable unset both run and pass, so the guard is conditional rather than a disable. The test ids do not move -- collection is byte-identical to pytest_cases.txt at 1243. --- tests/suites/model/test_runtime.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/suites/model/test_runtime.py b/tests/suites/model/test_runtime.py index 924c4db..06ed47b 100644 --- a/tests/suites/model/test_runtime.py +++ b/tests/suites/model/test_runtime.py @@ -131,13 +131,24 @@ def test_an_explicit_cap_is_what_the_plan_reports(): def test_the_plan_is_capped_by_the_scaling_ceiling(): """Auto mode never plans wider than where the work stops scaling, which - is the measured upper limit rather than the machine's size.""" + is the measured upper limit rather than the machine's size. + + Both assertions describe clause 3 of the policy, the one that reads the + free cores. OMP_NUM_THREADS selects clause 2 instead, where the runtime's + own answer stands and neither the ceiling nor availability revises it -- + so under a cap of 16 the plan is 16, wider than `max_width`, by design. + This test asserted clause 3 without saying so and passed only because no + run had ever set the variable; it is skipped there rather than weakened, + the way the C side's clause-2 test skips when the variable is absent. + """ import BEAM before = BEAM.threads() try: BEAM.set_threads(0) plan = BEAM.parallel_plan() + if plan["omp_num_threads"]: + pytest.skip("OMP_NUM_THREADS selects clause 2, not the ceiling") assert plan["plan"] <= plan["max_width"] assert plan["plan"] <= max(plan["available"], 1) finally: @@ -154,7 +165,14 @@ def test_a_pinned_availability_is_visible_in_the_plan(monkeypatch): try: BEAM.set_threads(0) plan = BEAM.parallel_plan() + # Observable in every environment: the pin is what the policy read. assert plan["available"] == 2 + if plan["omp_num_threads"]: + # Clause 2 stands aside for nothing, which is the property the C + # side asserts directly by pinning availability to 64 and + # expecting omp_get_max_threads() anyway + # (test_the_default_scope_stands_aside_for_omp_num_threads). + pytest.skip("OMP_NUM_THREADS selects clause 2, which ignores availability") assert plan["plan"] == 2 finally: BEAM.set_threads(before)