diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12fb806..42cf448 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -146,6 +146,83 @@ jobs: - name: Verification gate run: python verification/check.py + tooling: + # The tests that guard the gate itself -- the comparison logic, the record + # selector, the C inventory, the GIL-region and thread-count scans, the + # move bijection. They live outside `tests/` deliberately: CI copies + # `tests/` elsewhere and runs it against the INSTALLED package, so a test + # that imports this repository's tooling cannot live under it. + # + # Until now no workflow ran them, which is the same shape as the gate + # before 845a4f2. Two of them are hardcoded tripwires -- the `ALL` set in + # test_record_selector, the Makefile target spelling in + # test_wrap_link_guard -- and a tripwire nobody runs is a comment. + name: Verification tooling + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.11" + - name: Install BEAM in place with test dependencies + run: python -m pip install -e ".[dev]" + - name: Build the C test binaries the inventory tests read + # tests-only, not test: these tests ask the binaries what they contain. + # Running the C suite is the `native` job's business. + run: make -j"$(nproc)" tests-only + - name: Tooling test suite + run: python -m pytest verification/tests -q + + threads: + # One thread environment cannot find a defect that appears only in + # another, and two real ones in the parallelism work appeared only under a + # non-default cap: an assertion whose two outcomes collided on the same + # number, and a policy clause reading `omp_get_num_procs()` where it + # needed `omp_get_max_threads()` -- which planned 16 threads under + # OMP_NUM_THREADS=2. Running a test binary directly finds neither. + # + # The C runner pins OMP_NUM_THREADS=2 for every suite, so every CI run of + # it so far has used one environment. One is the case where a reduction or + # a per-thread buffer can be skipped entirely; three is odd and divides no + # chunk count evenly, which is the other half of the class. + # + # The Python leg is pinned to three for a second reason: unset is what the + # six matrix legs use today, and tests/conftest.py records that an unset + # cap lets every test claim every core and several then thrash rather than + # run. One environment here was chosen by the runner's hardware rather + # than by this repository. + name: Thread environments + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.11" + - name: Install BEAM in place with test dependencies + run: python -m pip install -e ".[dev]" + - name: Build the C test binaries once for both runs + run: make -j"$(nproc)" tests-only + - name: C suite, one thread per suite + run: tools/qa/run c --bin-dir build/tests/native --threads 1 + - name: C suite, three threads per suite + run: tools/qa/run c --bin-dir build/tests/native --threads 3 + - name: Python suite, three threads + # From the source tree against the editable install, unlike the six + # matrix legs, which run from a copy against the INSTALLED package to + # catch a file missing from the distribution. That is their job; this + # one varies the thread environment and nothing else, and running in + # place keeps the difference to one variable. + # + # faulthandler_timeout for the same reason the matrix legs carry it, + # only more so: a thread environment nobody has run before is where a + # deadlock would appear, and a hang should print its stacks rather + # than burn the 45-minute budget in silence. + env: + OMP_NUM_THREADS: "3" + run: python -m pytest tests -q -o faulthandler_timeout=300 + packaging: name: Build and check distributions runs-on: ubuntu-latest diff --git a/tests/suites/model/test_runtime.py b/tests/suites/model/test_runtime.py index 924c4db..06ed47b 100644 --- a/tests/suites/model/test_runtime.py +++ b/tests/suites/model/test_runtime.py @@ -131,13 +131,24 @@ def test_an_explicit_cap_is_what_the_plan_reports(): def test_the_plan_is_capped_by_the_scaling_ceiling(): """Auto mode never plans wider than where the work stops scaling, which - is the measured upper limit rather than the machine's size.""" + is the measured upper limit rather than the machine's size. + + Both assertions describe clause 3 of the policy, the one that reads the + free cores. OMP_NUM_THREADS selects clause 2 instead, where the runtime's + own answer stands and neither the ceiling nor availability revises it -- + so under a cap of 16 the plan is 16, wider than `max_width`, by design. + This test asserted clause 3 without saying so and passed only because no + run had ever set the variable; it is skipped there rather than weakened, + the way the C side's clause-2 test skips when the variable is absent. + """ import BEAM before = BEAM.threads() try: BEAM.set_threads(0) plan = BEAM.parallel_plan() + if plan["omp_num_threads"]: + pytest.skip("OMP_NUM_THREADS selects clause 2, not the ceiling") assert plan["plan"] <= plan["max_width"] assert plan["plan"] <= max(plan["available"], 1) finally: @@ -154,7 +165,14 @@ def test_a_pinned_availability_is_visible_in_the_plan(monkeypatch): try: BEAM.set_threads(0) plan = BEAM.parallel_plan() + # Observable in every environment: the pin is what the policy read. assert plan["available"] == 2 + if plan["omp_num_threads"]: + # Clause 2 stands aside for nothing, which is the property the C + # side asserts directly by pinning availability to 64 and + # expecting omp_get_max_threads() anyway + # (test_the_default_scope_stands_aside_for_omp_num_threads). + pytest.skip("OMP_NUM_THREADS selects clause 2, which ignores availability") assert plan["plan"] == 2 finally: BEAM.set_threads(before) diff --git a/verification/README.md b/verification/README.md index 039612d..e78df08 100644 --- a/verification/README.md +++ b/verification/README.md @@ -47,11 +47,19 @@ it went wrong both times. ## Running the gate ```bash -make clean && make test # 331 C tests +make clean && make test # 439 C tests .venv/bin/python -m pytest tests -q # Python suite .venv/bin/python verification/check.py # the contract below +.venv/bin/python -m pytest verification/tests -q # the tests guarding this gate ``` +The last line is the suite that guards the checks themselves -- the comparison +logic, the record selector, the inventory readers, the GIL-region and +thread-count scans. It lives outside `tests/` because CI copies `tests/` +elsewhere and runs it against the installed package, so a test that imports +this repository's tooling cannot live under it. It runs in CI as +`Verification tooling`; until that job existed it ran only by hand. + `check.py` compares, against `baseline/`: | Check | What it catches | @@ -62,10 +70,11 @@ make clean && make test # 331 C tests | pytest inventory | a test that stopped being collected -- a subset of a passing suite also passes | | C test inventory | the same for `RUN_TEST` cases | | thread plan wiring | a region that drops the GIL and resolves no thread plan, an unbalanced plan macro, or a path that escapes one | +| thread counts | a parallel region naming no thread count -- so it runs at one thread per core whatever the plan resolved -- or naming one outside the permitted forms | | numeric | 78 quantities, order 2/3/4 x default/uneven prior, as hex floats | | saved-model bytes | the sha256 of a written model | | thread independence | results at 1, 4 and 16 threads | -| exact | 351 quantities against brute-force enumeration of the whole support | +| exact | 416 quantities against brute-force enumeration of the whole support | Every row above `exact` pins REPRODUCIBILITY: each quantity is compared with a recording of ITSELF, so a quantity that has been wrong since the day it was diff --git a/verification/tests/test_ci_runs_the_checks.py b/verification/tests/test_ci_runs_the_checks.py new file mode 100644 index 0000000..460b515 --- /dev/null +++ b/verification/tests/test_ci_runs_the_checks.py @@ -0,0 +1,132 @@ +"""The checks this repository owns are of no use unless CI runs them. + +Two of them were not run by any workflow, and both had the same shape as the +verification gate before `845a4f2`: a check nobody runs does not report a +regression late, it reports nothing. + + * `verification/tests` -- the suite guarding the gate itself, including two + hardcoded tripwires (the `ALL` set in `test_record_selector`, the Makefile + target spelling in `test_wrap_link_guard`) that only pay off if something + runs them. + * More than one thread environment. Two real defects in the parallelism work + appeared ONLY under a non-default `OMP_NUM_THREADS`: an assertion whose two + outcomes collided on the same number, and a policy clause reading + `omp_get_num_procs()` where it needed `omp_get_max_threads()`. The C runner + pins `OMP_NUM_THREADS=2` for every suite, so a single run of it cannot see + that class at all. + +WHAT THESE TESTS PROVE, AND WHAT THEY DO NOT. They read the workflow text and +prove a command is PRESENT in it, outside a comment. They do not prove the job +is reached, that its `if:` lets it run, or that it passed -- the same +limitation the thread-plan wiring check turned out to have, and worth stating +here rather than discovering later. What they are is a tripwire: delete the +job and a test fails, which is the property that was missing. + +Deliberately text-based. Parsing the YAML properly would mean adding pyyaml to +the dev dependencies for one tripwire, and these assertions are about commands +appearing in a file, which does not need a parser. +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows" + +# The C runner's own default, which every CI run of the C suite has used. A +# second environment has to differ from THIS, not merely exist. +RUNNER_DEFAULT_THREADS = 2 + + +def workflow_lines() -> list[tuple[str, str]]: + """Every (file, line) in the workflows, with comment-only lines dropped. + + A `#` inside a quoted string is not a comment, but no workflow here has + one, and the alternative is a YAML parser. Lines whose first non-space + character is `#` are the ones that matter: commenting a job out is how a + check stops running without anyone deleting it. + """ + out = [] + for path in sorted(WORKFLOWS.glob("*.yml")): + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("#"): + continue + out.append((path.name, line)) + return out + + +def test_some_workflow_runs_the_tooling_suite(): + """verification/tests runs in CI, not only by hand.""" + hits = [(f, l) for (f, l) in workflow_lines() + if "verification/tests" in l and "pytest" in l] + assert hits, ( + "No workflow runs `pytest verification/tests`. These are the tests " + "that guard the gate; running them by hand means they guard it only " + "on the days someone remembers." + ) + + +def test_the_c_suite_runs_at_more_than_one_thread_count(): + """Some workflow runs the C suite at a thread count other than the default.""" + counts = set() + for (_f, line) in workflow_lines(): + for m in re.finditer(r"--threads[= ]+(\d+)", line): + counts.add(int(m.group(1))) + extra = counts - {RUNNER_DEFAULT_THREADS} + assert extra, ( + f"Every CI run of the C suite uses the runner default of " + f"{RUNNER_DEFAULT_THREADS} threads (found: {sorted(counts) or 'none'}). " + "Two real defects in this work were visible only at a different " + "thread count, so one environment cannot find that class." + ) + + +def test_a_thread_count_of_one_is_among_them(): + """One thread is the environment that catches 'assumes a team'.""" + counts = {int(m.group(1)) + for (_f, line) in workflow_lines() + for m in re.finditer(r"--threads[= ]+(\d+)", line)} + assert 1 in counts, ( + f"No workflow runs the C suite single-threaded (found: {sorted(counts)}). " + "A team of one is the case where a reduction, a barrier or a " + "per-thread buffer can be skipped entirely." + ) + + +def test_an_odd_thread_count_is_among_them(): + """An odd count catches 'assumes the work divides evenly'.""" + counts = {int(m.group(1)) + for (_f, line) in workflow_lines() + for m in re.finditer(r"--threads[= ]+(\d+)", line)} + odd = {c for c in counts if c > 1 and c % 2 == 1} + assert odd, ( + f"No workflow runs the C suite at an odd thread count above one " + f"(found: {sorted(counts)}). Powers of two hide off-by-one chunking: " + "every count CI has ever used divides the chunk counts evenly." + ) + + +def test_the_python_suite_sees_a_pinned_thread_environment(): + """Some workflow sets OMP_NUM_THREADS for Python tests.""" + hits = [(f, l) for (f, l) in workflow_lines() if "OMP_NUM_THREADS" in l] + assert hits, ( + "No workflow sets OMP_NUM_THREADS. The Python legs therefore all run " + "at whatever the runner's core count implies, which is one " + "environment chosen by the hardware rather than by this repository." + ) + + +@pytest.mark.parametrize("command", [ + "verification/check.py", # the gate + "make -j", # the C suite +]) +def test_the_checks_that_already_ran_are_still_wired(command): + """A regression guard for the jobs that were already there. + + Without this, a later edit could satisfy the tests above while quietly + dropping the checks CI did run. + """ + assert any(command in l for (_f, l) in workflow_lines()), \ + f"No workflow runs `{command}` any more."