From 52567a3e5bd1b0ee225f5bd5f52ba3fbd6aaf0a8 Mon Sep 17 00:00:00 2001 From: YohannMrg <98689112+yohannmarguier@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:31:35 +0200 Subject: [PATCH 1/5] Ignore IDE cmake-build-* directories A stray CLion cmake-build-debug/ tree (with a full vendored googletest checkout) was sitting untracked inside tests/contract/, polluting greps and the graphify knowledge graph. Only /build*/ at the repo root was ignored; IDE-style per-directory build trees were not. (cherry picked from commit 624d7e3309d2f9da2f2f31e9bf9819f1d17ddfbe) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f5697b3c..7fd179b9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ CMakeUserPresets.json /vcpkg_installed/ settings.json /build*/ +cmake-build-*/ /_deps/ /CMakeFiles CMakeCache.txt From bd72b32db2c795f07b15a4a713a9808d2c53ef25 Mon Sep 17 00:00:00 2001 From: YohannMrg <98689112+yohannmarguier@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:31:48 +0200 Subject: [PATCH 2/5] Add CMake presets as the canonical test entry points; drive CI with them CMakePresets.json defines the configure/build/test presets for every test leg: tests / tests-debug (the blocking functional matrix, issue #32), sanitize (the D7 ASan/UBSan leg, env and Death|CurrentlyCorrupts exclusions moved into the test preset), and mdsplus (backend + baked model tree, MDSPLUS_MODELS_PATH pointed at the bake). Test presets for the unit/integration tiers give the fast hermetic loop a name, and every label-filtered preset sets noTestsAction=error so an empty selection fails instead of passing silently (same concern as issue #31). contract-tests.yml, sanitizers.yml and mdsplus-contract.yml now call the presets instead of hand-assembled flags, so a local 'ctest --preset ...' reproduces a CI leg exactly. Only the DD version pin stays in mdsplus-contract.yml, next to the cache key it feeds. The UDA leg is untouched on purpose: its configure lives in docker/uda/run.sh, part of the pinned reference stack whose image cache is keyed on that file. Verified locally: unit (96), integration (163) and tests (263) presets all pass. (cherry picked from commit aade3e72cc4460936730f686b163a68e68f4f90d) --- .github/workflows/contract-tests.yml | 25 +++--- .github/workflows/mdsplus-contract.yml | 17 ++-- .github/workflows/sanitizers.yml | 28 +++--- CMakePresets.json | 118 +++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 39 deletions(-) create mode 100644 CMakePresets.json diff --git a/.github/workflows/contract-tests.yml b/.github/workflows/contract-tests.yml index 1787dfdd..64ddd3b1 100644 --- a/.github/workflows/contract-tests.yml +++ b/.github/workflows/contract-tests.yml @@ -15,7 +15,7 @@ on: jobs: contract-functional: - name: Contract tests (Linux, ${{ matrix.build_type }}, blocking) + name: Contract tests (Linux, ${{ matrix.preset }}, blocking) runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,8 +23,14 @@ jobs: # Both build modes (issue #32): the dlopen-failure plugin pin is # build-mode dependent — assert-abort death test in Debug, swallowed # assert tripwire under NDEBUG (RelWithDebInfo) — so each branch needs - # its own leg or one of them never runs anywhere. - build_type: [RelWithDebInfo, Debug] + # its own leg or one of them never runs anywhere. The presets + # (CMakePresets.json) are the single source of truth for flags and + # binary dirs; builddir here only feeds the label-partition check. + include: + - preset: tests # RelWithDebInfo + builddir: build + - preset: tests-debug + builddir: build-debug steps: - name: Checkout code @@ -38,20 +44,15 @@ jobs: sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev - name: Configure (tests on, no Python bindings) - run: > - cmake -S . -B build -G Ninja - -DAL_BUILD_TESTS=ON - -DAL_PYTHON_BINDINGS=OFF - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + run: cmake --preset ${{ matrix.preset }} - name: Build - run: cmake --build build + run: cmake --build --preset ${{ matrix.preset }} # Issue #31: an empty `ctest -L unit` exits 0, so a broken label # partition is invisible to the test run below — check it explicitly. - name: Verify unit/integration label partition - run: bash tests/contract/check_label_partition.sh build/tests/contract + run: bash tests/contract/check_label_partition.sh ${{ matrix.builddir }}/tests/contract - name: Run the whole test suite - working-directory: build - run: ctest --output-on-failure + run: ctest --preset ${{ matrix.preset }} diff --git a/.github/workflows/mdsplus-contract.yml b/.github/workflows/mdsplus-contract.yml index 8222b82c..ca035693 100644 --- a/.github/workflows/mdsplus-contract.yml +++ b/.github/workflows/mdsplus-contract.yml @@ -68,17 +68,15 @@ jobs: key: mdsplus-model-${{ env.DD_VERSION }}-${{ hashFiles('common/cmake/ALBuildDataDictionary.cmake', 'models/mdsplus/CMakeLists.txt') }} - name: Configure (bakes the DD model tree on a cache miss) + # The `mdsplus` preset (CMakePresets.json) carries the backend/model/ + # test flags and the build-mdsplus binary dir; only the DD pin stays + # here, next to the cache key it feeds. if: steps.cache-model.outputs.cache-hit != 'true' run: > docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace "${IMAGE_TAG}" bash -c " - cmake -B build-mdsplus -G Ninja - -DAL_BACKEND_MDSPLUS=ON - -DAL_BUILD_MDSPLUS_MODELS=ON - -DAL_BUILD_TESTS=ON - -DDD_VERSION=${DD_VERSION} - -DCMAKE_BUILD_TYPE=Release + cmake --preset mdsplus -DDD_VERSION=${DD_VERSION} " - name: Build contract suite @@ -86,7 +84,7 @@ jobs: docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace "${IMAGE_TAG}" bash -c " - cmake --build build-mdsplus --target contract_tests -j\$(nproc) + cmake --build --preset mdsplus " - name: Verify baked DD model tree is present (cache-hit and cache-miss paths) @@ -108,12 +106,13 @@ jobs: echo "Baked DD model tree complete under build-mdsplus/models/mdsplus." - name: Run MDSplus contract suite + # The `mdsplus` test preset selects the tier (-L mdsplus) and points + # MDSPLUS_MODELS_PATH at the baked model tree under build-mdsplus/. run: > docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace - -e MDSPLUS_MODELS_PATH=/workspace/build-mdsplus/models/mdsplus "${IMAGE_TAG}" bash -c " - cd build-mdsplus/tests/contract && ctest -L mdsplus --output-on-failure + ctest --preset mdsplus " - name: Note diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 41a16e2d..0d7b7d5c 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -32,23 +32,19 @@ jobs: sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev - name: Configure (sanitized, tests on, no Python bindings) - # AL_CONTRACT_SANITIZE instruments both libal and the test binary. Python - # bindings are off: a sanitized libal must not be linked into a wheel - # whose bindings lack the ASan runtime (see tests/contract/CMakeLists.txt). - run: > - cmake -S . -B build-san -G Ninja - -DAL_BUILD_TESTS=ON - -DAL_CONTRACT_SANITIZE=ON - -DAL_PYTHON_BINDINGS=OFF - -DCMAKE_BUILD_TYPE=RelWithDebInfo + # The `sanitize` preset (CMakePresets.json) sets AL_CONTRACT_SANITIZE, + # which instruments both libal and the test binary. Python bindings are + # off: a sanitized libal must not be linked into a wheel whose bindings + # lack the ASan runtime (see tests/contract/CMakeLists.txt). + run: cmake --preset sanitize - name: Build contract suite - run: cmake --build build-san --target contract_tests + run: cmake --build --preset sanitize - name: Run contract suite under sanitizers - working-directory: build-san - # Scope to the contract suite; halt/abort_on_error=0 so the whole suite - # runs and reports everything at once (informational leg). Two exclusions: + # The `sanitize` test preset carries the ASAN/LSAN/UBSAN_OPTIONS + # (halt/abort_on_error=0 so the whole suite runs and reports everything + # at once — informational leg) and two exclusions: # * `Death` tests deliberately crash child processes (asserting today's # crash-class defects) — not a memory-safety target, and ASan would # report the intentional crash as an error. @@ -58,11 +54,7 @@ jobs: # fail here. They guard against rot in the blocking functional leg # (contract-tests.yml). Real leaks/UAF in the ordinary paths still # surface. - env: - ASAN_OPTIONS: detect_leaks=1:halt_on_error=0:abort_on_error=0 - LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0 - run: ctest -L contract -E "Death|CurrentlyCorrupts" --output-on-failure + run: ctest --preset sanitize - name: Note if: always() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..96174b68 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,118 @@ +{ + "version": 3, + "cmakeMinimumRequired": { "major": 3, "minor": 21, "patch": 0 }, + "configurePresets": [ + { + "name": "tests", + "displayName": "Test build (RelWithDebInfo)", + "description": "Smoke + contract suites against the public C ABI; no Python bindings. Mirrors the blocking CI leg (contract-tests.yml).", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "AL_BUILD_TESTS": "ON", + "AL_PYTHON_BINDINGS": "OFF", + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + } + }, + { + "name": "tests-debug", + "displayName": "Test build (Debug)", + "description": "Debug leg of contract-tests.yml: the dlopen-failure plugin pin is build-mode dependent (issue #32), so CI runs both modes.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-debug", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } + }, + { + "name": "sanitize", + "displayName": "Sanitized test build (ASan/UBSan)", + "description": "AL_CONTRACT_SANITIZE=ON instruments libal and contract_tests (TEST_STRATEGY.md D7). Mirrors sanitizers.yml. Never combine with a wheel build: the bindings would link a sanitized libal without the ASan runtime.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-san", + "cacheVariables": { "AL_CONTRACT_SANITIZE": "ON" } + }, + { + "name": "mdsplus", + "displayName": "MDSplus test build (bakes the DD model tree)", + "description": "MDSplus backend + baked model tree + tests. Pass -DDD_VERSION= to pin the DD (models/mdsplus defaults to 'main'; mdsplus-contract.yml pins its own). Downloads the DD at configure time.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-mdsplus", + "cacheVariables": { + "AL_BACKEND_MDSPLUS": "ON", + "AL_BUILD_MDSPLUS_MODELS": "ON", + "CMAKE_BUILD_TYPE": "Release" + } + } + ], + "buildPresets": [ + { "name": "tests", "configurePreset": "tests" }, + { "name": "tests-debug", "configurePreset": "tests-debug" }, + { + "name": "sanitize", + "configurePreset": "sanitize", + "targets": [ "contract_tests" ] + }, + { + "name": "mdsplus", + "configurePreset": "mdsplus", + "targets": [ "contract_tests" ] + } + ], + "testPresets": [ + { + "name": "unit", + "displayName": "Hermetic inner loop: pure lookups, Memory backend, plugin registry", + "configurePreset": "tests", + "filter": { "include": { "label": "^unit$" } }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "integration", + "displayName": "On-disk tier: HDF5 / ASCII / Flexbuffers backends", + "configurePreset": "tests", + "filter": { "include": { "label": "^integration$" } }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "tests", + "displayName": "Everything registered in the test build (contract + smoke)", + "configurePreset": "tests", + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "tests-debug", + "displayName": "Everything, Debug build (issue #32 leg)", + "configurePreset": "tests-debug", + "inherits": "tests" + }, + { + "name": "sanitize", + "displayName": "Contract suite under ASan/UBSan (informational CI leg)", + "configurePreset": "sanitize", + "filter": { + "include": { "label": "^contract$" }, + "exclude": { "name": "Death|CurrentlyCorrupts" } + }, + "environment": { + "ASAN_OPTIONS": "detect_leaks=1:halt_on_error=0:abort_on_error=0", + "LSAN_OPTIONS": "suppressions=${sourceDir}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1", + "UBSAN_OPTIONS": "print_stacktrace=1:halt_on_error=0" + }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "mdsplus", + "displayName": "MDSplus tier against the baked model tree", + "configurePreset": "mdsplus", + "filter": { "include": { "label": "^mdsplus$" } }, + "environment": { + "MDSPLUS_MODELS_PATH": "${sourceDir}/build-mdsplus/models/mdsplus" + }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + } + ] +} From f4b6cd719f6d9153e7daf1d36f337f663c49ed8e Mon Sep 17 00:00:00 2001 From: YohannMrg <98689112+yohannmarguier@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:31:59 +0200 Subject: [PATCH 3/5] Document how to run the test suites (tests/README.md + routing stubs) tests/README.md is the operational front door: what lives where (smoke / contract / Python suites), the CTest tier table with per-tier requirements, the preset commands that reproduce each CI leg, how to read DISABLED_+tripwire (xfail) and GTEST_SKIP (divergence / unconfigured backend) output, and the fixture tools. It routes to TEST_STRATEGY.md for rationale and TRACEABILITY.md for the coverage ledger rather than repeating them. README.md and CONTRIBUTING.md gain three-line 'running the tests' stubs that link to it. (cherry picked from commit 10bda17c304d5e3ebe7171102102ece794bbfb16) --- CONTRIBUTING.md | 6 +++- README.md | 11 +++++++ tests/README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 tests/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69e18b1e..c7d8ceb3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,11 @@ profile and create your own feature branch off of the latest develop commit. Make sure to regularly sync your branch with the latest commits from `develop` (find instructions [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)); -5. when your development is ready, create a pull request (find instructions +5. make sure the test suite passes before submitting: see +[tests/README.md](tests/README.md) for how to build and run it +(`cmake --preset tests && cmake --build --preset tests && ctest --preset tests` +is the same run as the blocking CI check); +6. when your development is ready, create a pull request (find instructions [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)). diff --git a/README.md b/README.md index 70a0480b..673700e7 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,17 @@ cmake --build build --target install See [Developer Guide](docs/source/developers/index.rst) for build instructions. +### Running the tests + +The C/C++ and Python test suites, their CTest tiers, and the CMake presets +that drive them (locally and in CI) are documented in +[tests/README.md](tests/README.md). The short version: + +```bash +cmake --preset tests && cmake --build --preset tests +ctest --preset unit # fast hermetic loop; `ctest --preset tests` for everything +``` + ## Using IMAS-Core with High-Level Languages diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..fb4b8431 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,79 @@ +# Running the tests + +Everything here is driven through the presets in [`CMakePresets.json`](../CMakePresets.json) +— the same entry points CI uses, so a local run reproduces a CI leg exactly. + +```sh +cmake --preset tests # configure (AL_BUILD_TESTS=ON, no Python bindings) +cmake --build --preset tests # build libal + test binaries +ctest --preset unit # fast hermetic inner loop (< 1 s) +ctest --preset tests # everything the test build registers +``` + +## What lives where + +| Suite | Location | Framework | Purpose | +|---|---|---|---| +| Smoke tests | `tests/testlowlevel.cpp`, `tests/testlowlevel_c.c` | none (plain executables) | End-to-end C ABI exercise; the `.c` build doubles as a compile canary that `al_lowlevel.h` stays C-compatible. | +| Contract suite | `tests/contract/` | GoogleTest + CTest | Pins the behavior of the public C ABI (`al_lowlevel.h` + `al_const.h`) capability by capability. The *why* is [`TEST_STRATEGY.md`](../TEST_STRATEGY.md); the coverage ledger is [`tests/contract/TRACEABILITY.md`](contract/TRACEABILITY.md). | +| Python binding tests | `python/tests/` | pytest | Bindings-level checks (import, exceptions, `imasdef` constants, HDF5 permissions). | + +## Test tiers (CTest labels) + +Every contract test carries exactly one tier, assigned post-discovery by +`tests/contract/assign_contract_labels.cmake.in` (the tier definitions live +there) and verified disjoint by `tests/contract/check_label_partition.sh`. + +| `ctest --preset …` | Label | What runs | Needs | +|---|---|---|---| +| `unit` | `unit` | Pure lookups (`Introspection.*`), the in-RAM Memory backend matrix, plugin registry lifecycle + ABI death tests, URI ownership | Nothing — hermetic, no disk | +| `integration` | `integration` | HDF5 / ASCII / Flexbuffers round trips on disk | Disk only | +| `tests` / `tests-debug` | *(all)* | Contract suite + smoke tests, in the build mode CI's blocking legs use (both modes matter — issue #32) | — | +| `sanitize` | `contract` | The suite under ASan/UBSan (`AL_CONTRACT_SANITIZE=ON`), excluding `Death` and `CurrentlyCorrupts` cases — see the comments in `sanitizers.yml` for why | Linux for leak detection; on macOS prepend `ASAN_OPTIONS=detect_leaks=0` | +| `mdsplus` | `mdsplus` | MDSplus tier against the DD model tree the `mdsplus` configure preset bakes | MDSplus installed; configure downloads the DD (`-DDD_VERSION=` to pin) | +| *(no preset)* | `uda` | UDA tier against the pinned reference stack | `docker/uda/run.sh` builds and runs it inside the container image (see `docker/uda/README.md`); there is deliberately no host preset | + +Each test preset (except the run-everything ones' smoke tests) fails on an +empty selection (`noTestsAction: error`), so a broken label partition can't +pass silently. + +For a one-off finer cut, plain CTest still works from the build directory: +`ctest -R Introspection`, `ctest -L integration -E Ascii`, etc. + +## Reading the output + +The suite encodes its own status vocabulary (full definitions in the +[TRACEABILITY.md preamble](contract/TRACEABILITY.md)): + +- **`DISABLED_…` test + a paired `CurrentlyCorrupts`/tripwire test** — an + *xfail*: the disabled test asserts the *correct* behavior of a known defect, + the tripwire pins today's wrong behavior. The tripwire going red means + someone fixed the defect — flip the pair, don't "fix the test". +- **`GTEST_SKIP()`** — either a legitimate backend *divergence* (the test + shape doesn't transfer to that backend) or an unconfigured optional backend + (e.g. `MDSPLUS_MODELS_PATH` / `UDA_HOST` unset). The skip message says which. + +## CI legs + +| Workflow | Preset(s) | Gate | +|---|---|---| +| `contract-tests.yml` | `tests`, `tests-debug` | **Blocking** on every PR | +| `sanitizers.yml` | `sanitize` | Informational (D7) | +| `mdsplus-contract.yml` | `mdsplus` (in Docker, DD pinned in the workflow) | Informational | +| `uda-contract.yml` | — (`docker/uda/run.sh`) | Informational | + +## Python tests + +```sh +pip install .[test] +pytest # testpaths is rooted at python/tests in pyproject.toml +``` + +## Fixture tools + +Two build-gated helper executables prepare states that must *not* be reachable +through the public C ABI (out-of-band file/tree surgery); the tests invoke +them as subprocesses: `hdf5_fixture_tool.cpp` (rewrites the stored +HDF5 backend version) and `mdsplus_fixture_tool.cpp` (rewrites the model +tree's version node). They are fixture *preparation*, never assertion — the +contract suite asserts only through the C ABI. From 8d435ae24bed04bdda5fef127cc30cd5999a9aa4 Mon Sep 17 00:00:00 2001 From: YohannMrg <98689112+yohannmarguier@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:53:30 +0200 Subject: [PATCH 4/5] Make ./test delegate to the CMake presets instead of duplicating flags The root ./test dispatcher (the one-verb wrapper for the Docker-backed MDSplus/UDA tiers) predates CMakePresets.json and hand-assembled its own configure lines, which had already drifted from CI: 'common' built Release without AL_PYTHON_BINDINGS=OFF while the blocking leg runs RelWithDebInfo. Its host and in-container recipes now call the same tests/mdsplus presets CI uses, so the script only adds orchestration (image builds, bind mounts, the summary table) and there is exactly one place flags live. Only the DD pin stays inline, matching mdsplus-contract.yml. The UDA verb is untouched (uda-run.sh owns its configure by design). tests/README.md now documents ./test in the quick start; .gitignore gains .pytest_cache/. Verified: ./test common passes end-to-end (263 tests). (cherry picked from commit b10520653495efed3ab78fd4a156b18f97fa1723) --- .gitignore | 1 + test | 32 ++++++++++++++++++-------------- tests/README.md | 13 ++++++++++++- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 7fd179b9..f67d841c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ python/lib/ /.venv/ /vcpkg/ /UDA-*/ +.pytest_cache/ diff --git a/test b/test index 39e43a64..6c5c6444 100755 --- a/test +++ b/test @@ -20,6 +20,11 @@ # Boost::filesystem installed). AL_BACKEND_MDSPLUS/UDA default OFF, so their # contract tiers compile out and this ctest runs only the common-backend tests. # +# Flags live in CMakePresets.json, not here: every tier delegates to the same +# `tests` / `mdsplus` presets CI runs, so this script only adds orchestration +# (Docker images, bind mounts, the summary table) on top — never a second copy +# of the configure line. +# # The mdsplus/uda backing images are built automatically on first use and reused # after that (rebuild by hand with `docker build`, or `docker image rm` to # force it). Both legs bind-mount the repo at /workspace, so they always @@ -102,14 +107,13 @@ print_summary() { run_common() { section "COMMON" "always-on backends (HDF5, memory, ASCII, flexbuffers) — host-native" - # No -G so an existing build/ keeps its generator; a fresh one takes CMake's - # default. getconf is portable across macOS and Linux. The && chain returns - # non-zero on the first failing step (configure/build/ctest). - local jobs; jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4) - cmake -S "$REPO_ROOT" -B "$REPO_ROOT/build" \ - -DAL_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release \ - && cmake --build "$REPO_ROOT/build" -j"$jobs" \ - && ctest --test-dir "$REPO_ROOT/build" --output-on-failure + # The `tests` presets are the same entry points as CI's blocking leg + # (contract-tests.yml): RelWithDebInfo, Ninja, no Python bindings, build/. + # The && chain returns non-zero on the first failing step. + cd "$REPO_ROOT" \ + && cmake --preset tests \ + && cmake --build --preset tests \ + && ctest --preset tests } @@ -118,15 +122,15 @@ run_common() { run_mdsplus() { section "MDS-PLUS" "Docker: bake DD-4.1.1 model tree, build, ctest -L mdsplus" ensure_image imas-core-mdsplus:dev "$REPO_ROOT/docker/mdsplus" + # The `mdsplus` presets carry the backend/model/test flags, the + # build-mdsplus binary dir, the tier filter and MDSPLUS_MODELS_PATH; only + # the DD pin is passed here (mdsplus-contract.yml pins its own the same way). docker run --rm --platform "$PLATFORM" \ -v "$REPO_ROOT":/workspace -w /workspace \ - -e MDSPLUS_MODELS_PATH=/workspace/build-mdsplus/models/mdsplus \ imas-core-mdsplus:dev bash -c ' - cmake -B build-mdsplus -G Ninja \ - -DAL_BACKEND_MDSPLUS=ON -DAL_BUILD_MDSPLUS_MODELS=ON \ - -DAL_BUILD_TESTS=ON -DDD_VERSION=4.1.1 -DCMAKE_BUILD_TYPE=Release && - cmake --build build-mdsplus -j"$(nproc)" && - cd build-mdsplus/tests/contract && ctest -L mdsplus --output-on-failure + cmake --preset mdsplus -DDD_VERSION=4.1.1 && + cmake --build --preset mdsplus && + ctest --preset mdsplus ' } diff --git a/tests/README.md b/tests/README.md index fb4b8431..b7837e6b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,6 +10,17 @@ ctest --preset unit # fast hermetic inner loop (< 1 s) ctest --preset tests # everything the test build registers ``` +For the tiers that need a Docker backend (MDSplus, UDA), the [`./test`](../test) +dispatcher at the repo root wraps the image build + `docker run` recipes behind +one verb and delegates all cmake/ctest flags to these same presets: + +```sh +./test common # host-native: the three preset commands above +./test mdsplus # Docker: bake the DD model tree, run the mdsplus tier +./test uda # Docker: assemble the UDA reference stack, run the uda tier +./test all # every tier + a PASS/FAIL summary table +``` + ## What lives where | Suite | Location | Framework | Purpose | @@ -31,7 +42,7 @@ there) and verified disjoint by `tests/contract/check_label_partition.sh`. | `tests` / `tests-debug` | *(all)* | Contract suite + smoke tests, in the build mode CI's blocking legs use (both modes matter — issue #32) | — | | `sanitize` | `contract` | The suite under ASan/UBSan (`AL_CONTRACT_SANITIZE=ON`), excluding `Death` and `CurrentlyCorrupts` cases — see the comments in `sanitizers.yml` for why | Linux for leak detection; on macOS prepend `ASAN_OPTIONS=detect_leaks=0` | | `mdsplus` | `mdsplus` | MDSplus tier against the DD model tree the `mdsplus` configure preset bakes | MDSplus installed; configure downloads the DD (`-DDD_VERSION=` to pin) | -| *(no preset)* | `uda` | UDA tier against the pinned reference stack | `docker/uda/run.sh` builds and runs it inside the container image (see `docker/uda/README.md`); there is deliberately no host preset | +| *(no preset)* | `uda` | UDA tier against the pinned reference stack | `./test uda` (or `docker/uda/run.sh` inside the container — see `docker/uda/README.md`); there is deliberately no host preset | Each test preset (except the run-everything ones' smoke tests) fails on an empty selection (`noTestsAction: error`), so a broken label partition can't From 9f0ebc93fb001cfdd165e400a30326574564f8cd Mon Sep 17 00:00:00 2001 From: YohannMrg <98689112+yohannmarguier@users.noreply.github.com> Date: Sat, 18 Jul 2026 16:12:59 +0200 Subject: [PATCH 5/5] Make contract fixtures linkable outside the test tree (issue #51) Extract equilibrium_seed.h and the GTest-free subset of al_contract.h's generic typed round-trip helpers it needs (DType, write_data, read_data, element_count) into a new top-level fixtures/ directory, wired as an INTERFACE library (al_contract_fixtures) added unconditionally in the root CMakeLists.txt, independent of AL_BUILD_TESTS. This lets a future benchmarks/ target (PRD #50) link the exact same workload the contract suite pins, with no fixture drift and no duplication. tests/contract/al_contract.h now includes the shared header instead of redefining those symbols; contract_tests links the new target so unqualified #include "equilibrium_seed.h" in the existing test files keeps resolving. An opt-in throwaway executable (AL_BUILD_FIXTURE_SMOKE, off by default) proves the fixture builds and round-trips against the Memory backend from a target outside tests/, with no GoogleTest link. (cherry picked from commit 690f5392a58e4c73131d6b723c72105e9fbdcc9f) --- CMakeLists.txt | 12 ++ fixtures/CMakeLists.txt | 44 ++++++++ fixtures/al_contract_abi.h | 106 ++++++++++++++++++ .../contract => fixtures}/equilibrium_seed.h | 2 +- fixtures/fixture_smoke.cpp | 77 +++++++++++++ tests/contract/CMakeLists.txt | 2 +- tests/contract/TRACEABILITY.md | 4 +- tests/contract/al_contract.h | 89 ++------------- tests/contract/test_uda.cpp | 2 +- 9 files changed, 253 insertions(+), 85 deletions(-) create mode 100644 fixtures/CMakeLists.txt create mode 100644 fixtures/al_contract_abi.h rename {tests/contract => fixtures}/equilibrium_seed.h (99%) create mode 100644 fixtures/fixture_smoke.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b2d72f3..dcbb5780 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,13 @@ option(AL_PYTHON_BINDINGS "Build Python bindings" OFF) option(AL_BUILD_TESTS "Build the C/C++ low-level ABI smoke tests (tests/)" OFF) +# The fixtures/al_contract_fixtures INTERFACE library itself is always +# available (see below) so it is genuinely usable outside AL_BUILD_TESTS; this +# only gates the throwaway demonstration executable (issue #51 acceptance +# criterion 3), off by default like every other extra target in this file. +option(AL_BUILD_FIXTURE_SMOKE + "Build a throwaway executable proving tests/contract fixtures are linkable outside the test tree" OFF) + # When ON, the al C++ library is not built here; instead it is located with # find_package(al-core CONFIG). Enables a two-stage build where libimas-core # (the C/C++ library) and imas-core (the Python wrapper) are packaged @@ -213,6 +220,11 @@ endif() add_subdirectory(src) +# Shared contract fixtures (issue #51): the equilibrium seed and the ABI +# helpers it needs, usable outside tests/ (e.g. by a future benchmarks/ +# tree). Added unconditionally so it never depends on AL_BUILD_TESTS. +add_subdirectory(fixtures) + # Dummy Executable # ############################################################################## set( EXE_FLAG ) diff --git a/fixtures/CMakeLists.txt b/fixtures/CMakeLists.txt new file mode 100644 index 00000000..c674ed3d --- /dev/null +++ b/fixtures/CMakeLists.txt @@ -0,0 +1,44 @@ +######################################################################################################################## +# Shared contract fixtures (issue #51). +# +# The realistic equilibrium seed (equilibrium_seed.h) and the generic typed +# ABI round-trip helpers it needs (al_contract_abi.h), factored out of +# tests/contract/ so a build target outside the test tree -- the future +# benchmarks/ tree (PRD #50, issues #52/#53) -- can link the exact same +# workload the contract suite pins. No fixture drift between what is tested +# and what is measured (spec #50, Option C). +# +# Header-only and carries no GoogleTest dependency (unlike +# tests/contract/al_contract.h), so a non-test binary can link it for free. +# Added unconditionally (not gated by AL_BUILD_TESTS) so it is genuinely +# usable independent of the test build. + +add_library(al_contract_fixtures INTERFACE) +target_include_directories(al_contract_fixtures INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(al_contract_fixtures INTERFACE al) +target_compile_features(al_contract_fixtures INTERFACE cxx_std_17) + +######################################################################################################################## +# Throwaway proof (issue #51 acceptance criterion 3), opt-in via +# AL_BUILD_FIXTURE_SMOKE -- off by default like every other extra target in +# the root CMakeLists.txt (AL_BUILD_TESTS, AL_BUILD_BENCHMARKS), so an +# ordinary/minimal build (e.g. the Python wheel) never pays for it. The +# al_contract_fixtures library above stays unconditional regardless: that is +# the actual "usable outside the test tree" requirement, this executable is +# just a demonstration of it. +# +# A minimal translation unit outside tests/ that includes the fixture, builds, +# and round-trips it against a real backend -- demonstrating the fixture is +# usable without AL_BUILD_TESTS or GoogleTest. Uses the always-on, zero-I/O +# Memory backend so this needs no on-disk setup and no optional backend. +if(AL_BUILD_FIXTURE_SMOKE) + add_executable(al_contract_fixtures_smoke fixture_smoke.cpp) + target_link_libraries(al_contract_fixtures_smoke PRIVATE al_contract_fixtures) + add_dependencies(al_contract_fixtures_smoke al) + target_compile_features(al_contract_fixtures_smoke PRIVATE cxx_std_17) + + if(NOT WIN32 AND NOT APPLE) + # stdc++fs is only needed on Linux with older GCC (std::filesystem in the smoke TU). + target_link_libraries(al_contract_fixtures_smoke PRIVATE stdc++fs) + endif() +endif() diff --git a/fixtures/al_contract_abi.h b/fixtures/al_contract_abi.h new file mode 100644 index 00000000..8f4d8933 --- /dev/null +++ b/fixtures/al_contract_abi.h @@ -0,0 +1,106 @@ +// Generic typed round-trip helpers over al_write_data / al_read_data +// (issue #51). Factored out of tests/contract/al_contract.h so a build target +// outside tests/contract/ -- notably a future benchmarks/ tree (PRD #50, issue +// #52) -- can link the exact same write_data/read_data machinery +// equilibrium_seed.h uses, without also pulling in GoogleTest. tests/contract/ +// al_contract.h includes this header for its own copy of these helpers, so +// there is exactly one definition either way. +// +// Binds only to the `extern "C"` surface (al_lowlevel.h + al_const.h), per +// TEST_STRATEGY.md decision D1: no C++ internals, no Python bindings. + +#ifndef AL_CONTRACT_ABI_H +#define AL_CONTRACT_ABI_H + +#include +#include + +#include +#include +#include +#include + +namespace al_contract { + +// --- datatype descriptor: element C++ type -> ABI datatype id + label ------- +template +struct DType; +template <> +struct DType { + static constexpr int value = CHAR_DATA; + static constexpr const char* name = "CHAR"; +}; +template <> +struct DType { + static constexpr int value = INTEGER_DATA; + static constexpr const char* name = "INTEGER"; +}; +template <> +struct DType { + static constexpr int value = DOUBLE_DATA; + static constexpr const char* name = "DOUBLE"; +}; +template <> +struct DType> { + static constexpr int value = COMPLEX_DATA; + static constexpr const char* name = "COMPLEX"; +}; + +inline std::size_t element_count(const std::vector& shape) { + std::size_t n = 1; + for (int d : shape) n *= static_cast(d); + return n; +} + +// --- generic write ---------------------------------------------------------- +// al_write_data takes a non-const void* and int* size but does not mutate +// them for a write; the const_casts keep the caller's data/shape const. +template +al_status_t write_data(int ctx, const char* field, const std::vector& shape, + const std::vector& data) { + const int dim = static_cast(shape.size()); + int* size = dim ? const_cast(shape.data()) : nullptr; + return al_write_data(ctx, field, "", const_cast(data.data()), + DType::value, dim, size); +} + +// --- generic read ----------------------------------------------------------- +// Honors the two ownership regimes the ABI uses (mirrors testlowlevel.cpp): +// dim == 0 : the core writes into the caller's buffer; nothing to free. +// dim >= 1 : the core malloc's the buffer and returns it via *data; the +// caller owns it, so we copy out and free. +// On success out_shape is the shape the core reported (scalar -> empty) and +// out_data holds element_count(out_shape) elements. +template +al_status_t read_data(int ctx, const char* field, int expected_rank, + std::vector* out_shape, std::vector* out_data) { + int size[MAXDIM] = {0}; + + if (expected_rank == 0) { + T scalar{}; + void* buf = &scalar; + al_status_t s = + al_read_data(ctx, field, "", &buf, DType::value, 0, size); + if (s.code == 0) { + out_shape->clear(); + out_data->assign(1, scalar); + } + return s; + } + + void* buf = nullptr; + al_status_t s = al_read_data(ctx, field, "", &buf, DType::value, + expected_rank, size); + if (s.code == 0 && buf != nullptr) { + out_shape->assign(size, size + expected_rank); + const std::size_t n = element_count(*out_shape); + T* p = static_cast(buf); + out_data->assign(p, p + n); + } + free(buf); + return s; +} + +} // namespace al_contract + +#endif // AL_CONTRACT_ABI_H diff --git a/tests/contract/equilibrium_seed.h b/fixtures/equilibrium_seed.h similarity index 99% rename from tests/contract/equilibrium_seed.h rename to fixtures/equilibrium_seed.h index 07fa60d5..182ba10f 100644 --- a/tests/contract/equilibrium_seed.h +++ b/fixtures/equilibrium_seed.h @@ -38,7 +38,7 @@ #ifndef AL_CONTRACT_EQUILIBRIUM_SEED_H #define AL_CONTRACT_EQUILIBRIUM_SEED_H -#include "al_contract.h" +#include "al_contract_abi.h" #include #include diff --git a/fixtures/fixture_smoke.cpp b/fixtures/fixture_smoke.cpp new file mode 100644 index 00000000..dddd0aef --- /dev/null +++ b/fixtures/fixture_smoke.cpp @@ -0,0 +1,77 @@ +// Throwaway proof (issue #51 acceptance criterion 3): a minimal translation +// unit living outside tests/ that includes equilibrium_seed.h, builds, and +// exercises it against a real backend through the public C ABI only -- no +// GoogleTest, no AL_BUILD_TESTS. Demonstrates that the fixture extracted by +// this issue is genuinely linkable from a target like the future +// benchmarks/ tree (PRD #50), not just from tests/contract/. +// +// Uses the always-on, zero-I/O Memory backend so this needs no on-disk setup +// and no optional backend enabled. + +#include "equilibrium_seed.h" + +#include +#include + +#include +#include +#include +#include + +int main() { + // The Memory backend stores nothing on disk, but URI construction still + // resolves the legacy "user" field as a base directory, so it must exist + // (mirrors tests/contract/al_contract.h's TempBase for the on-disk cases). + const std::filesystem::path base = + std::filesystem::temp_directory_path() / "al_contract_fixtures_smoke"; + std::filesystem::create_directories(base); + const std::string base_str = base.string(); + + char* uri = nullptr; + al_status_t s = al_build_uri_from_legacy_parameters( + MEMORY_BACKEND, /*pulse=*/1, /*run=*/0, /*user=*/base_str.c_str(), + /*tokamak=*/"test", /*version=*/"3", /*options=*/"", &uri); + if (s.code != 0) { + std::fprintf(stderr, "al_build_uri_from_legacy_parameters failed: %s\n", + s.message); + return 1; + } + + int pulse_ctx = -1; + s = al_begin_dataentry_action(uri, FORCE_CREATE_PULSE, &pulse_ctx); + free(uri); + if (s.code != 0) { + std::fprintf(stderr, "al_begin_dataentry_action failed: %s\n", s.message); + return 1; + } + + s = equilibrium_seed::write(pulse_ctx); + if (s.code != 0) { + std::fprintf(stderr, "equilibrium_seed::write failed: %s\n", s.message); + return 1; + } + + uint64_t observed_hash = 0; + s = equilibrium_seed::read_and_hash(pulse_ctx, &observed_hash); + if (s.code != 0) { + std::fprintf(stderr, "equilibrium_seed::read_and_hash failed: %s\n", + s.message); + return 1; + } + + al_close_pulse(pulse_ctx, CLOSE_PULSE); + + if (observed_hash != equilibrium_seed::expected_hash()) { + std::fprintf(stderr, + "structural hash mismatch: observed=%llu expected=%llu\n", + static_cast(observed_hash), + static_cast( + equilibrium_seed::expected_hash())); + return 1; + } + + std::printf("equilibrium seed round-tripped through the Memory backend " + "outside the test tree: hash=%llu\n", + static_cast(observed_hash)); + return 0; +} diff --git a/tests/contract/CMakeLists.txt b/tests/contract/CMakeLists.txt index 7f338d9e..b22a72bf 100644 --- a/tests/contract/CMakeLists.txt +++ b/tests/contract/CMakeLists.txt @@ -85,7 +85,7 @@ add_executable(contract_tests test_uda_unique_surface.cpp ) -target_link_libraries(contract_tests PRIVATE al GTest::gtest GTest::gtest_main) +target_link_libraries(contract_tests PRIVATE al al_contract_fixtures GTest::gtest GTest::gtest_main) add_dependencies(contract_tests al) target_compile_features(contract_tests PRIVATE cxx_std_17) diff --git a/tests/contract/TRACEABILITY.md b/tests/contract/TRACEABILITY.md index 86519bed..9a59e6e7 100644 --- a/tests/contract/TRACEABILITY.md +++ b/tests/contract/TRACEABILITY.md @@ -140,7 +140,9 @@ genuine defect (expected-fail). The deterministic, in-repo single-version equilibrium-seed generator (a scalar, a timebase-carrying 2-D array, and a constraints AOS — no committed binary blobs; oracle = a content hash recomputed from the generator functions -themselves, `tests/contract/equilibrium_seed.h`), round-tripped on the three +themselves, `fixtures/equilibrium_seed.h` — moved outside tests/contract/ by +issue #51 so a future benchmarks/ target can link it directly), round-tripped +on the three backends where AOS content is actually readable back within a session. | Capability | Inventory ref | Unit | Integ. | Test(s) | Status | diff --git a/tests/contract/al_contract.h b/tests/contract/al_contract.h index 71d44efb..616a5b55 100644 --- a/tests/contract/al_contract.h +++ b/tests/contract/al_contract.h @@ -12,6 +12,8 @@ #ifndef AL_CONTRACT_H #define AL_CONTRACT_H +#include "al_contract_abi.h" + #include #include @@ -20,8 +22,6 @@ #include #include #include -#include -#include #include #include #include @@ -345,30 +345,12 @@ inline al_status_t read_char_array(int ctx, const char* field, // itself the proof of the version-free property. This block generalizes the // two hand-written scalar/char helpers above to the whole // datatype × shape space. - -// --- datatype descriptor: element C++ type -> ABI datatype id + label ------- -template -struct DType; -template <> -struct DType { - static constexpr int value = CHAR_DATA; - static constexpr const char* name = "CHAR"; -}; -template <> -struct DType { - static constexpr int value = INTEGER_DATA; - static constexpr const char* name = "INTEGER"; -}; -template <> -struct DType { - static constexpr int value = DOUBLE_DATA; - static constexpr const char* name = "DOUBLE"; -}; -template <> -struct DType> { - static constexpr int value = COMPLEX_DATA; - static constexpr const char* name = "COMPLEX"; -}; +// +// DType/write_data/read_data/element_count (the subset equilibrium_seed.h also +// needs) live in al_contract_abi.h (issue #51), included above, so a build +// target outside tests/contract/ can link them without GoogleTest. The +// synthetic-buffer generator and its shape helper, used exclusively by this +// suite's own matrix tests, stay here. // The core's "this value means absent" sentinels (src/al_lowlevel.cpp:41-44). // A *scalar* write whose value equals its sentinel is silently dropped by @@ -420,12 +402,6 @@ inline std::vector shape_for_rank(int rank) { return s; } -inline std::size_t element_count(const std::vector& shape) { - std::size_t n = 1; - for (int d : shape) n *= static_cast(d); - return n; -} - template std::vector synth_buffer(std::size_t n) { std::vector v; @@ -434,55 +410,6 @@ std::vector synth_buffer(std::size_t n) { return v; } -// --- generic write ---------------------------------------------------------- -// al_write_data takes a non-const void* and int* size but does not mutate -// them for a write; the const_casts keep the caller's data/shape const. -template -al_status_t write_data(int ctx, const char* field, const std::vector& shape, - const std::vector& data) { - const int dim = static_cast(shape.size()); - int* size = dim ? const_cast(shape.data()) : nullptr; - return al_write_data(ctx, field, "", const_cast(data.data()), - DType::value, dim, size); -} - -// --- generic read ----------------------------------------------------------- -// Honors the two ownership regimes the ABI uses (mirrors testlowlevel.cpp): -// dim == 0 : the core writes into the caller's buffer; nothing to free. -// dim >= 1 : the core malloc's the buffer and returns it via *data; the -// caller owns it, so we copy out and free. -// On success out_shape is the shape the core reported (scalar -> empty) and -// out_data holds element_count(out_shape) elements. -template -al_status_t read_data(int ctx, const char* field, int expected_rank, - std::vector* out_shape, std::vector* out_data) { - int size[MAXDIM] = {0}; - - if (expected_rank == 0) { - T scalar{}; - void* buf = &scalar; - al_status_t s = - al_read_data(ctx, field, "", &buf, DType::value, 0, size); - if (s.code == 0) { - out_shape->clear(); - out_data->assign(1, scalar); - } - return s; - } - - void* buf = nullptr; - al_status_t s = al_read_data(ctx, field, "", &buf, DType::value, - expected_rank, size); - if (s.code == 0 && buf != nullptr) { - out_shape->assign(size, size + expected_rank); - const std::size_t n = element_count(*out_shape); - T* p = static_cast(buf); - out_data->assign(p, p + n); - } - free(buf); - return s; -} - } // namespace al_contract #endif // AL_CONTRACT_H diff --git a/tests/contract/test_uda.cpp b/tests/contract/test_uda.cpp index d9a5c8de..af862b88 100644 --- a/tests/contract/test_uda.cpp +++ b/tests/contract/test_uda.cpp @@ -115,7 +115,7 @@ TEST_F(UdaSmokeRoundTrip, ScalarSeededViaHdf5ReadsBackThroughUda) { // // Broadens the tracer bullet above from one scalar to the full equilibrium-seed // composite shape (static scalar + top-level timebase array + a real -// time_slice/profiles_1d/constraints AOS, tests/contract/equilibrium_seed.h — +// time_slice/profiles_1d/constraints AOS, fixtures/equilibrium_seed.h — // issues #4/#33/D5), reusing the seed's own generator and structural hash // oracle unchanged: seed via HDF5 (fixture setup, same role the baked model // tree plays for MDSplus), reopen the identical path through the UDA backend in