Contract test suite (5/5): CMake presets, test docs, shared fixtures

.github/workflows/contract-tests.yml

@@ -15,16 +15,22 @@ on:
 
 jobs:
   contract-functional:
-    name: Contract tests (Linux, ${{ matrix.build_type }}, blocking)
+    name: Contract tests (Linux, ${{ matrix.preset }}, blocking)
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         # Both build modes (issue #32): the dlopen-failure plugin pin is
         # build-mode dependent — assert-abort death test in Debug, swallowed
         # assert tripwire under NDEBUG (RelWithDebInfo) — so each branch needs
-        # its own leg or one of them never runs anywhere.
-        build_type: [RelWithDebInfo, Debug]
+        # its own leg or one of them never runs anywhere. The presets
+        # (CMakePresets.json) are the single source of truth for flags and
+        # binary dirs; builddir here only feeds the label-partition check.
+        include:
+          - preset: tests        # RelWithDebInfo
+            builddir: build
+          - preset: tests-debug
+            builddir: build-debug
 
     steps:
       - name: Checkout code
@@ -38,20 +44,15 @@ jobs:
           sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev
 
       - name: Configure (tests on, no Python bindings)
-        run: >
-          cmake -S . -B build -G Ninja
-          -DAL_BUILD_TESTS=ON
-          -DAL_PYTHON_BINDINGS=OFF
-          -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
+        run: cmake --preset ${{ matrix.preset }}
 
       - name: Build
-        run: cmake --build build
+        run: cmake --build --preset ${{ matrix.preset }}
 
       # Issue #31: an empty `ctest -L unit` exits 0, so a broken label
       # partition is invisible to the test run below — check it explicitly.
       - name: Verify unit/integration label partition
-        run: bash tests/contract/check_label_partition.sh build/tests/contract
+        run: bash tests/contract/check_label_partition.sh ${{ matrix.builddir }}/tests/contract
 
       - name: Run the whole test suite
-        working-directory: build
-        run: ctest --output-on-failure
+        run: ctest --preset ${{ matrix.preset }}

.github/workflows/mdsplus-contract.yml

@@ -68,25 +68,23 @@ jobs:
           key: mdsplus-model-${{ env.DD_VERSION }}-${{ hashFiles('common/cmake/ALBuildDataDictionary.cmake', 'models/mdsplus/CMakeLists.txt') }}
 
       - name: Configure (bakes the DD model tree on a cache miss)
+        # The `mdsplus` preset (CMakePresets.json) carries the backend/model/
+        # test flags and the build-mdsplus binary dir; only the DD pin stays
+        # here, next to the cache key it feeds.
         if: steps.cache-model.outputs.cache-hit != 'true'
         run: >
           docker run --rm --platform linux/arm64
           -v "$PWD":/workspace -w /workspace
           "${IMAGE_TAG}" bash -c "
-          cmake -B build-mdsplus -G Ninja
-          -DAL_BACKEND_MDSPLUS=ON
-          -DAL_BUILD_MDSPLUS_MODELS=ON
-          -DAL_BUILD_TESTS=ON
-          -DDD_VERSION=${DD_VERSION}
-          -DCMAKE_BUILD_TYPE=Release
+          cmake --preset mdsplus -DDD_VERSION=${DD_VERSION}
           "
 
       - name: Build contract suite
         run: >
           docker run --rm --platform linux/arm64
           -v "$PWD":/workspace -w /workspace
           "${IMAGE_TAG}" bash -c "
-          cmake --build build-mdsplus --target contract_tests -j\$(nproc)
+          cmake --build --preset mdsplus
           "
 
       - name: Verify baked DD model tree is present (cache-hit and cache-miss paths)
@@ -108,12 +106,13 @@ jobs:
           echo "Baked DD model tree complete under build-mdsplus/models/mdsplus."
 
       - name: Run MDSplus contract suite
+        # The `mdsplus` test preset selects the tier (-L mdsplus) and points
+        # MDSPLUS_MODELS_PATH at the baked model tree under build-mdsplus/.
         run: >
           docker run --rm --platform linux/arm64
           -v "$PWD":/workspace -w /workspace
-          -e MDSPLUS_MODELS_PATH=/workspace/build-mdsplus/models/mdsplus
           "${IMAGE_TAG}" bash -c "
-          cd build-mdsplus/tests/contract && ctest -L mdsplus --output-on-failure
+          ctest --preset mdsplus
           "
 
       - name: Note

.github/workflows/sanitizers.yml

@@ -32,23 +32,19 @@ jobs:
           sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev
 
       - name: Configure (sanitized, tests on, no Python bindings)
-        # AL_CONTRACT_SANITIZE instruments both libal and the test binary. Python
-        # bindings are off: a sanitized libal must not be linked into a wheel
-        # whose bindings lack the ASan runtime (see tests/contract/CMakeLists.txt).
-        run: >
-          cmake -S . -B build-san -G Ninja
-          -DAL_BUILD_TESTS=ON
-          -DAL_CONTRACT_SANITIZE=ON
-          -DAL_PYTHON_BINDINGS=OFF
-          -DCMAKE_BUILD_TYPE=RelWithDebInfo
+        # The `sanitize` preset (CMakePresets.json) sets AL_CONTRACT_SANITIZE,
+        # which instruments both libal and the test binary. Python bindings are
+        # off: a sanitized libal must not be linked into a wheel whose bindings
+        # lack the ASan runtime (see tests/contract/CMakeLists.txt).
+        run: cmake --preset sanitize
 
       - name: Build contract suite
-        run: cmake --build build-san --target contract_tests
+        run: cmake --build --preset sanitize
 
       - name: Run contract suite under sanitizers
-        working-directory: build-san
-        # Scope to the contract suite; halt/abort_on_error=0 so the whole suite
-        # runs and reports everything at once (informational leg). Two exclusions:
+        # The `sanitize` test preset carries the ASAN/LSAN/UBSAN_OPTIONS
+        # (halt/abort_on_error=0 so the whole suite runs and reports everything
+        # at once — informational leg) and two exclusions:
         #  * `Death` tests deliberately crash child processes (asserting today's
         #    crash-class defects) — not a memory-safety target, and ASan would
         #    report the intentional crash as an error.
@@ -58,11 +54,7 @@ jobs:
         #    fail here. They guard against rot in the blocking functional leg
         #    (contract-tests.yml). Real leaks/UAF in the ordinary paths still
         #    surface.
-        env:
-          ASAN_OPTIONS: detect_leaks=1:halt_on_error=0:abort_on_error=0
-          LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1
-          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0
-        run: ctest -L contract -E "Death|CurrentlyCorrupts" --output-on-failure
+        run: ctest --preset sanitize
 
       - name: Note
         if: always()

.gitignore

@@ -27,6 +27,7 @@ CMakeUserPresets.json
 /vcpkg_installed/
 settings.json
 /build*/
+cmake-build-*/
 /_deps/
 /CMakeFiles
 CMakeCache.txt
@@ -44,3 +45,4 @@ python/lib/
 /.venv/
 /vcpkg/
 /UDA-*/
+.pytest_cache/

CMakeLists.txt

@@ -37,6 +37,13 @@ option(AL_PYTHON_BINDINGS "Build Python bindings" OFF)
 
 option(AL_BUILD_TESTS "Build the C/C++ low-level ABI smoke tests (tests/)" OFF)
 
+# The fixtures/al_contract_fixtures INTERFACE library itself is always
+# available (see below) so it is genuinely usable outside AL_BUILD_TESTS; this
+# only gates the throwaway demonstration executable (issue #51 acceptance
+# criterion 3), off by default like every other extra target in this file.
+option(AL_BUILD_FIXTURE_SMOKE
+       "Build a throwaway executable proving tests/contract fixtures are linkable outside the test tree" OFF)
+
 # When ON, the al C++ library is not built here; instead it is located with
 # find_package(al-core CONFIG). Enables a two-stage build where libimas-core
 # (the C/C++ library) and imas-core (the Python wrapper) are packaged
@@ -213,6 +220,11 @@ endif()
 
 add_subdirectory(src)
 
+# Shared contract fixtures (issue #51): the equilibrium seed and the ABI
+# helpers it needs, usable outside tests/ (e.g. by a future benchmarks/
+# tree). Added unconditionally so it never depends on AL_BUILD_TESTS.
+add_subdirectory(fixtures)
+
 # Dummy Executable
 # ##############################################################################
 set( EXE_FLAG  )

CMakePresets.json

@@ -0,0 +1,118 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": { "major": 3, "minor": 21, "patch": 0 },
+  "configurePresets": [
+    {
+      "name": "tests",
+      "displayName": "Test build (RelWithDebInfo)",
+      "description": "Smoke + contract suites against the public C ABI; no Python bindings. Mirrors the blocking CI leg (contract-tests.yml).",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "AL_BUILD_TESTS": "ON",
+        "AL_PYTHON_BINDINGS": "OFF",
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo"
+      }
+    },
+    {
+      "name": "tests-debug",
+      "displayName": "Test build (Debug)",
+      "description": "Debug leg of contract-tests.yml: the dlopen-failure plugin pin is build-mode dependent (issue #32), so CI runs both modes.",
+      "inherits": "tests",
+      "binaryDir": "${sourceDir}/build-debug",
+      "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" }
+    },
+    {
+      "name": "sanitize",
+      "displayName": "Sanitized test build (ASan/UBSan)",
+      "description": "AL_CONTRACT_SANITIZE=ON instruments libal and contract_tests (TEST_STRATEGY.md D7). Mirrors sanitizers.yml. Never combine with a wheel build: the bindings would link a sanitized libal without the ASan runtime.",
+      "inherits": "tests",
+      "binaryDir": "${sourceDir}/build-san",
+      "cacheVariables": { "AL_CONTRACT_SANITIZE": "ON" }
+    },
+    {
+      "name": "mdsplus",
+      "displayName": "MDSplus test build (bakes the DD model tree)",
+      "description": "MDSplus backend + baked model tree + tests. Pass -DDD_VERSION=<tag> to pin the DD (models/mdsplus defaults to 'main'; mdsplus-contract.yml pins its own). Downloads the DD at configure time.",
+      "inherits": "tests",
+      "binaryDir": "${sourceDir}/build-mdsplus",
+      "cacheVariables": {
+        "AL_BACKEND_MDSPLUS": "ON",
+        "AL_BUILD_MDSPLUS_MODELS": "ON",
+        "CMAKE_BUILD_TYPE": "Release"
+      }
+    }
+  ],
+  "buildPresets": [
+    { "name": "tests", "configurePreset": "tests" },
+    { "name": "tests-debug", "configurePreset": "tests-debug" },
+    {
+      "name": "sanitize",
+      "configurePreset": "sanitize",
+      "targets": [ "contract_tests" ]
+    },
+    {
+      "name": "mdsplus",
+      "configurePreset": "mdsplus",
+      "targets": [ "contract_tests" ]
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "unit",
+      "displayName": "Hermetic inner loop: pure lookups, Memory backend, plugin registry",
+      "configurePreset": "tests",
+      "filter": { "include": { "label": "^unit$" } },
+      "output": { "outputOnFailure": true },
+      "execution": { "noTestsAction": "error" }
+    },
+    {
+      "name": "integration",
+      "displayName": "On-disk tier: HDF5 / ASCII / Flexbuffers backends",
+      "configurePreset": "tests",
+      "filter": { "include": { "label": "^integration$" } },
+      "output": { "outputOnFailure": true },
+      "execution": { "noTestsAction": "error" }
+    },
+    {
+      "name": "tests",
+      "displayName": "Everything registered in the test build (contract + smoke)",
+      "configurePreset": "tests",
+      "output": { "outputOnFailure": true },
+      "execution": { "noTestsAction": "error" }
+    },
+    {
+      "name": "tests-debug",
+      "displayName": "Everything, Debug build (issue #32 leg)",
+      "configurePreset": "tests-debug",
+      "inherits": "tests"
+    },
+    {
+      "name": "sanitize",
+      "displayName": "Contract suite under ASan/UBSan (informational CI leg)",
+      "configurePreset": "sanitize",
+      "filter": {
+        "include": { "label": "^contract$" },
+        "exclude": { "name": "Death|CurrentlyCorrupts" }
+      },
+      "environment": {
+        "ASAN_OPTIONS": "detect_leaks=1:halt_on_error=0:abort_on_error=0",
+        "LSAN_OPTIONS": "suppressions=${sourceDir}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1",
+        "UBSAN_OPTIONS": "print_stacktrace=1:halt_on_error=0"
+      },
+      "output": { "outputOnFailure": true },
+      "execution": { "noTestsAction": "error" }
+    },
+    {
+      "name": "mdsplus",
+      "displayName": "MDSplus tier against the baked model tree",
+      "configurePreset": "mdsplus",
+      "filter": { "include": { "label": "^mdsplus$" } },
+      "environment": {
+        "MDSPLUS_MODELS_PATH": "${sourceDir}/build-mdsplus/models/mdsplus"
+      },
+      "output": { "outputOnFailure": true },
+      "execution": { "noTestsAction": "error" }
+    }
+  ]
+}

CONTRIBUTING.md

@@ -30,7 +30,11 @@ profile and create your own feature branch off of the latest develop commit.
 Make sure to regularly sync your branch with the latest commits from `develop` 
 (find instructions 
 [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork));
-5. when your development is ready, create a pull request (find instructions 
+5. make sure the test suite passes before submitting: see 
+[tests/README.md](tests/README.md) for how to build and run it 
+(`cmake --preset tests && cmake --build --preset tests && ctest --preset tests` 
+is the same run as the blocking CI check);
+6. when your development is ready, create a pull request (find instructions 
 [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)).
 
 

README.md

@@ -42,6 +42,17 @@ cmake --build build --target install
 
 See [Developer Guide](docs/source/developers/index.rst) for build instructions.
 
+### Running the tests
+
+The C/C++ and Python test suites, their CTest tiers, and the CMake presets
+that drive them (locally and in CI) are documented in
+[tests/README.md](tests/README.md). The short version:
+
+```bash
+cmake --preset tests && cmake --build --preset tests
+ctest --preset unit    # fast hermetic loop; `ctest --preset tests` for everything
+```
+
 
 ## Using IMAS-Core with High-Level Languages
 

fixtures/CMakeLists.txt

@@ -0,0 +1,44 @@
+########################################################################################################################
+# Shared contract fixtures (issue #51).
+#
+# The realistic equilibrium seed (equilibrium_seed.h) and the generic typed
+# ABI round-trip helpers it needs (al_contract_abi.h), factored out of
+# tests/contract/ so a build target outside the test tree -- the future
+# benchmarks/ tree (PRD #50, issues #52/#53) -- can link the exact same
+# workload the contract suite pins. No fixture drift between what is tested
+# and what is measured (spec #50, Option C).
+#
+# Header-only and carries no GoogleTest dependency (unlike
+# tests/contract/al_contract.h), so a non-test binary can link it for free.
+# Added unconditionally (not gated by AL_BUILD_TESTS) so it is genuinely
+# usable independent of the test build.
+
+add_library(al_contract_fixtures INTERFACE)
+target_include_directories(al_contract_fixtures INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(al_contract_fixtures INTERFACE al)
+target_compile_features(al_contract_fixtures INTERFACE cxx_std_17)
+
+########################################################################################################################
+# Throwaway proof (issue #51 acceptance criterion 3), opt-in via
+# AL_BUILD_FIXTURE_SMOKE -- off by default like every other extra target in
+# the root CMakeLists.txt (AL_BUILD_TESTS, AL_BUILD_BENCHMARKS), so an
+# ordinary/minimal build (e.g. the Python wheel) never pays for it. The
+# al_contract_fixtures library above stays unconditional regardless: that is
+# the actual "usable outside the test tree" requirement, this executable is
+# just a demonstration of it.
+#
+# A minimal translation unit outside tests/ that includes the fixture, builds,
+# and round-trips it against a real backend -- demonstrating the fixture is
+# usable without AL_BUILD_TESTS or GoogleTest. Uses the always-on, zero-I/O
+# Memory backend so this needs no on-disk setup and no optional backend.
+if(AL_BUILD_FIXTURE_SMOKE)
+  add_executable(al_contract_fixtures_smoke fixture_smoke.cpp)
+  target_link_libraries(al_contract_fixtures_smoke PRIVATE al_contract_fixtures)
+  add_dependencies(al_contract_fixtures_smoke al)
+  target_compile_features(al_contract_fixtures_smoke PRIVATE cxx_std_17)
+
+  if(NOT WIN32 AND NOT APPLE)
+    # stdc++fs is only needed on Linux with older GCC (std::filesystem in the smoke TU).
+    target_link_libraries(al_contract_fixtures_smoke PRIVATE stdc++fs)
+  endif()
+endif()