diff --git a/.github/workflows/contract-tests.yml b/.github/workflows/contract-tests.yml index 1787dfdd..64ddd3b1 100644 --- a/.github/workflows/contract-tests.yml +++ b/.github/workflows/contract-tests.yml @@ -15,7 +15,7 @@ on: jobs: contract-functional: - name: Contract tests (Linux, ${{ matrix.build_type }}, blocking) + name: Contract tests (Linux, ${{ matrix.preset }}, blocking) runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,8 +23,14 @@ jobs: # Both build modes (issue #32): the dlopen-failure plugin pin is # build-mode dependent — assert-abort death test in Debug, swallowed # assert tripwire under NDEBUG (RelWithDebInfo) — so each branch needs - # its own leg or one of them never runs anywhere. - build_type: [RelWithDebInfo, Debug] + # its own leg or one of them never runs anywhere. The presets + # (CMakePresets.json) are the single source of truth for flags and + # binary dirs; builddir here only feeds the label-partition check. + include: + - preset: tests # RelWithDebInfo + builddir: build + - preset: tests-debug + builddir: build-debug steps: - name: Checkout code @@ -38,20 +44,15 @@ jobs: sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev - name: Configure (tests on, no Python bindings) - run: > - cmake -S . -B build -G Ninja - -DAL_BUILD_TESTS=ON - -DAL_PYTHON_BINDINGS=OFF - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + run: cmake --preset ${{ matrix.preset }} - name: Build - run: cmake --build build + run: cmake --build --preset ${{ matrix.preset }} # Issue #31: an empty `ctest -L unit` exits 0, so a broken label # partition is invisible to the test run below — check it explicitly. - name: Verify unit/integration label partition - run: bash tests/contract/check_label_partition.sh build/tests/contract + run: bash tests/contract/check_label_partition.sh ${{ matrix.builddir }}/tests/contract - name: Run the whole test suite - working-directory: build - run: ctest --output-on-failure + run: ctest --preset ${{ matrix.preset }} diff --git a/.github/workflows/mdsplus-contract.yml b/.github/workflows/mdsplus-contract.yml index 8222b82c..ca035693 100644 --- a/.github/workflows/mdsplus-contract.yml +++ b/.github/workflows/mdsplus-contract.yml @@ -68,17 +68,15 @@ jobs: key: mdsplus-model-${{ env.DD_VERSION }}-${{ hashFiles('common/cmake/ALBuildDataDictionary.cmake', 'models/mdsplus/CMakeLists.txt') }} - name: Configure (bakes the DD model tree on a cache miss) + # The `mdsplus` preset (CMakePresets.json) carries the backend/model/ + # test flags and the build-mdsplus binary dir; only the DD pin stays + # here, next to the cache key it feeds. if: steps.cache-model.outputs.cache-hit != 'true' run: > docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace "${IMAGE_TAG}" bash -c " - cmake -B build-mdsplus -G Ninja - -DAL_BACKEND_MDSPLUS=ON - -DAL_BUILD_MDSPLUS_MODELS=ON - -DAL_BUILD_TESTS=ON - -DDD_VERSION=${DD_VERSION} - -DCMAKE_BUILD_TYPE=Release + cmake --preset mdsplus -DDD_VERSION=${DD_VERSION} " - name: Build contract suite @@ -86,7 +84,7 @@ jobs: docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace "${IMAGE_TAG}" bash -c " - cmake --build build-mdsplus --target contract_tests -j\$(nproc) + cmake --build --preset mdsplus " - name: Verify baked DD model tree is present (cache-hit and cache-miss paths) @@ -108,12 +106,13 @@ jobs: echo "Baked DD model tree complete under build-mdsplus/models/mdsplus." - name: Run MDSplus contract suite + # The `mdsplus` test preset selects the tier (-L mdsplus) and points + # MDSPLUS_MODELS_PATH at the baked model tree under build-mdsplus/. run: > docker run --rm --platform linux/arm64 -v "$PWD":/workspace -w /workspace - -e MDSPLUS_MODELS_PATH=/workspace/build-mdsplus/models/mdsplus "${IMAGE_TAG}" bash -c " - cd build-mdsplus/tests/contract && ctest -L mdsplus --output-on-failure + ctest --preset mdsplus " - name: Note diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 41a16e2d..0d7b7d5c 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -32,23 +32,19 @@ jobs: sudo apt-get install -y ninja-build libhdf5-dev libboost-filesystem-dev - name: Configure (sanitized, tests on, no Python bindings) - # AL_CONTRACT_SANITIZE instruments both libal and the test binary. Python - # bindings are off: a sanitized libal must not be linked into a wheel - # whose bindings lack the ASan runtime (see tests/contract/CMakeLists.txt). - run: > - cmake -S . -B build-san -G Ninja - -DAL_BUILD_TESTS=ON - -DAL_CONTRACT_SANITIZE=ON - -DAL_PYTHON_BINDINGS=OFF - -DCMAKE_BUILD_TYPE=RelWithDebInfo + # The `sanitize` preset (CMakePresets.json) sets AL_CONTRACT_SANITIZE, + # which instruments both libal and the test binary. Python bindings are + # off: a sanitized libal must not be linked into a wheel whose bindings + # lack the ASan runtime (see tests/contract/CMakeLists.txt). + run: cmake --preset sanitize - name: Build contract suite - run: cmake --build build-san --target contract_tests + run: cmake --build --preset sanitize - name: Run contract suite under sanitizers - working-directory: build-san - # Scope to the contract suite; halt/abort_on_error=0 so the whole suite - # runs and reports everything at once (informational leg). Two exclusions: + # The `sanitize` test preset carries the ASAN/LSAN/UBSAN_OPTIONS + # (halt/abort_on_error=0 so the whole suite runs and reports everything + # at once — informational leg) and two exclusions: # * `Death` tests deliberately crash child processes (asserting today's # crash-class defects) — not a memory-safety target, and ASan would # report the intentional crash as an error. @@ -58,11 +54,7 @@ jobs: # fail here. They guard against rot in the blocking functional leg # (contract-tests.yml). Real leaks/UAF in the ordinary paths still # surface. - env: - ASAN_OPTIONS: detect_leaks=1:halt_on_error=0:abort_on_error=0 - LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0 - run: ctest -L contract -E "Death|CurrentlyCorrupts" --output-on-failure + run: ctest --preset sanitize - name: Note if: always() diff --git a/.gitignore b/.gitignore index f5697b3c..f67d841c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ CMakeUserPresets.json /vcpkg_installed/ settings.json /build*/ +cmake-build-*/ /_deps/ /CMakeFiles CMakeCache.txt @@ -44,3 +45,4 @@ python/lib/ /.venv/ /vcpkg/ /UDA-*/ +.pytest_cache/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b2d72f3..dcbb5780 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,13 @@ option(AL_PYTHON_BINDINGS "Build Python bindings" OFF) option(AL_BUILD_TESTS "Build the C/C++ low-level ABI smoke tests (tests/)" OFF) +# The fixtures/al_contract_fixtures INTERFACE library itself is always +# available (see below) so it is genuinely usable outside AL_BUILD_TESTS; this +# only gates the throwaway demonstration executable (issue #51 acceptance +# criterion 3), off by default like every other extra target in this file. +option(AL_BUILD_FIXTURE_SMOKE + "Build a throwaway executable proving tests/contract fixtures are linkable outside the test tree" OFF) + # When ON, the al C++ library is not built here; instead it is located with # find_package(al-core CONFIG). Enables a two-stage build where libimas-core # (the C/C++ library) and imas-core (the Python wrapper) are packaged @@ -213,6 +220,11 @@ endif() add_subdirectory(src) +# Shared contract fixtures (issue #51): the equilibrium seed and the ABI +# helpers it needs, usable outside tests/ (e.g. by a future benchmarks/ +# tree). Added unconditionally so it never depends on AL_BUILD_TESTS. +add_subdirectory(fixtures) + # Dummy Executable # ############################################################################## set( EXE_FLAG ) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..96174b68 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,118 @@ +{ + "version": 3, + "cmakeMinimumRequired": { "major": 3, "minor": 21, "patch": 0 }, + "configurePresets": [ + { + "name": "tests", + "displayName": "Test build (RelWithDebInfo)", + "description": "Smoke + contract suites against the public C ABI; no Python bindings. Mirrors the blocking CI leg (contract-tests.yml).", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "AL_BUILD_TESTS": "ON", + "AL_PYTHON_BINDINGS": "OFF", + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + } + }, + { + "name": "tests-debug", + "displayName": "Test build (Debug)", + "description": "Debug leg of contract-tests.yml: the dlopen-failure plugin pin is build-mode dependent (issue #32), so CI runs both modes.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-debug", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } + }, + { + "name": "sanitize", + "displayName": "Sanitized test build (ASan/UBSan)", + "description": "AL_CONTRACT_SANITIZE=ON instruments libal and contract_tests (TEST_STRATEGY.md D7). Mirrors sanitizers.yml. Never combine with a wheel build: the bindings would link a sanitized libal without the ASan runtime.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-san", + "cacheVariables": { "AL_CONTRACT_SANITIZE": "ON" } + }, + { + "name": "mdsplus", + "displayName": "MDSplus test build (bakes the DD model tree)", + "description": "MDSplus backend + baked model tree + tests. Pass -DDD_VERSION= to pin the DD (models/mdsplus defaults to 'main'; mdsplus-contract.yml pins its own). Downloads the DD at configure time.", + "inherits": "tests", + "binaryDir": "${sourceDir}/build-mdsplus", + "cacheVariables": { + "AL_BACKEND_MDSPLUS": "ON", + "AL_BUILD_MDSPLUS_MODELS": "ON", + "CMAKE_BUILD_TYPE": "Release" + } + } + ], + "buildPresets": [ + { "name": "tests", "configurePreset": "tests" }, + { "name": "tests-debug", "configurePreset": "tests-debug" }, + { + "name": "sanitize", + "configurePreset": "sanitize", + "targets": [ "contract_tests" ] + }, + { + "name": "mdsplus", + "configurePreset": "mdsplus", + "targets": [ "contract_tests" ] + } + ], + "testPresets": [ + { + "name": "unit", + "displayName": "Hermetic inner loop: pure lookups, Memory backend, plugin registry", + "configurePreset": "tests", + "filter": { "include": { "label": "^unit$" } }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "integration", + "displayName": "On-disk tier: HDF5 / ASCII / Flexbuffers backends", + "configurePreset": "tests", + "filter": { "include": { "label": "^integration$" } }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "tests", + "displayName": "Everything registered in the test build (contract + smoke)", + "configurePreset": "tests", + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "tests-debug", + "displayName": "Everything, Debug build (issue #32 leg)", + "configurePreset": "tests-debug", + "inherits": "tests" + }, + { + "name": "sanitize", + "displayName": "Contract suite under ASan/UBSan (informational CI leg)", + "configurePreset": "sanitize", + "filter": { + "include": { "label": "^contract$" }, + "exclude": { "name": "Death|CurrentlyCorrupts" } + }, + "environment": { + "ASAN_OPTIONS": "detect_leaks=1:halt_on_error=0:abort_on_error=0", + "LSAN_OPTIONS": "suppressions=${sourceDir}/tests/contract/sanitizer_suppressions.txt:print_suppressions=1", + "UBSAN_OPTIONS": "print_stacktrace=1:halt_on_error=0" + }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + }, + { + "name": "mdsplus", + "displayName": "MDSplus tier against the baked model tree", + "configurePreset": "mdsplus", + "filter": { "include": { "label": "^mdsplus$" } }, + "environment": { + "MDSPLUS_MODELS_PATH": "${sourceDir}/build-mdsplus/models/mdsplus" + }, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error" } + } + ] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69e18b1e..c7d8ceb3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,11 @@ profile and create your own feature branch off of the latest develop commit. Make sure to regularly sync your branch with the latest commits from `develop` (find instructions [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)); -5. when your development is ready, create a pull request (find instructions +5. make sure the test suite passes before submitting: see +[tests/README.md](tests/README.md) for how to build and run it +(`cmake --preset tests && cmake --build --preset tests && ctest --preset tests` +is the same run as the blocking CI check); +6. when your development is ready, create a pull request (find instructions [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)). diff --git a/README.md b/README.md index 70a0480b..673700e7 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,17 @@ cmake --build build --target install See [Developer Guide](docs/source/developers/index.rst) for build instructions. +### Running the tests + +The C/C++ and Python test suites, their CTest tiers, and the CMake presets +that drive them (locally and in CI) are documented in +[tests/README.md](tests/README.md). The short version: + +```bash +cmake --preset tests && cmake --build --preset tests +ctest --preset unit # fast hermetic loop; `ctest --preset tests` for everything +``` + ## Using IMAS-Core with High-Level Languages diff --git a/fixtures/CMakeLists.txt b/fixtures/CMakeLists.txt new file mode 100644 index 00000000..c674ed3d --- /dev/null +++ b/fixtures/CMakeLists.txt @@ -0,0 +1,44 @@ +######################################################################################################################## +# Shared contract fixtures (issue #51). +# +# The realistic equilibrium seed (equilibrium_seed.h) and the generic typed +# ABI round-trip helpers it needs (al_contract_abi.h), factored out of +# tests/contract/ so a build target outside the test tree -- the future +# benchmarks/ tree (PRD #50, issues #52/#53) -- can link the exact same +# workload the contract suite pins. No fixture drift between what is tested +# and what is measured (spec #50, Option C). +# +# Header-only and carries no GoogleTest dependency (unlike +# tests/contract/al_contract.h), so a non-test binary can link it for free. +# Added unconditionally (not gated by AL_BUILD_TESTS) so it is genuinely +# usable independent of the test build. + +add_library(al_contract_fixtures INTERFACE) +target_include_directories(al_contract_fixtures INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(al_contract_fixtures INTERFACE al) +target_compile_features(al_contract_fixtures INTERFACE cxx_std_17) + +######################################################################################################################## +# Throwaway proof (issue #51 acceptance criterion 3), opt-in via +# AL_BUILD_FIXTURE_SMOKE -- off by default like every other extra target in +# the root CMakeLists.txt (AL_BUILD_TESTS, AL_BUILD_BENCHMARKS), so an +# ordinary/minimal build (e.g. the Python wheel) never pays for it. The +# al_contract_fixtures library above stays unconditional regardless: that is +# the actual "usable outside the test tree" requirement, this executable is +# just a demonstration of it. +# +# A minimal translation unit outside tests/ that includes the fixture, builds, +# and round-trips it against a real backend -- demonstrating the fixture is +# usable without AL_BUILD_TESTS or GoogleTest. Uses the always-on, zero-I/O +# Memory backend so this needs no on-disk setup and no optional backend. +if(AL_BUILD_FIXTURE_SMOKE) + add_executable(al_contract_fixtures_smoke fixture_smoke.cpp) + target_link_libraries(al_contract_fixtures_smoke PRIVATE al_contract_fixtures) + add_dependencies(al_contract_fixtures_smoke al) + target_compile_features(al_contract_fixtures_smoke PRIVATE cxx_std_17) + + if(NOT WIN32 AND NOT APPLE) + # stdc++fs is only needed on Linux with older GCC (std::filesystem in the smoke TU). + target_link_libraries(al_contract_fixtures_smoke PRIVATE stdc++fs) + endif() +endif() diff --git a/fixtures/al_contract_abi.h b/fixtures/al_contract_abi.h new file mode 100644 index 00000000..8f4d8933 --- /dev/null +++ b/fixtures/al_contract_abi.h @@ -0,0 +1,106 @@ +// Generic typed round-trip helpers over al_write_data / al_read_data +// (issue #51). Factored out of tests/contract/al_contract.h so a build target +// outside tests/contract/ -- notably a future benchmarks/ tree (PRD #50, issue +// #52) -- can link the exact same write_data/read_data machinery +// equilibrium_seed.h uses, without also pulling in GoogleTest. tests/contract/ +// al_contract.h includes this header for its own copy of these helpers, so +// there is exactly one definition either way. +// +// Binds only to the `extern "C"` surface (al_lowlevel.h + al_const.h), per +// TEST_STRATEGY.md decision D1: no C++ internals, no Python bindings. + +#ifndef AL_CONTRACT_ABI_H +#define AL_CONTRACT_ABI_H + +#include +#include + +#include +#include +#include +#include + +namespace al_contract { + +// --- datatype descriptor: element C++ type -> ABI datatype id + label ------- +template +struct DType; +template <> +struct DType { + static constexpr int value = CHAR_DATA; + static constexpr const char* name = "CHAR"; +}; +template <> +struct DType { + static constexpr int value = INTEGER_DATA; + static constexpr const char* name = "INTEGER"; +}; +template <> +struct DType { + static constexpr int value = DOUBLE_DATA; + static constexpr const char* name = "DOUBLE"; +}; +template <> +struct DType> { + static constexpr int value = COMPLEX_DATA; + static constexpr const char* name = "COMPLEX"; +}; + +inline std::size_t element_count(const std::vector& shape) { + std::size_t n = 1; + for (int d : shape) n *= static_cast(d); + return n; +} + +// --- generic write ---------------------------------------------------------- +// al_write_data takes a non-const void* and int* size but does not mutate +// them for a write; the const_casts keep the caller's data/shape const. +template +al_status_t write_data(int ctx, const char* field, const std::vector& shape, + const std::vector& data) { + const int dim = static_cast(shape.size()); + int* size = dim ? const_cast(shape.data()) : nullptr; + return al_write_data(ctx, field, "", const_cast(data.data()), + DType::value, dim, size); +} + +// --- generic read ----------------------------------------------------------- +// Honors the two ownership regimes the ABI uses (mirrors testlowlevel.cpp): +// dim == 0 : the core writes into the caller's buffer; nothing to free. +// dim >= 1 : the core malloc's the buffer and returns it via *data; the +// caller owns it, so we copy out and free. +// On success out_shape is the shape the core reported (scalar -> empty) and +// out_data holds element_count(out_shape) elements. +template +al_status_t read_data(int ctx, const char* field, int expected_rank, + std::vector* out_shape, std::vector* out_data) { + int size[MAXDIM] = {0}; + + if (expected_rank == 0) { + T scalar{}; + void* buf = &scalar; + al_status_t s = + al_read_data(ctx, field, "", &buf, DType::value, 0, size); + if (s.code == 0) { + out_shape->clear(); + out_data->assign(1, scalar); + } + return s; + } + + void* buf = nullptr; + al_status_t s = al_read_data(ctx, field, "", &buf, DType::value, + expected_rank, size); + if (s.code == 0 && buf != nullptr) { + out_shape->assign(size, size + expected_rank); + const std::size_t n = element_count(*out_shape); + T* p = static_cast(buf); + out_data->assign(p, p + n); + } + free(buf); + return s; +} + +} // namespace al_contract + +#endif // AL_CONTRACT_ABI_H diff --git a/tests/contract/equilibrium_seed.h b/fixtures/equilibrium_seed.h similarity index 99% rename from tests/contract/equilibrium_seed.h rename to fixtures/equilibrium_seed.h index 07fa60d5..182ba10f 100644 --- a/tests/contract/equilibrium_seed.h +++ b/fixtures/equilibrium_seed.h @@ -38,7 +38,7 @@ #ifndef AL_CONTRACT_EQUILIBRIUM_SEED_H #define AL_CONTRACT_EQUILIBRIUM_SEED_H -#include "al_contract.h" +#include "al_contract_abi.h" #include #include diff --git a/fixtures/fixture_smoke.cpp b/fixtures/fixture_smoke.cpp new file mode 100644 index 00000000..dddd0aef --- /dev/null +++ b/fixtures/fixture_smoke.cpp @@ -0,0 +1,77 @@ +// Throwaway proof (issue #51 acceptance criterion 3): a minimal translation +// unit living outside tests/ that includes equilibrium_seed.h, builds, and +// exercises it against a real backend through the public C ABI only -- no +// GoogleTest, no AL_BUILD_TESTS. Demonstrates that the fixture extracted by +// this issue is genuinely linkable from a target like the future +// benchmarks/ tree (PRD #50), not just from tests/contract/. +// +// Uses the always-on, zero-I/O Memory backend so this needs no on-disk setup +// and no optional backend enabled. + +#include "equilibrium_seed.h" + +#include +#include + +#include +#include +#include +#include + +int main() { + // The Memory backend stores nothing on disk, but URI construction still + // resolves the legacy "user" field as a base directory, so it must exist + // (mirrors tests/contract/al_contract.h's TempBase for the on-disk cases). + const std::filesystem::path base = + std::filesystem::temp_directory_path() / "al_contract_fixtures_smoke"; + std::filesystem::create_directories(base); + const std::string base_str = base.string(); + + char* uri = nullptr; + al_status_t s = al_build_uri_from_legacy_parameters( + MEMORY_BACKEND, /*pulse=*/1, /*run=*/0, /*user=*/base_str.c_str(), + /*tokamak=*/"test", /*version=*/"3", /*options=*/"", &uri); + if (s.code != 0) { + std::fprintf(stderr, "al_build_uri_from_legacy_parameters failed: %s\n", + s.message); + return 1; + } + + int pulse_ctx = -1; + s = al_begin_dataentry_action(uri, FORCE_CREATE_PULSE, &pulse_ctx); + free(uri); + if (s.code != 0) { + std::fprintf(stderr, "al_begin_dataentry_action failed: %s\n", s.message); + return 1; + } + + s = equilibrium_seed::write(pulse_ctx); + if (s.code != 0) { + std::fprintf(stderr, "equilibrium_seed::write failed: %s\n", s.message); + return 1; + } + + uint64_t observed_hash = 0; + s = equilibrium_seed::read_and_hash(pulse_ctx, &observed_hash); + if (s.code != 0) { + std::fprintf(stderr, "equilibrium_seed::read_and_hash failed: %s\n", + s.message); + return 1; + } + + al_close_pulse(pulse_ctx, CLOSE_PULSE); + + if (observed_hash != equilibrium_seed::expected_hash()) { + std::fprintf(stderr, + "structural hash mismatch: observed=%llu expected=%llu\n", + static_cast(observed_hash), + static_cast( + equilibrium_seed::expected_hash())); + return 1; + } + + std::printf("equilibrium seed round-tripped through the Memory backend " + "outside the test tree: hash=%llu\n", + static_cast(observed_hash)); + return 0; +} diff --git a/test b/test index 39e43a64..6c5c6444 100755 --- a/test +++ b/test @@ -20,6 +20,11 @@ # Boost::filesystem installed). AL_BACKEND_MDSPLUS/UDA default OFF, so their # contract tiers compile out and this ctest runs only the common-backend tests. # +# Flags live in CMakePresets.json, not here: every tier delegates to the same +# `tests` / `mdsplus` presets CI runs, so this script only adds orchestration +# (Docker images, bind mounts, the summary table) on top — never a second copy +# of the configure line. +# # The mdsplus/uda backing images are built automatically on first use and reused # after that (rebuild by hand with `docker build`, or `docker image rm` to # force it). Both legs bind-mount the repo at /workspace, so they always @@ -102,14 +107,13 @@ print_summary() { run_common() { section "COMMON" "always-on backends (HDF5, memory, ASCII, flexbuffers) — host-native" - # No -G so an existing build/ keeps its generator; a fresh one takes CMake's - # default. getconf is portable across macOS and Linux. The && chain returns - # non-zero on the first failing step (configure/build/ctest). - local jobs; jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4) - cmake -S "$REPO_ROOT" -B "$REPO_ROOT/build" \ - -DAL_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release \ - && cmake --build "$REPO_ROOT/build" -j"$jobs" \ - && ctest --test-dir "$REPO_ROOT/build" --output-on-failure + # The `tests` presets are the same entry points as CI's blocking leg + # (contract-tests.yml): RelWithDebInfo, Ninja, no Python bindings, build/. + # The && chain returns non-zero on the first failing step. + cd "$REPO_ROOT" \ + && cmake --preset tests \ + && cmake --build --preset tests \ + && ctest --preset tests } @@ -118,15 +122,15 @@ run_common() { run_mdsplus() { section "MDS-PLUS" "Docker: bake DD-4.1.1 model tree, build, ctest -L mdsplus" ensure_image imas-core-mdsplus:dev "$REPO_ROOT/docker/mdsplus" + # The `mdsplus` presets carry the backend/model/test flags, the + # build-mdsplus binary dir, the tier filter and MDSPLUS_MODELS_PATH; only + # the DD pin is passed here (mdsplus-contract.yml pins its own the same way). docker run --rm --platform "$PLATFORM" \ -v "$REPO_ROOT":/workspace -w /workspace \ - -e MDSPLUS_MODELS_PATH=/workspace/build-mdsplus/models/mdsplus \ imas-core-mdsplus:dev bash -c ' - cmake -B build-mdsplus -G Ninja \ - -DAL_BACKEND_MDSPLUS=ON -DAL_BUILD_MDSPLUS_MODELS=ON \ - -DAL_BUILD_TESTS=ON -DDD_VERSION=4.1.1 -DCMAKE_BUILD_TYPE=Release && - cmake --build build-mdsplus -j"$(nproc)" && - cd build-mdsplus/tests/contract && ctest -L mdsplus --output-on-failure + cmake --preset mdsplus -DDD_VERSION=4.1.1 && + cmake --build --preset mdsplus && + ctest --preset mdsplus ' } diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..b7837e6b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,90 @@ +# Running the tests + +Everything here is driven through the presets in [`CMakePresets.json`](../CMakePresets.json) +— the same entry points CI uses, so a local run reproduces a CI leg exactly. + +```sh +cmake --preset tests # configure (AL_BUILD_TESTS=ON, no Python bindings) +cmake --build --preset tests # build libal + test binaries +ctest --preset unit # fast hermetic inner loop (< 1 s) +ctest --preset tests # everything the test build registers +``` + +For the tiers that need a Docker backend (MDSplus, UDA), the [`./test`](../test) +dispatcher at the repo root wraps the image build + `docker run` recipes behind +one verb and delegates all cmake/ctest flags to these same presets: + +```sh +./test common # host-native: the three preset commands above +./test mdsplus # Docker: bake the DD model tree, run the mdsplus tier +./test uda # Docker: assemble the UDA reference stack, run the uda tier +./test all # every tier + a PASS/FAIL summary table +``` + +## What lives where + +| Suite | Location | Framework | Purpose | +|---|---|---|---| +| Smoke tests | `tests/testlowlevel.cpp`, `tests/testlowlevel_c.c` | none (plain executables) | End-to-end C ABI exercise; the `.c` build doubles as a compile canary that `al_lowlevel.h` stays C-compatible. | +| Contract suite | `tests/contract/` | GoogleTest + CTest | Pins the behavior of the public C ABI (`al_lowlevel.h` + `al_const.h`) capability by capability. The *why* is [`TEST_STRATEGY.md`](../TEST_STRATEGY.md); the coverage ledger is [`tests/contract/TRACEABILITY.md`](contract/TRACEABILITY.md). | +| Python binding tests | `python/tests/` | pytest | Bindings-level checks (import, exceptions, `imasdef` constants, HDF5 permissions). | + +## Test tiers (CTest labels) + +Every contract test carries exactly one tier, assigned post-discovery by +`tests/contract/assign_contract_labels.cmake.in` (the tier definitions live +there) and verified disjoint by `tests/contract/check_label_partition.sh`. + +| `ctest --preset …` | Label | What runs | Needs | +|---|---|---|---| +| `unit` | `unit` | Pure lookups (`Introspection.*`), the in-RAM Memory backend matrix, plugin registry lifecycle + ABI death tests, URI ownership | Nothing — hermetic, no disk | +| `integration` | `integration` | HDF5 / ASCII / Flexbuffers round trips on disk | Disk only | +| `tests` / `tests-debug` | *(all)* | Contract suite + smoke tests, in the build mode CI's blocking legs use (both modes matter — issue #32) | — | +| `sanitize` | `contract` | The suite under ASan/UBSan (`AL_CONTRACT_SANITIZE=ON`), excluding `Death` and `CurrentlyCorrupts` cases — see the comments in `sanitizers.yml` for why | Linux for leak detection; on macOS prepend `ASAN_OPTIONS=detect_leaks=0` | +| `mdsplus` | `mdsplus` | MDSplus tier against the DD model tree the `mdsplus` configure preset bakes | MDSplus installed; configure downloads the DD (`-DDD_VERSION=` to pin) | +| *(no preset)* | `uda` | UDA tier against the pinned reference stack | `./test uda` (or `docker/uda/run.sh` inside the container — see `docker/uda/README.md`); there is deliberately no host preset | + +Each test preset (except the run-everything ones' smoke tests) fails on an +empty selection (`noTestsAction: error`), so a broken label partition can't +pass silently. + +For a one-off finer cut, plain CTest still works from the build directory: +`ctest -R Introspection`, `ctest -L integration -E Ascii`, etc. + +## Reading the output + +The suite encodes its own status vocabulary (full definitions in the +[TRACEABILITY.md preamble](contract/TRACEABILITY.md)): + +- **`DISABLED_…` test + a paired `CurrentlyCorrupts`/tripwire test** — an + *xfail*: the disabled test asserts the *correct* behavior of a known defect, + the tripwire pins today's wrong behavior. The tripwire going red means + someone fixed the defect — flip the pair, don't "fix the test". +- **`GTEST_SKIP()`** — either a legitimate backend *divergence* (the test + shape doesn't transfer to that backend) or an unconfigured optional backend + (e.g. `MDSPLUS_MODELS_PATH` / `UDA_HOST` unset). The skip message says which. + +## CI legs + +| Workflow | Preset(s) | Gate | +|---|---|---| +| `contract-tests.yml` | `tests`, `tests-debug` | **Blocking** on every PR | +| `sanitizers.yml` | `sanitize` | Informational (D7) | +| `mdsplus-contract.yml` | `mdsplus` (in Docker, DD pinned in the workflow) | Informational | +| `uda-contract.yml` | — (`docker/uda/run.sh`) | Informational | + +## Python tests + +```sh +pip install .[test] +pytest # testpaths is rooted at python/tests in pyproject.toml +``` + +## Fixture tools + +Two build-gated helper executables prepare states that must *not* be reachable +through the public C ABI (out-of-band file/tree surgery); the tests invoke +them as subprocesses: `hdf5_fixture_tool.cpp` (rewrites the stored +HDF5 backend version) and `mdsplus_fixture_tool.cpp` (rewrites the model +tree's version node). They are fixture *preparation*, never assertion — the +contract suite asserts only through the C ABI. diff --git a/tests/contract/CMakeLists.txt b/tests/contract/CMakeLists.txt index 7f338d9e..b22a72bf 100644 --- a/tests/contract/CMakeLists.txt +++ b/tests/contract/CMakeLists.txt @@ -85,7 +85,7 @@ add_executable(contract_tests test_uda_unique_surface.cpp ) -target_link_libraries(contract_tests PRIVATE al GTest::gtest GTest::gtest_main) +target_link_libraries(contract_tests PRIVATE al al_contract_fixtures GTest::gtest GTest::gtest_main) add_dependencies(contract_tests al) target_compile_features(contract_tests PRIVATE cxx_std_17) diff --git a/tests/contract/TRACEABILITY.md b/tests/contract/TRACEABILITY.md index 86519bed..9a59e6e7 100644 --- a/tests/contract/TRACEABILITY.md +++ b/tests/contract/TRACEABILITY.md @@ -140,7 +140,9 @@ genuine defect (expected-fail). The deterministic, in-repo single-version equilibrium-seed generator (a scalar, a timebase-carrying 2-D array, and a constraints AOS — no committed binary blobs; oracle = a content hash recomputed from the generator functions -themselves, `tests/contract/equilibrium_seed.h`), round-tripped on the three +themselves, `fixtures/equilibrium_seed.h` — moved outside tests/contract/ by +issue #51 so a future benchmarks/ target can link it directly), round-tripped +on the three backends where AOS content is actually readable back within a session. | Capability | Inventory ref | Unit | Integ. | Test(s) | Status | diff --git a/tests/contract/al_contract.h b/tests/contract/al_contract.h index 71d44efb..616a5b55 100644 --- a/tests/contract/al_contract.h +++ b/tests/contract/al_contract.h @@ -12,6 +12,8 @@ #ifndef AL_CONTRACT_H #define AL_CONTRACT_H +#include "al_contract_abi.h" + #include #include @@ -20,8 +22,6 @@ #include #include #include -#include -#include #include #include #include @@ -345,30 +345,12 @@ inline al_status_t read_char_array(int ctx, const char* field, // itself the proof of the version-free property. This block generalizes the // two hand-written scalar/char helpers above to the whole // datatype × shape space. - -// --- datatype descriptor: element C++ type -> ABI datatype id + label ------- -template -struct DType; -template <> -struct DType { - static constexpr int value = CHAR_DATA; - static constexpr const char* name = "CHAR"; -}; -template <> -struct DType { - static constexpr int value = INTEGER_DATA; - static constexpr const char* name = "INTEGER"; -}; -template <> -struct DType { - static constexpr int value = DOUBLE_DATA; - static constexpr const char* name = "DOUBLE"; -}; -template <> -struct DType> { - static constexpr int value = COMPLEX_DATA; - static constexpr const char* name = "COMPLEX"; -}; +// +// DType/write_data/read_data/element_count (the subset equilibrium_seed.h also +// needs) live in al_contract_abi.h (issue #51), included above, so a build +// target outside tests/contract/ can link them without GoogleTest. The +// synthetic-buffer generator and its shape helper, used exclusively by this +// suite's own matrix tests, stay here. // The core's "this value means absent" sentinels (src/al_lowlevel.cpp:41-44). // A *scalar* write whose value equals its sentinel is silently dropped by @@ -420,12 +402,6 @@ inline std::vector shape_for_rank(int rank) { return s; } -inline std::size_t element_count(const std::vector& shape) { - std::size_t n = 1; - for (int d : shape) n *= static_cast(d); - return n; -} - template std::vector synth_buffer(std::size_t n) { std::vector v; @@ -434,55 +410,6 @@ std::vector synth_buffer(std::size_t n) { return v; } -// --- generic write ---------------------------------------------------------- -// al_write_data takes a non-const void* and int* size but does not mutate -// them for a write; the const_casts keep the caller's data/shape const. -template -al_status_t write_data(int ctx, const char* field, const std::vector& shape, - const std::vector& data) { - const int dim = static_cast(shape.size()); - int* size = dim ? const_cast(shape.data()) : nullptr; - return al_write_data(ctx, field, "", const_cast(data.data()), - DType::value, dim, size); -} - -// --- generic read ----------------------------------------------------------- -// Honors the two ownership regimes the ABI uses (mirrors testlowlevel.cpp): -// dim == 0 : the core writes into the caller's buffer; nothing to free. -// dim >= 1 : the core malloc's the buffer and returns it via *data; the -// caller owns it, so we copy out and free. -// On success out_shape is the shape the core reported (scalar -> empty) and -// out_data holds element_count(out_shape) elements. -template -al_status_t read_data(int ctx, const char* field, int expected_rank, - std::vector* out_shape, std::vector* out_data) { - int size[MAXDIM] = {0}; - - if (expected_rank == 0) { - T scalar{}; - void* buf = &scalar; - al_status_t s = - al_read_data(ctx, field, "", &buf, DType::value, 0, size); - if (s.code == 0) { - out_shape->clear(); - out_data->assign(1, scalar); - } - return s; - } - - void* buf = nullptr; - al_status_t s = al_read_data(ctx, field, "", &buf, DType::value, - expected_rank, size); - if (s.code == 0 && buf != nullptr) { - out_shape->assign(size, size + expected_rank); - const std::size_t n = element_count(*out_shape); - T* p = static_cast(buf); - out_data->assign(p, p + n); - } - free(buf); - return s; -} - } // namespace al_contract #endif // AL_CONTRACT_H diff --git a/tests/contract/test_uda.cpp b/tests/contract/test_uda.cpp index d9a5c8de..af862b88 100644 --- a/tests/contract/test_uda.cpp +++ b/tests/contract/test_uda.cpp @@ -115,7 +115,7 @@ TEST_F(UdaSmokeRoundTrip, ScalarSeededViaHdf5ReadsBackThroughUda) { // // Broadens the tracer bullet above from one scalar to the full equilibrium-seed // composite shape (static scalar + top-level timebase array + a real -// time_slice/profiles_1d/constraints AOS, tests/contract/equilibrium_seed.h — +// time_slice/profiles_1d/constraints AOS, fixtures/equilibrium_seed.h — // issues #4/#33/D5), reusing the seed's own generator and structural hash // oracle unchanged: seed via HDF5 (fixture setup, same role the baked model // tree plays for MDSplus), reopen the identical path through the UDA backend in