From b3157e24f86921b800ae92be76bfb5122c2e0956 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:33:02 -0700 Subject: [PATCH 01/12] [Claude] Split the install scripts and add the Axom, Conduit and HDF5 builds build_functions.sh had grown to ~600 lines. It is now a small loader that sources four files, so the existing unix_*_install.sh entry points work unchanged: - build_helpers.sh: run_with_log, clone_if_missing, sync_submodules, prepare_build_dir - build_functions_common.sh: BLT, CAMP, RAJA, Umpire, CHAI - build_functions_mfem.sh: Hypre, METIS, MFEM - build_functions_exaconstit.sh: SNLS, ExaCMech, HDF5, Conduit, Axom, ExaConstit and build_all_dependencies Every RADIUSS-stack package now shares a single BLT checkout (v0.7.2) through BLT_SOURCE_DIR. SNLS (v0.4.4) is built on its own with the batch solvers enabled, and ExaCMech is built against it. The scripts also build the I/O stack the upcoming checkpoint/restart support uses: HDF5 1.14.6 (serial, C only; the 2.x series has caused CMake problems downstream), Conduit v0.9.7 with MPI and HDF5, and Axom v0.15.0 with Sidre. All versions live in dependency_versions.sh, and preflight_checks.sh prints them in the build summary. The CPU configs (cpu_mac_config.sh, cpu_intel_config.sh) now build CHAI with CHAI_ENABLE_MANAGED_PTR=ON. SNLS and ExaCMech use chai::managed_ptr whenever SNLS is built with the RAJA Portability Suite, so ExaCMech failed to build against a CPU CHAI built without it. On the CMake side, Axom is an optional TPL: passing AXOM_DIR finds Axom's package config and sets ENABLE_AXOM. A stub `lumberjack` target works around a dangling reference in Axom's exported axom::slic target. Nothing links against Axom yet. doc/install.md describes the new script layout, versions, build order and directories, and the manual build steps. developers_guide.md lists Axom as an optional dependency. The full dependency stack was built with these scripts on macOS (CPU). On that machine SNLS installs its CMake package to share/snls/cmake and Umpire to lib rather than lib64, so the SNLS_DIR and UMPIRE_DIR paths the scripts pass to ExaConstit had to be adjusted by hand there. --- .../SetupThirdPartyLibraries.cmake | 38 ++ developers_guide.md | 1 + doc/install.md | 74 ++- scripts/install/common/build_functions.sh | 587 +----------------- .../install/common/build_functions_common.sh | 280 +++++++++ .../common/build_functions_exaconstit.sh | 455 ++++++++++++++ .../install/common/build_functions_mfem.sh | 145 +++++ scripts/install/common/build_helpers.sh | 65 ++ scripts/install/common/dependency_versions.sh | 51 +- scripts/install/common/preflight_checks.sh | 35 +- scripts/install/configs/cpu_intel_config.sh | 6 +- scripts/install/configs/cpu_mac_config.sh | 6 +- 12 files changed, 1138 insertions(+), 605 deletions(-) create mode 100644 scripts/install/common/build_functions_common.sh create mode 100644 scripts/install/common/build_functions_exaconstit.sh create mode 100644 scripts/install/common/build_functions_mfem.sh create mode 100644 scripts/install/common/build_helpers.sh diff --git a/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake b/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake index bf0df071..275bdb11 100644 --- a/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake +++ b/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake @@ -9,6 +9,7 @@ set(_tpls exacmech mfem caliper + axom threads) foreach(_tpl ${_tpls}) @@ -122,6 +123,43 @@ if(SNLS_USE_RAJA_PORT_SUITE) endif() endif() # End SNLS_USE_RAJA_PORT_SUITE check +################################ +# Axom (optional) +################################ +# Axom installs a proper CMake package config (axom-config.cmake under +# ${AXOM_DIR}/lib/cmake/axom). find_package CONFIG mode picks it up +# automatically and imports the roll-up `axom` target plus per-component +# targets (axom::core, axom::sidre, axom::slic, ...). We consume the +# roll-up target so whatever components Axom was built with come along +# transitively -- including sidre and its Conduit / HDF5 dependencies, +# which axom-config.cmake locates from the paths Axom was built against. + +if (DEFINED AXOM_DIR) + find_package(axom REQUIRED CONFIG + NO_DEFAULT_PATH + PATHS ${AXOM_DIR}) + if (axom_FOUND) + # ---- Workaround for upstream Axom export bug ---- + # axom::slic's INTERFACE_LINK_LIBRARIES contains a bare 'lumberjack' + # entry inherited from BLT's internal target tracking when Axom is + # built with AXOM_ENABLE_LUMBERJACK=ON. Lumberjack is not in + # AXOM_COMPONENTS_ENABLED (it's a feature folded into slic, not a + # component built as its own library), so the reference is dangling. + # Without a stub here, every consumer of axom::slic gets -llumberjack + # on its link line and the linker fails to find it. + if (NOT TARGET lumberjack) + add_library(lumberjack INTERFACE IMPORTED) + endif() + option(ENABLE_AXOM "Enable Axom" ON) + message(STATUS "Found Axom: ${AXOM_DIR}") + else() + message(FATAL_ERROR "Unable to find Axom with given path ${AXOM_DIR}") + endif() +else() + message(STATUS "Axom support disabled") +endif() + + ################################ # Caliper ################################ diff --git a/developers_guide.md b/developers_guide.md index 5f2880c4..239763ae 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -67,6 +67,7 @@ For detailed installation instructions, refer to the build scripts in `scripts/i **Optional Dependencies:** - **ADIOS2**: (MFEM-based) High-performance I/O for visualization - **Caliper**: Performance profiling +- **Axom** (with Sidre, Conduit and HDF5): Checkpoint/restart support, enabled with `-DAXOM_DIR=/lib/cmake` ### Basic Build Process ```bash diff --git a/doc/install.md b/doc/install.md index d0a9b68f..06b770b4 100644 --- a/doc/install.md +++ b/doc/install.md @@ -80,7 +80,11 @@ scripts/install/ ├── common/ │ ├── dependency_versions.sh # Centralized version control │ ├── preflight_checks.sh # Validation and utilities -│ └── build_functions.sh # Shared build logic +│ ├── build_functions.sh # Loader that sources the files below +│ ├── build_helpers.sh # Logging, cloning, build-dir helpers +│ ├── build_functions_common.sh # BLT, CAMP, RAJA, Umpire, CHAI +│ ├── build_functions_mfem.sh # Hypre, METIS, MFEM +│ └── build_functions_exaconstit.sh # SNLS, ExaCMech, HDF5, Conduit, Axom, ExaConstit ├── configs/ │ ├── cpu_intel_config.sh # Intel compiler configuration │ ├── cpu_mac_config.sh # macOS configuration @@ -319,12 +323,23 @@ All dependency versions are centralized in `common/dependency_versions.sh`: code ExaConstit/scripts/install/common/dependency_versions.sh ``` ```bash +# Build infrastructure (shared by every RADIUSS-stack package via BLT_SOURCE_DIR) +export BLT_VER="v0.7.2" + # Portability libraries export CAMP_VER="v2025.12.0" export RAJA_VER="v2025.12.2" export UMPIRE_VER="v2025.12.0" export CHAI_VER="v2025.12.0" +# Standalone SNLS (batch solvers enabled) +export SNLS_VER="v0.4.4" + +# Restart / checkpoint I/O (Axom Sidre and its dependencies) +export HDF5_VER="hdf5_1.14.6" # Stay on the 1.14 series +export CONDUIT_VER="v0.9.7" +export AXOM_VER="v0.14.0" + # Material models export EXACMECH_REPO="https://github.com/LLNL/ExaCMech.git" export EXACMECH_BRANCH="v0.4.3" # Tag or branch to check out @@ -453,18 +468,31 @@ your_build_directory/ ├── camp/ │ ├── build_${BUILD_SUFFIX}/ # Build artifacts │ └── install_${BUILD_SUFFIX}/ # Installed library +├── blt/ # Shared BLT checkout (no build step) ├── RAJA/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ -├── Umpire/ # GPU builds only +├── Umpire/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── CHAI/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ -├── CHAI/ # GPU builds only +├── SNLS/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ ├── ExaCMech/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ +├── hdf5/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── conduit/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── axom/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ ├── hypre/ │ ├── build_${BUILD_SUFFIX}/ │ └── src/hypre_${BUILD_SUFFIX}/ # Installed library @@ -499,10 +527,8 @@ less my_raja_build # View the build log ### **Disk Space Requirements** Typical disk space usage: -- **CPU build**: ~1 GB total -- **GPU build (CUDA/HIP)**: ~2 GB total - - Includes additional Umpire and CHAI libraries - - GPU architectures add to binary sizes +- **CPU build**: a few GB total (HDF5, Conduit, and Axom checkouts make up a large share) +- **GPU build (CUDA/HIP)**: larger still, since GPU architectures add to binary sizes --- @@ -740,16 +766,21 @@ If you prefer to build manually or need more control over the build process: ### **Prerequisites** You'll need to manually build all dependencies first: -1. **CAMP** (v2025.12.0) -2. **RAJA** (v2025.12.2) -3. **Umpire** (v2025.12.0) - GPU builds only -4. **CHAI** (v2025.12.0) - GPU builds only -5. **ExaCMech** (v0.4.3) +1. **BLT** (v0.7.2) - shared by every package below via `-DBLT_SOURCE_DIR` +2. **CAMP** (v2025.12.0) +3. **RAJA** (v2025.12.2) +4. **Umpire** (v2025.12.0) +5. **CHAI** (v2025.12.0) 6. **Hypre** (v3.1.0) 7. **METIS** (5.1.0) 8. **MFEM** (`exaconstit-dev` branch of https://github.com/rcarson3/mfem; v0.10.0 was tested with commit `4a13eddcf5e6dcbccd4ba67f3aa703eb22a0fb40`) +9. **SNLS** (v0.4.4) - built with `USE_BATCH_SOLVERS=ON` +10. **ExaCMech** (v0.4.3) - built against the standalone SNLS via `SNLS_DIR` +11. **HDF5** (1.14.6) - serial, C library only +12. **Conduit** (v0.9.7) - with MPI and HDF5 +13. **Axom** (v0.14.0) - with Sidre (optional for ExaConstit; enabled when `AXOM_DIR` is given) -See `scripts/install/common/build_functions.sh` for the exact build commands and CMake options. +See the `scripts/install/common/build_functions_*.sh` files for the exact build commands and CMake options. ### **Building ExaConstit** @@ -772,11 +803,16 @@ cmake .. \ -DENABLE_OPENMP=OFF \ -DENABLE_FORTRAN=OFF \ -DPYTHON_EXECUTABLE=/usr/bin/python3 \ + -DBLT_SOURCE_DIR=${BLT_DIR} \ -DMFEM_DIR=${MFEM_INSTALL_DIR}/lib/cmake/mfem/ \ -DECMECH_DIR=${EXACMECH_INSTALL_DIR}/ \ - -DSNLS_DIR=${EXACMECH_INSTALL_DIR}/ \ + -DSNLS_DIR=${SNLS_INSTALL_DIR}/lib/cmake/snls/ \ + -DAXOM_DIR=${AXOM_INSTALL_DIR}/lib/cmake/ \ -DRAJA_DIR=${RAJA_INSTALL_DIR}/lib/cmake/raja/ \ -Dcamp_DIR=${CAMP_INSTALL_DIR}/lib/cmake/camp/ \ + -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ + -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ + -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai \ -DCMAKE_BUILD_TYPE=Release # 4. Build @@ -796,10 +832,7 @@ cmake .. \ -DCMAKE_CUDA_COMPILER=${CUDA_ROOT}/bin/nvcc \ -DCMAKE_CUDA_HOST_COMPILER=${HOST_CXX_COMPILER} \ -DCMAKE_CUDA_ARCHITECTURES=80 \ - -DENABLE_CUDA=ON \ - -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ - -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ - -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai + -DENABLE_CUDA=ON ``` For HIP builds, add: @@ -809,10 +842,7 @@ cmake .. \ -DCMAKE_CXX_COMPILER=${ROCM_ROOT}/bin/amdclang++ \ -DCMAKE_HIP_COMPILER=${ROCM_ROOT}/bin/amdclang++ \ -DCMAKE_HIP_ARCHITECTURES=gfx942 \ - -DENABLE_HIP=ON \ - -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ - -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ - -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai + -DENABLE_HIP=ON ``` --- diff --git a/scripts/install/common/build_functions.sh b/scripts/install/common/build_functions.sh index c7e80015..b56ed605 100644 --- a/scripts/install/common/build_functions.sh +++ b/scripts/install/common/build_functions.sh @@ -1,562 +1,27 @@ #!/usr/bin/env bash -# Common build functions for all ExaConstit dependencies - -# Logging wrapper -run_with_log() { - local log="$1"; shift - "$@" |& tee "$log" -} - -# Clone repository only if missing, initialize submodules on first clone -clone_if_missing() { - local repo="$1" branch="$2" dest="$3" - if [ ! -d "$dest/.git" ]; then - echo "Cloning ${dest}..." - git clone --branch "$branch" "$repo" "$dest" - cd "$dest" - if [ -f .gitmodules ]; then - git submodule update --init --recursive - fi - cd "$BASE_DIR" - else - echo "${dest} already exists, skipping clone." - fi -} - -# Optional: force submodule sync when explicitly requested -sync_submodules() { - local dest="$1" - if [ "${SYNC_SUBMODULES}" = "ON" ] && [ -f "$dest/.gitmodules" ]; then - echo "Syncing submodules in ${dest}..." - cd "$dest" - git submodule sync --recursive - git submodule update --init --recursive - cd "$BASE_DIR" - fi -} - -# Respect REBUILD flag when preparing build directories -prepare_build_dir() { - local dir="$1" - if [ "${REBUILD}" = "ON" ]; then - mkdir -p "$dir" - rm -rf "$dir"/* - echo "Cleaned build directory: ${dir}" - else - if [ ! -d "$dir" ]; then - mkdir -p "$dir" - echo "Created build directory: ${dir}" - else - echo "Reusing existing build directory: ${dir}" - fi - fi -} - -########################################### -# CAMP -########################################### -build_camp() { - echo "==========================================" - echo "Building CAMP" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/camp.git" "${CAMP_VER}" "${BASE_DIR}/camp" - sync_submodules "${BASE_DIR}/camp" - - prepare_build_dir "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - ) - fi - - run_with_log my_camp_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_camp_build make -j "${MAKE_JOBS}" - run_with_log my_camp_install make install - - CAMP_ROOT="${BASE_DIR}/camp/install_${BUILD_SUFFIX}" - export CAMP_ROOT - echo "CAMP installed to: ${CAMP_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# RAJA -########################################### -build_raja() { - echo "==========================================" - echo "Building RAJA" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/RAJA.git" "${RAJA_VER}" "${BASE_DIR}/RAJA" - sync_submodules "${BASE_DIR}/RAJA" - - prepare_build_dir "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DRAJA_ENABLE_TESTS=OFF - -DRAJA_ENABLE_EXAMPLES=OFF - -DRAJA_ENABLE_BENCHMARKS=OFF - -DRAJA_ENABLE_REPRODUCERS=OFF - -DRAJA_ENABLE_EXERCISES=OFF - -DRAJA_ENABLE_VECTORIZATION=OFF - -DRAJA_ENABLE_DOCUMENTATION=OFF - -DRAJA_USE_DOUBLE=ON - -DRAJA_TIMER=chrono - -DENABLE_OPENMP="${OPENMP_ON}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - ) - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DRAJA_USE_BARE_PTR=ON - ) - fi - fi - - run_with_log my_raja_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_raja_build make -j "${MAKE_JOBS}" - run_with_log my_raja_install make install - - RAJA_ROOT="${BASE_DIR}/RAJA/install_${BUILD_SUFFIX}" - export RAJA_ROOT - echo "RAJA installed to: ${RAJA_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# Umpire (GPU only) -########################################### -build_umpire() { - if [ "${BUILD_TYPE}" = "cpu" ]; then - echo "Skipping Umpire (not needed for CPU builds)" - return 0 - fi - - echo "==========================================" - echo "Building Umpire" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/Umpire.git" "${UMPIRE_VER}" "${BASE_DIR}/Umpire" - sync_submodules "${BASE_DIR}/Umpire" - - prepare_build_dir "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_MPI=OFF - -DUMPIRE_ENABLE_C=OFF - -DENABLE_FORTRAN=OFF - -DENABLE_GMOCK=OFF - -DUMPIRE_ENABLE_IPC_SHARED_MEMORY=OFF - -DUMPIRE_ENABLE_TOOLS=ON - -DUMPIRE_ENABLE_BACKTRACE=ON - -DUMPIRE_ENABLE_BACKTRACE_SYMBOLS=ON - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - run_with_log my_umpire_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_umpire_build make -j "${MAKE_JOBS}" - run_with_log my_umpire_install make install - - UMPIRE_ROOT="${BASE_DIR}/Umpire/install_${BUILD_SUFFIX}" - export UMPIRE_ROOT - - # Find fmt directory - FMT_DIR_CMAKE=$(find "${UMPIRE_ROOT}" -name 'fmtConfig.cmake' -print -quit || true) - if [ -n "${FMT_DIR_CMAKE}" ]; then - FMT_DIR=$(dirname "${FMT_DIR_CMAKE}") - else - FMT_DIR="${UMPIRE_ROOT}" - fi - export FMT_DIR - - echo "Umpire installed to: ${UMPIRE_ROOT}" - echo "fmt found at: ${FMT_DIR}" - cd "${BASE_DIR}" -} - -########################################### -# CHAI (GPU only) -########################################### -build_chai() { - if [ "${BUILD_TYPE}" = "cpu" ]; then - echo "Skipping CHAI (not needed for CPU builds)" - return 0 - fi - - echo "==========================================" - echo "Building CHAI" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/CHAI.git" "${CHAI_VER}" "${BASE_DIR}/CHAI" - sync_submodules "${BASE_DIR}/CHAI" - - prepare_build_dir "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_EXAMPLES=OFF - -DENABLE_DOCS=OFF - -DENABLE_GMOCK=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_MPI=OFF - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -DCHAI_ENABLE_RAJA_PLUGIN=ON - -DCHAI_ENABLE_RAJA_NESTED_TEST=OFF - -DCHAI_THIN_GPU_ALLOCATE="${CHAI_THIN_GPU_ALLOCATE}" - -DCHAI_ENABLE_PINNED="${CHAI_ENABLE_PINNED}" - -DCHAI_DISABLE_RM="${CHAI_DISABLE_RM}" - -DCHAI_ENABLE_PICK="${CHAI_ENABLE_PICK}" - -DCHAI_DEBUG="${CHAI_DEBUG}" - -DCHAI_ENABLE_GPU_SIMULATION_MODE="${CHAI_ENABLE_GPU_SIMULATION_MODE}" - -DCHAI_ENABLE_UM="${CHAI_ENABLE_UM}" - -DCHAI_ENABLE_MANAGED_PTR="${CHAI_ENABLE_MANAGED_PTR}" - -DCHAI_ENABLE_MANAGED_PTR_ON_GPU="${CHAI_ENABLE_MANAGED_PTR_ON_GPU}" - -Dfmt_DIR="${FMT_DIR}" - -Dumpire_DIR="${UMPIRE_ROOT}" - -DRAJA_DIR="${RAJA_ROOT}" - -Dcamp_DIR="${CAMP_ROOT}" - ) - - run_with_log my_chai_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_chai_build make -j "${MAKE_JOBS}" - run_with_log my_chai_install make install - - CHAI_ROOT="${BASE_DIR}/CHAI/install_${BUILD_SUFFIX}" - export CHAI_ROOT - echo "CHAI installed to: ${CHAI_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# ExaCMech -########################################### -build_exacmech() { - echo "==========================================" - echo "Building ExaCMech" - echo "==========================================" - - clone_if_missing "${EXACMECH_REPO}" "${EXACMECH_BRANCH}" "${BASE_DIR}/ExaCMech" - sync_submodules "${BASE_DIR}/ExaCMech" - - prepare_build_dir "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_MINIAPPS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" - -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -DFMT_DIR="${FMT_DIR}" - -DUMPIRE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" - -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" - ) - fi - - run_with_log my_ecmech_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_ecmech_build make -j "${MAKE_JOBS}" - run_with_log my_ecmech_install make install - - ECMECH_ROOT="${BASE_DIR}/ExaCMech/install_${BUILD_SUFFIX}" - export ECMECH_ROOT - echo "ExaCMech installed to: ${ECMECH_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# Hypre -########################################### -build_hypre() { - echo "==========================================" - echo "Building Hypre" - echo "==========================================" - - if [ ! -d "${BASE_DIR}/hypre" ]; then - git clone https://github.com/hypre-space/hypre.git --branch "${HYPRE_VER}" --single-branch "${BASE_DIR}/hypre" - fi - - prepare_build_dir "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" - - run_with_log my_hypre_config cmake ../src \ - -DCMAKE_INSTALL_PREFIX=../src/hypre_${BUILD_SUFFIX}/ \ - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \ - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ - -DMPI_C_COMPILER="${MPI_C_COMPILER}" \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - - run_with_log my_hypre_build make -j "${MAKE_JOBS}" - run_with_log my_hypre_install make install - - HYPRE_ROOT="${BASE_DIR}/hypre/src/hypre_${BUILD_SUFFIX}" - export HYPRE_ROOT - echo "Hypre installed to: ${HYPRE_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# METIS -########################################### -build_metis() { - echo "==========================================" - echo "Building METIS" - echo "==========================================" - - if [ ! -d "${BASE_DIR}/metis-${METIS_VER}" ]; then - curl -o metis-${METIS_VER}.tar.gz "${METIS_URL}" - tar -xzf metis-${METIS_VER}.tar.gz - rm metis-${METIS_VER}.tar.gz - fi - - cd "${BASE_DIR}/metis-${METIS_VER}" - - # METIS doesn't have a proper incremental build, so always clean - make distclean 2>/dev/null || true - - prepare_build_dir "${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" - - run_with_log my_metis_config make config \ - prefix="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" \ - CC="${CMAKE_C_COMPILER}" \ - CXX="${CMAKE_CXX_COMPILER}" - - run_with_log my_metis_build make -j "${MAKE_JOBS}" - run_with_log my_metis_install make install - - METIS_ROOT="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" - export METIS_ROOT - echo "METIS installed to: ${METIS_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# MFEM -########################################### -build_mfem() { - echo "==========================================" - echo "Building MFEM" - echo "==========================================" - - clone_if_missing "${MFEM_REPO}" "${MFEM_BRANCH}" "${BASE_DIR}/mfem" - # Don't sync submodules for MFEM to preserve local changes - - prepare_build_dir "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DMFEM_USE_MPI=YES - -DMFEM_USE_SIMD=NO - -DMETIS_DIR="${METIS_ROOT}" - -DHYPRE_DIR="${HYPRE_ROOT}" - -DMFEM_USE_RAJA=YES - -DRAJA_DIR="${RAJA_ROOT}" - -DRAJA_REQUIRED_PACKAGES="camp" - -DMFEM_USE_CAMP=ON - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - -DMFEM_USE_OPENMP="${OPENMP_ON}" - -DMFEM_USE_ZLIB=YES - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - ) - - if [ "${BUILD_TYPE}" = "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" - ) - else - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DMFEM_USE_${GPU_BACKEND}=ON - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - ) - - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DCMAKE_CUDA_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CUDA_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_CUDA=ON - ) - elif [ "${GPU_BACKEND}" = "HIP" ]; then - CMAKE_ARGS+=( - -DHIP_ARCH="${MFEM_HIP_ARCHITECTURES}" - -DCMAKE_HIP_ARCHITECTURES="${MFEM_HIP_ARCHITECTURES}" - ) - fi - fi - - run_with_log my_mfem_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_mfem_build make -j "${MAKE_JOBS}" - run_with_log my_mfem_install make install - - MFEM_ROOT="${BASE_DIR}/mfem/install_${BUILD_SUFFIX}" - export MFEM_ROOT - echo "MFEM installed to: ${MFEM_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# ExaConstit -########################################### -build_exaconstit() { - echo "==========================================" - echo "Building ExaConstit" - echo "==========================================" - - clone_if_missing "${EXACONSTIT_REPO}" "${EXACONSTIT_BRANCH}" "${BASE_DIR}/ExaConstit" - sync_submodules "${BASE_DIR}/ExaConstit" - - prepare_build_dir "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" - -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" - -DENABLE_TESTS="${ENABLE_TESTS_EXACONSTIT}" - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_FORTRAN=OFF - -DENABLE_SNLS_V03=ON - -DCMAKE_INSTALL_PREFIX=../install_dir/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DMFEM_DIR="${MFEM_ROOT}/lib/cmake/mfem" - -DECMECH_DIR="${ECMECH_ROOT}" - -DSNLS_DIR="${ECMECH_ROOT}" - -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" - -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" = "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" - ) - else - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DENABLE_${GPU_BACKEND}=ON - -DFMT_DIR="${FMT_DIR}" - -DUMPIRE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" - -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" - ) - - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" - -DBLT_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" - ) - elif [ "${GPU_BACKEND}" = "HIP" ]; then - CMAKE_ARGS+=( - -DCMAKE_HIP_FLAGS="${CMAKE_GPU_FLAGS}" - ) - fi - fi - - run_with_log my_exconstit_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_exconstit_build make -j "${MAKE_JOBS}" - - EXACONSTIT_ROOT="${BASE_DIR}/ExaConstit/install_dir" - export EXACONSTIT_ROOT - echo "==========================================" - echo "ExaConstit build complete!" - echo "Install prefix: ${EXACONSTIT_ROOT}" - echo "==========================================" - cd "${BASE_DIR}" -} - -########################################### -# Main orchestration function -########################################### -build_all_dependencies() { - build_camp - build_raja - build_umpire - build_chai - build_exacmech - build_hypre - build_metis - build_mfem - build_exaconstit -} \ No newline at end of file +# Meta-loader for the ExaConstit build functions. +# +# The build logic was split into a helpers file and three layer files +# grouped by dependency tier; this file simply sources them in +# dependency order so existing entry-point scripts (unix_*_install.sh) +# keep working unchanged. +# +# build_helpers.sh Shared helper functions +# (run_with_log, clone_if_missing, +# sync_submodules, prepare_build_dir). +# build_functions_common.sh BLT, CAMP, RAJA, Umpire, CHAI -- the +# shared portability stack. +# build_functions_mfem.sh Hypre, METIS, MFEM -- the FEM stack. +# build_functions_exaconstit.sh SNLS, ExaCMech, Axom, ExaConstit, +# plus the build_all_dependencies +# orchestrator. + +# Resolve our own location so each file sources its sibling. +_BUILD_FUNCTIONS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source "${_BUILD_FUNCTIONS_DIR}/build_helpers.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_common.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_mfem.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_exaconstit.sh" + +unset _BUILD_FUNCTIONS_DIR diff --git a/scripts/install/common/build_functions_common.sh b/scripts/install/common/build_functions_common.sh new file mode 100644 index 00000000..e0e0cd45 --- /dev/null +++ b/scripts/install/common/build_functions_common.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# Common-stack build functions: BLT, CAMP, RAJA, Umpire, CHAI. +# +# These are the shared portability / utility libraries used by both +# the MFEM stack and the ExaConstit application stack. Helpers live +# in build_helpers.sh; the MFEM-stack and application-stack functions +# live in build_functions_mfem.sh and build_functions_exaconstit.sh +# respectively. +# +# Note: Umpire and CHAI are built on every platform now. The batch +# SNLS solvers depend on the full RAJA Portability Suite, and ExaCMech +# transitively links the same set, so making CHAI/Umpire available on +# CPU keeps the dependency graph uniform across CPU and GPU builds. + +########################################### +# BLT +########################################### +# BLT is a CMake-only build helper (header / macro / module library). +# It has no compile or install step. We clone it once and point every +# downstream LLNL/RADIUSS package at it via -DBLT_SOURCE_DIR=${BLT_ROOT}. +# This keeps every package on the same BLT version regardless of what +# their bundled submodule happens to point at. +build_blt() { + echo "==========================================" + echo "Cloning BLT (${BLT_VER})" + echo "==========================================" + + clone_if_missing "${BLT_REPO}" "${BLT_VER}" "${BASE_DIR}/blt" + + BLT_ROOT="${BASE_DIR}/blt" + export BLT_ROOT + echo "BLT available at: ${BLT_ROOT}" + echo "Downstream packages will consume it via -DBLT_SOURCE_DIR" + cd "${BASE_DIR}" +} + +########################################### +# CAMP +########################################### +build_camp() { + echo "==========================================" + echo "Building CAMP" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/camp.git" "${CAMP_VER}" "${BASE_DIR}/camp" + sync_submodules "${BASE_DIR}/camp" + + prepare_build_dir "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_camp_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_camp_build make -j "${MAKE_JOBS}" + run_with_log my_camp_install make install + + CAMP_ROOT="${BASE_DIR}/camp/install_${BUILD_SUFFIX}" + export CAMP_ROOT + echo "CAMP installed to: ${CAMP_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# RAJA +########################################### +build_raja() { + echo "==========================================" + echo "Building RAJA" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/RAJA.git" "${RAJA_VER}" "${BASE_DIR}/RAJA" + sync_submodules "${BASE_DIR}/RAJA" + + prepare_build_dir "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DRAJA_ENABLE_TESTS=OFF + -DRAJA_ENABLE_EXAMPLES=OFF + -DRAJA_ENABLE_BENCHMARKS=OFF + -DRAJA_ENABLE_REPRODUCERS=OFF + -DRAJA_ENABLE_EXERCISES=OFF + -DRAJA_ENABLE_VECTORIZATION=OFF + -DRAJA_ENABLE_DOCUMENTATION=OFF + -DRAJA_USE_DOUBLE=ON + -DRAJA_TIMER=chrono + -DENABLE_OPENMP="${OPENMP_ON}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DRAJA_USE_BARE_PTR=ON + ) + fi + fi + + run_with_log my_raja_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_raja_build make -j "${MAKE_JOBS}" + run_with_log my_raja_install make install + + RAJA_ROOT="${BASE_DIR}/RAJA/install_${BUILD_SUFFIX}" + export RAJA_ROOT + echo "RAJA installed to: ${RAJA_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Umpire +########################################### +# Built on both CPU and GPU. SNLS's batch solvers depend on Umpire, and +# we want batch solvers available regardless of platform. +build_umpire() { + echo "==========================================" + echo "Building Umpire" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/Umpire.git" "${UMPIRE_VER}" "${BASE_DIR}/Umpire" + sync_submodules "${BASE_DIR}/Umpire" + + prepare_build_dir "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=OFF + -DUMPIRE_ENABLE_C=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_GMOCK=OFF + -DUMPIRE_ENABLE_IPC_SHARED_MEMORY=OFF + -DUMPIRE_ENABLE_TOOLS=ON + -DUMPIRE_ENABLE_BACKTRACE=ON + -DUMPIRE_ENABLE_BACKTRACE_SYMBOLS=ON + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_umpire_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_umpire_build make -j "${MAKE_JOBS}" + run_with_log my_umpire_install make install + + UMPIRE_ROOT="${BASE_DIR}/Umpire/install_${BUILD_SUFFIX}" + export UMPIRE_ROOT + + # Find fmt directory (Umpire vendors fmt and exports a CMake config for it) + FMT_DIR_CMAKE=$(find "${UMPIRE_ROOT}" -name 'fmtConfig.cmake' -print -quit || true) + if [ -n "${FMT_DIR_CMAKE}" ]; then + FMT_DIR=$(dirname "${FMT_DIR_CMAKE}") + else + FMT_DIR="${UMPIRE_ROOT}" + fi + export FMT_DIR + + echo "Umpire installed to: ${UMPIRE_ROOT}" + echo "fmt found at: ${FMT_DIR}" + cd "${BASE_DIR}" +} + +########################################### +# CHAI +########################################### +# Built on both CPU and GPU. SNLS's batch solvers consume CHAI's +# ManagedArray and managed_ptr plumbing; on CPU CHAI's GPU-specific +# knobs (pinned, UM, managed_ptr_on_gpu, etc.) are OFF in the platform +# configs, but CHAI_ENABLE_MANAGED_PTR must stay ON since SNLS and +# ExaCMech use chai::managed_ptr on every platform. +build_chai() { + echo "==========================================" + echo "Building CHAI" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/CHAI.git" "${CHAI_VER}" "${BASE_DIR}/CHAI" + sync_submodules "${BASE_DIR}/CHAI" + + prepare_build_dir "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_EXAMPLES=OFF + -DENABLE_DOCS=OFF + -DENABLE_GMOCK=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=OFF + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DCHAI_ENABLE_RAJA_PLUGIN=ON + -DCHAI_ENABLE_RAJA_NESTED_TEST=OFF + -DCHAI_THIN_GPU_ALLOCATE="${CHAI_THIN_GPU_ALLOCATE}" + -DCHAI_ENABLE_PINNED="${CHAI_ENABLE_PINNED}" + -DCHAI_DISABLE_RM="${CHAI_DISABLE_RM}" + -DCHAI_ENABLE_PICK="${CHAI_ENABLE_PICK}" + -DCHAI_DEBUG="${CHAI_DEBUG}" + -DCHAI_ENABLE_GPU_SIMULATION_MODE="${CHAI_ENABLE_GPU_SIMULATION_MODE}" + -DCHAI_ENABLE_UM="${CHAI_ENABLE_UM}" + -DCHAI_ENABLE_MANAGED_PTR="${CHAI_ENABLE_MANAGED_PTR}" + -DCHAI_ENABLE_MANAGED_PTR_ON_GPU="${CHAI_ENABLE_MANAGED_PTR_ON_GPU}" + -Dfmt_DIR="${FMT_DIR}" + -Dumpire_DIR="${UMPIRE_ROOT}" + -DRAJA_DIR="${RAJA_ROOT}" + -Dcamp_DIR="${CAMP_ROOT}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_chai_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_chai_build make -j "${MAKE_JOBS}" + run_with_log my_chai_install make install + + CHAI_ROOT="${BASE_DIR}/CHAI/install_${BUILD_SUFFIX}" + export CHAI_ROOT + echo "CHAI installed to: ${CHAI_ROOT}" + cd "${BASE_DIR}" +} diff --git a/scripts/install/common/build_functions_exaconstit.sh b/scripts/install/common/build_functions_exaconstit.sh new file mode 100644 index 00000000..324a3482 --- /dev/null +++ b/scripts/install/common/build_functions_exaconstit.sh @@ -0,0 +1,455 @@ +#!/usr/bin/env bash +# ExaConstit application-stack build functions: SNLS, ExaCMech, HDF5, +# Conduit, Axom, and ExaConstit. Also defines the top-level +# build_all_dependencies orchestrator. +# +# Depends on the helpers in build_helpers.sh, the common stack defined +# in build_functions_common.sh (BLT, CAMP, RAJA, Umpire, CHAI), and +# MFEM defined in build_functions_mfem.sh. +# +# Axom lives here rather than in the common stack because it will +# eventually depend on MFEM, which puts it logically downstream of the +# MFEM-stack build file and alongside the other application-tier +# packages. HDF5 and Conduit live here because they are only needed +# for Axom's Sidre component (restart / checkpoint I/O). + +########################################### +# SNLS +########################################### +# Lifted out of ExaCMech and built standalone with the batch-solver +# option always enabled. Batch solvers require the full RAJA +# Portability Suite (RAJA + Umpire + CHAI + camp); since the common +# stack now builds Umpire and CHAI on every platform, this is uniform +# across CPU and GPU. +build_snls() { + echo "==========================================" + echo "Building SNLS" + echo "==========================================" + + clone_if_missing "${SNLS_REPO}" "${SNLS_VER}" "${BASE_DIR}/SNLS" + sync_submodules "${BASE_DIR}/SNLS" + + prepare_build_dir "${BASE_DIR}/SNLS/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/SNLS/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # Batch solvers ON everywhere -> needs the full Portability Suite. + -DUSE_BATCH_SOLVERS=ON + -DUSE_RAJA_ONLY=OFF + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + -DUMPIRE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + -DFMT_DIR="${FMT_DIR}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_snls_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_snls_build make -j "${MAKE_JOBS}" + run_with_log my_snls_install make install + + SNLS_ROOT="${BASE_DIR}/SNLS/install_${BUILD_SUFFIX}" + export SNLS_ROOT + echo "SNLS installed to: ${SNLS_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# ExaCMech +########################################### +# Consumes the standalone SNLS instead of its bundled submodule. +# ExaCMech's CMakeLists auto-sets its internal USE_BUILT_SNLS=ON when +# SNLS_DIR is defined, so we only need to pass SNLS_DIR -- no other +# external-SNLS toggle required. +# +# Because the standalone SNLS is built with USE_BATCH_SOLVERS=ON, it +# pulls CHAI / Umpire / fmt into ExaCMech's link line transitively. +# So FMT_DIR / UMPIRE_DIR / CHAI_DIR are passed unconditionally now, +# regardless of whether ExaCMech itself is being built with GPU support. +build_exacmech() { + echo "==========================================" + echo "Building ExaCMech" + echo "==========================================" + + clone_if_missing "${EXACMECH_REPO}" "${EXACMECH_BRANCH}" "${BASE_DIR}/ExaCMech" + sync_submodules "${BASE_DIR}/ExaCMech" + + prepare_build_dir "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_MINIAPPS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # External SNLS: defining SNLS_DIR is sufficient; ExaCMech sets + # USE_BUILT_SNLS=ON internally when it sees this variable. + -DSNLS_DIR="${SNLS_ROOT}/lib/cmake/snls" + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + # SNLS was built with batch solvers, so ExaCMech needs the full + # Portability Suite resolved transitively even on CPU builds. + -DFMT_DIR="${FMT_DIR}" + -DUMPIRE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_ecmech_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_ecmech_build make -j "${MAKE_JOBS}" + run_with_log my_ecmech_install make install + + ECMECH_ROOT="${BASE_DIR}/ExaCMech/install_${BUILD_SUFFIX}" + export ECMECH_ROOT + echo "ExaCMech installed to: ${ECMECH_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# HDF5 +########################################### +# Serial HDF5 used by Conduit's relay I/O, which is what Sidre's +# sidre_hdf5 protocol writes through. Sidre's parallel I/O (IOManager) +# writes one HDF5 file per rank group itself, so a parallel HDF5 build +# is not needed. Only the C library is built; Fortran / C++ / examples +# and tests are skipped. Tools stay on so h5dump / h5ls are available +# for inspecting restart files. +build_hdf5() { + echo "==========================================" + echo "Building HDF5" + echo "==========================================" + + clone_if_missing "${HDF5_REPO}" "${HDF5_VER}" "${BASE_DIR}/hdf5" + + prepare_build_dir "${BASE_DIR}/hdf5/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/hdf5/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DBUILD_SHARED_LIBS=ON + -DBUILD_STATIC_LIBS=ON + -DBUILD_TESTING=OFF + -DHDF5_ENABLE_PARALLEL=OFF + -DHDF5_BUILD_CPP_LIB=OFF + -DHDF5_BUILD_FORTRAN=OFF + -DHDF5_BUILD_JAVA=OFF + -DHDF5_BUILD_EXAMPLES=OFF + -DHDF5_BUILD_TOOLS=ON + -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF + -DHDF5_ENABLE_SZIP_SUPPORT=OFF + ) + + run_with_log my_hdf5_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_hdf5_build make -j "${MAKE_JOBS}" + run_with_log my_hdf5_install make install + + HDF5_ROOT="${BASE_DIR}/hdf5/install_${BUILD_SUFFIX}" + export HDF5_ROOT + echo "HDF5 installed to: ${HDF5_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Conduit +########################################### +# Hard dependency of Axom's Sidre component. Built with MPI (Axom +# requires conduit_relay_mpi when AXOM_ENABLE_MPI=ON) and HDF5 relay +# support. Conduit expects HDF5_DIR to be the HDF5 install prefix. +# +# Like Axom, Conduit's CMakeLists lives in src/, so the configure step +# points at ../src. +build_conduit() { + echo "==========================================" + echo "Building Conduit" + echo "==========================================" + + clone_if_missing "${CONDUIT_REPO}" "${CONDUIT_VER}" "${BASE_DIR}/conduit" + sync_submodules "${BASE_DIR}/conduit" + + prepare_build_dir "${BASE_DIR}/conduit/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/conduit/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DBLT_CXX_STD="c++${CMAKE_CXX_STANDARD}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DBUILD_SHARED_LIBS=OFF + -DENABLE_TESTS=OFF + -DENABLE_EXAMPLES=OFF + -DENABLE_UTILS=OFF + -DENABLE_DOCS=OFF + -DENABLE_PYTHON=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=ON + -DMPI_C_COMPILER="${MPI_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DHDF5_DIR="${HDF5_ROOT}" + ) + + run_with_log my_conduit_config cmake ../src "${CMAKE_ARGS[@]}" + run_with_log my_conduit_build make -j "${MAKE_JOBS}" + run_with_log my_conduit_install make install + + CONDUIT_ROOT="${BASE_DIR}/conduit/install_${BUILD_SUFFIX}" + export CONDUIT_ROOT + echo "Conduit installed to: ${CONDUIT_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Axom +########################################### +# Built with the core component (always on) plus spin, primal, slam, +# and sidre. Slic is enabled explicitly because spin, sidre and other +# components rely on it for logging. Sidre is what the restart / +# checkpoint logic is built on; it pulls in Conduit (required) and +# HDF5 (for the sidre_hdf5 I/O protocol). +# +# Axom's CMakeLists lives in the src/ subdirectory, so the configure +# step points at ../src rather than ../ like the other packages. +build_axom() { + echo "==========================================" + echo "Building Axom" + echo "==========================================" + + clone_if_missing "${AXOM_REPO}" "${AXOM_VER}" "${BASE_DIR}/axom" + sync_submodules "${BASE_DIR}/axom" + + prepare_build_dir "${BASE_DIR}/axom/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/axom/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DBLT_CXX_STD="c++${CMAKE_CXX_STANDARD}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # Disable everything by default, then turn on what we need. + -DAXOM_ENABLE_ALL_COMPONENTS=OFF + -DAXOM_ENABLE_SPIN=ON + -DAXOM_ENABLE_SLIC=ON + -DAXOM_ENABLE_SIDRE=ON + -DAXOM_ENABLE_INLET=OFF + -DAXOM_ENABLE_KLEE=OFF + -DAXOM_ENABLE_LUMBERJACK=ON + -DAXOM_ENABLE_MINT=OFF + -DAXOM_ENABLE_MIR=OFF + -DAXOM_ENABLE_MULTIMAT=OFF + -DAXOM_ENABLE_PRIMAL=ON + -DAXOM_ENABLE_QUEST=OFF + -DAXOM_ENABLE_SLAM=ON + # Build settings -- skip everything that isn't the library itself. + -DAXOM_ENABLE_TESTS=OFF + -DAXOM_ENABLE_EXAMPLES=OFF + -DAXOM_ENABLE_TUTORIALS=OFF + -DAXOM_ENABLE_DOCS=OFF + -DAXOM_ENABLE_TOOLS=OFF + -DENABLE_BENCHMARKS=OFF + -DENABLE_FORTRAN=OFF + # Parallelism / dependencies + # AXOM_ENABLE_MPI / AXOM_ENABLE_OPENMP are cmake_dependent_options + # gated on BLT's ENABLE_MPI / ENABLE_OPENMP, so those must be set + # too or Axom silently builds without MPI (no Sidre IOManager). + -DENABLE_MPI=ON + -DENABLE_OPENMP="${OPENMP_ON}" + -DAXOM_ENABLE_MPI=ON + -DAXOM_ENABLE_OPENMP="${OPENMP_ON}" + -DMPI_C_COMPILER="${MPI_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DCAMP_DIR="${CAMP_ROOT}" + -DRAJA_DIR="${RAJA_ROOT}" + -DUMPIRE_DIR="${UMPIRE_ROOT}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + # Sidre dependencies + -DCONDUIT_DIR="${CONDUIT_ROOT}" + -DHDF5_DIR="${HDF5_ROOT}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + # Spin's GPU paths run through RAJA -> Umpire memory plumbing. + CMAKE_ARGS+=( + -DAXOM_ENABLE_${GPU_BACKEND}=ON + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + ) + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCUDA_TOOLKIT_ROOT_DIR="${CUDA_TOOLKIT_ROOT_DIR}" + ) + fi + fi + + run_with_log my_axom_config cmake ../src "${CMAKE_ARGS[@]}" + run_with_log my_axom_build make -j "${MAKE_JOBS}" + run_with_log my_axom_install make install + + AXOM_ROOT="${BASE_DIR}/axom/install_${BUILD_SUFFIX}" + export AXOM_ROOT + echo "Axom installed to: ${AXOM_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# ExaConstit +########################################### +# Like ExaCMech, the SNLS-batch transitive deps mean we pass FMT_DIR / +# UMPIRE_DIR / CHAI_DIR unconditionally now (previously GPU-only). +build_exaconstit() { + echo "==========================================" + echo "Building ExaConstit" + echo "==========================================" + + clone_if_missing "${EXACONSTIT_REPO}" "${EXACONSTIT_BRANCH}" "${BASE_DIR}/ExaConstit" + sync_submodules "${BASE_DIR}/ExaConstit" + + prepare_build_dir "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" + -DENABLE_TESTS="${ENABLE_TESTS_EXACONSTIT}" + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_FORTRAN=OFF + -DENABLE_SNLS_V03=ON + -DCMAKE_INSTALL_PREFIX=../install_dir/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DMFEM_DIR="${MFEM_ROOT}/lib/cmake/mfem" + -DECMECH_DIR="${ECMECH_ROOT}" + -DSNLS_DIR="${SNLS_ROOT}/lib/cmake/snls" + -DAXOM_DIR="${AXOM_ROOT}/lib/cmake" + -Daxom_DIR="${AXOM_ROOT}/lib/cmake" + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + # SNLS-batch transitive deps (now needed on CPU builds too). + -DFMT_DIR="${FMT_DIR}" + -DUMPIRE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + ) + + if [ "${BUILD_TYPE}" = "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + ) + else + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DENABLE_${GPU_BACKEND}=ON + ) + + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" + -DBLT_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + ) + elif [ "${GPU_BACKEND}" = "HIP" ]; then + CMAKE_ARGS+=( + -DCMAKE_HIP_FLAGS="${CMAKE_GPU_FLAGS}" + ) + fi + fi + + run_with_log my_exconstit_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_exconstit_build make -j "${MAKE_JOBS}" + + EXACONSTIT_ROOT="${BASE_DIR}/ExaConstit/install_dir" + export EXACONSTIT_ROOT + echo "==========================================" + echo "ExaConstit build complete!" + echo "Install prefix: ${EXACONSTIT_ROOT}" + echo "==========================================" + cd "${BASE_DIR}" +} + +########################################### +# Main orchestration function +########################################### +# Build order honors the dependency graph: +# 1. BLT (header-only build helper, must come first so every +# downstream package can point at it). +# 2. RAJA Portability Suite: CAMP -> RAJA -> Umpire -> CHAI +# (Umpire and CHAI now built on every platform). +# 3. MFEM stack: Hypre, METIS, MFEM. +# 4. Application stack: SNLS -> ExaCMech -> HDF5 -> Conduit -> Axom +# -> ExaConstit. SNLS and ExaCMech come first because the SNLS +# batch solver path is a hard dependency; HDF5 and Conduit feed +# Axom's Sidre component; Axom is placed before ExaConstit since +# ExaConstit consumes it (and Axom will eventually pick up MFEM). +build_all_dependencies() { + # Common stack + build_blt + build_camp + build_raja + build_umpire + build_chai + + # MFEM stack + build_hypre + build_metis + build_mfem + + # Application stack + build_snls + build_exacmech + build_hdf5 + build_conduit + build_axom + build_exaconstit +} diff --git a/scripts/install/common/build_functions_mfem.sh b/scripts/install/common/build_functions_mfem.sh new file mode 100644 index 00000000..263d602d --- /dev/null +++ b/scripts/install/common/build_functions_mfem.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# MFEM-stack build functions: Hypre, METIS, MFEM. +# +# Depends on the helpers in build_helpers.sh and the common stack +# defined in build_functions_common.sh (specifically RAJA / CAMP, +# which MFEM consumes). + +########################################### +# Hypre +########################################### +build_hypre() { + echo "==========================================" + echo "Building Hypre" + echo "==========================================" + + if [ ! -d "${BASE_DIR}/hypre" ]; then + git clone https://github.com/hypre-space/hypre.git --branch "${HYPRE_VER}" --single-branch "${BASE_DIR}/hypre" + fi + + prepare_build_dir "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" + + run_with_log my_hypre_config cmake ../src \ + -DCMAKE_INSTALL_PREFIX=../src/hypre_${BUILD_SUFFIX}/ \ + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \ + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ + -DMPI_C_COMPILER="${MPI_C_COMPILER}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + + run_with_log my_hypre_build make -j "${MAKE_JOBS}" + run_with_log my_hypre_install make install + + HYPRE_ROOT="${BASE_DIR}/hypre/src/hypre_${BUILD_SUFFIX}" + export HYPRE_ROOT + echo "Hypre installed to: ${HYPRE_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# METIS +########################################### +build_metis() { + echo "==========================================" + echo "Building METIS" + echo "==========================================" + + if [ ! -d "${BASE_DIR}/metis-${METIS_VER}" ]; then + curl -o metis-${METIS_VER}.tar.gz "${METIS_URL}" + tar -xzf metis-${METIS_VER}.tar.gz + rm metis-${METIS_VER}.tar.gz + fi + + cd "${BASE_DIR}/metis-${METIS_VER}" + + # METIS doesn't have a proper incremental build, so always clean + make distclean 2>/dev/null || true + + prepare_build_dir "${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" + + run_with_log my_metis_config make config \ + prefix="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" \ + CC="${CMAKE_C_COMPILER}" \ + CXX="${CMAKE_CXX_COMPILER}" + + run_with_log my_metis_build make -j "${MAKE_JOBS}" + run_with_log my_metis_install make install + + METIS_ROOT="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" + export METIS_ROOT + echo "METIS installed to: ${METIS_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# MFEM +########################################### +build_mfem() { + echo "==========================================" + echo "Building MFEM" + echo "==========================================" + + clone_if_missing "${MFEM_REPO}" "${MFEM_BRANCH}" "${BASE_DIR}/mfem" + # Don't sync submodules for MFEM to preserve local changes + + prepare_build_dir "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DMFEM_USE_MPI=YES + -DMFEM_USE_SIMD=NO + -DMETIS_DIR="${METIS_ROOT}" + -DHYPRE_DIR="${HYPRE_ROOT}" + -DMFEM_USE_RAJA=YES + -DRAJA_DIR="${RAJA_ROOT}" + -DRAJA_REQUIRED_PACKAGES="camp" + -DMFEM_USE_CAMP=ON + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + -DMFEM_USE_OPENMP="${OPENMP_ON}" + -DMFEM_USE_ZLIB=YES + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + ) + + if [ "${BUILD_TYPE}" = "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" + ) + else + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DMFEM_USE_${GPU_BACKEND}=ON + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + ) + + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCMAKE_CUDA_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CUDA_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_CUDA=ON + ) + elif [ "${GPU_BACKEND}" = "HIP" ]; then + CMAKE_ARGS+=( + -DHIP_ARCH="${MFEM_HIP_ARCHITECTURES}" + -DCMAKE_HIP_ARCHITECTURES="${MFEM_HIP_ARCHITECTURES}" + ) + fi + fi + + run_with_log my_mfem_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_mfem_build make -j "${MAKE_JOBS}" + run_with_log my_mfem_install make install + + MFEM_ROOT="${BASE_DIR}/mfem/install_${BUILD_SUFFIX}" + export MFEM_ROOT + echo "MFEM installed to: ${MFEM_ROOT}" + cd "${BASE_DIR}" +} diff --git a/scripts/install/common/build_helpers.sh b/scripts/install/common/build_helpers.sh new file mode 100644 index 00000000..8165e759 --- /dev/null +++ b/scripts/install/common/build_helpers.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Shared helper functions used by every build function. +# +# Kept separate from the build_functions_*.sh files so the per-library +# build logic stays focused on CMake invocations rather than the +# logging / cloning / build-dir-prep plumbing. + +########################################### +# Logging wrapper +########################################### +run_with_log() { + local log="$1"; shift + "$@" |& tee "$log" +} + +########################################### +# Clone repository only if missing, initialize submodules on first clone +########################################### +clone_if_missing() { + local repo="$1" branch="$2" dest="$3" + if [ ! -d "$dest/.git" ]; then + echo "Cloning ${dest}..." + git clone --branch "$branch" "$repo" "$dest" + cd "$dest" + if [ -f .gitmodules ]; then + git submodule update --init --recursive + fi + cd "$BASE_DIR" + else + echo "${dest} already exists, skipping clone." + fi +} + +########################################### +# Optional: force submodule sync when explicitly requested +########################################### +sync_submodules() { + local dest="$1" + if [ "${SYNC_SUBMODULES}" = "ON" ] && [ -f "$dest/.gitmodules" ]; then + echo "Syncing submodules in ${dest}..." + cd "$dest" + git submodule sync --recursive + git submodule update --init --recursive + cd "$BASE_DIR" + fi +} + +########################################### +# Respect REBUILD flag when preparing build directories +########################################### +prepare_build_dir() { + local dir="$1" + if [ "${REBUILD}" = "ON" ]; then + mkdir -p "$dir" + rm -rf "$dir"/* + echo "Cleaned build directory: ${dir}" + else + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + echo "Created build directory: ${dir}" + else + echo "Reusing existing build directory: ${dir}" + fi + fi +} diff --git a/scripts/install/common/dependency_versions.sh b/scripts/install/common/dependency_versions.sh index 1babd0a9..500b9c96 100644 --- a/scripts/install/common/dependency_versions.sh +++ b/scripts/install/common/dependency_versions.sh @@ -1,7 +1,17 @@ #!/usr/bin/env bash # Central version control for all dependencies -# Portability libraries +########################################### +# Build infrastructure +########################################### +# BLT lifted out so all RADIUSS-stack packages share a single BLT and stay in sync. +# Each package below is pointed at this via -DBLT_SOURCE_DIR=${BLT_ROOT}. +export BLT_REPO="https://github.com/LLNL/blt.git" +export BLT_VER="v0.7.2" + +########################################### +# Portability libraries (RAJA Portability Suite) +########################################### export CAMP_VER="v2025.12.0" export RAJA_VER="v2025.12.2" #export UMPIRE_VER="v2025.09.0" @@ -10,11 +20,44 @@ export RAJA_VER="v2025.12.2" export UMPIRE_VER="v2025.12.0" export CHAI_VER="v2025.12.0" +########################################### +# SNLS (lifted out of ExaCMech so it can be built standalone with the +# RAJA Portability Suite and the batch-solver option always enabled) +########################################### +export SNLS_REPO="https://github.com/LLNL/SNLS.git" +export SNLS_VER="v0.4.4" + +########################################### +# Restart / checkpoint I/O stack (Axom Sidre) +########################################### +# Sidre requires Conduit, and Conduit's HDF5 relay provides the +# sidre_hdf5 I/O protocol. Stick with the HDF5 1.14 series; the 2.x +# series has caused CMake configuration issues downstream. +export HDF5_REPO="https://github.com/HDFGroup/hdf5.git" +export HDF5_VER="hdf5_1.14.6" +# Conduit v0.9.7 is the version Axom v0.15.0 was released against. +export CONDUIT_REPO="https://github.com/LLNL/conduit.git" +export CONDUIT_VER="v0.9.7" + +########################################### +# Axom (HPC utility library suite) +########################################### +# Built with core + slic + spin + primal + slam + sidre. Axom will +# eventually consume MFEM as well, which is why build_axom lives in the +# application-stack build file (build_functions_exaconstit.sh) rather +# than the common stack. +export AXOM_REPO="https://github.com/LLNL/axom.git" +export AXOM_VER="v0.14.0" + +########################################### # Material models +########################################### export EXACMECH_REPO="https://github.com/LLNL/ExaCMech.git" export EXACMECH_BRANCH="v0.4.3" +########################################### # FEM infrastructure +########################################### export HYPRE_VER="v3.1.0" export METIS_VER="5.1.0" export METIS_URL="https://mfem.github.io/tpls/metis-${METIS_VER}.tar.gz" @@ -22,10 +65,14 @@ export METIS_URL="https://mfem.github.io/tpls/metis-${METIS_VER}.tar.gz" export MFEM_REPO="https://github.com/rcarson3/mfem.git" export MFEM_BRANCH="exaconstit-dev" +########################################### # Main application +########################################### export EXACONSTIT_REPO="https://github.com/llnl/ExaConstit.git" export EXACONSTIT_BRANCH="exaconstit-dev" +########################################### # Build standards +########################################### export CMAKE_CXX_STANDARD="17" -export CMAKE_BUILD_TYPE="Release" \ No newline at end of file +export CMAKE_BUILD_TYPE="Release" diff --git a/scripts/install/common/preflight_checks.sh b/scripts/install/common/preflight_checks.sh index 6defa1e1..4d5f4f20 100644 --- a/scripts/install/common/preflight_checks.sh +++ b/scripts/install/common/preflight_checks.sh @@ -18,16 +18,16 @@ resolve_base_dir() { BASE_DIR=$(pwd -P) echo "Using current directory as build directory: ${BASE_DIR}" fi - + export BASE_DIR - + echo "==========================================" echo "Build Configuration:" echo " Base directory: ${BASE_DIR}" echo " All dependencies will be cloned and built here" echo "==========================================" echo "" - + # Optional: warn if running from ExaConstit source tree if [[ "${BASE_DIR}" == *"/ExaConstit"* ]]; then echo "⚠️ WARNING: You appear to be building inside the ExaConstit source tree." @@ -50,18 +50,18 @@ check_required_paths() { local missing=0 for p in "$@"; do if [[ "$p" == */bin/* ]]; then - if [ ! -x "$p" ]; then + if [ ! -x "$p" ]; then echo "ERROR: Missing executable: $p" >&2 missing=1 fi else - if [ ! -e "$p" ]; then + if [ ! -e "$p" ]; then echo "ERROR: Missing path: $p" >&2 missing=1 fi fi done - if [ "$missing" -ne 0 ]; then + if [ "$missing" -ne 0 ]; then echo "ERROR: Required paths missing. Exiting." >&2 exit 1 fi @@ -114,14 +114,17 @@ print_build_summary() { echo " Linker: ${CMAKE_EXE_LINKER_FLAGS}" echo "" echo "Key Versions:" + echo " BLT: ${BLT_VER}" echo " CAMP: ${CAMP_VER}" echo " RAJA: ${RAJA_VER}" - if [ "${BUILD_TYPE}" != "cpu" ]; then - echo " Umpire: ${UMPIRE_VER}" - echo " CHAI: ${CHAI_VER}" - fi + echo " Umpire: ${UMPIRE_VER}" + echo " CHAI: ${CHAI_VER}" echo " Hypre: ${HYPRE_VER}" echo " MFEM: ${MFEM_BRANCH}" + echo " SNLS: ${SNLS_VER}" + echo " HDF5: ${HDF5_VER}" + echo " Conduit: ${CONDUIT_VER}" + echo " Axom: ${AXOM_VER}" echo " ExaCMech: ${EXACMECH_BRANCH}" echo " ExaConstit: ${EXACONSTIT_BRANCH}" echo "==========================================" @@ -130,19 +133,19 @@ print_build_summary() { # Validate configuration before proceeding validate_configuration() { echo "Validating configuration..." - + # Check compilers exist check_required_paths "${CMAKE_C_COMPILER}" "${CMAKE_CXX_COMPILER}" - + if [ "${BUILD_TYPE}" != "cpu" ]; then check_required_paths "${CMAKE_GPU_COMPILER}" fi - + # Check MPI wrappers check_required_paths "${MPI_C_COMPILER}" "${MPI_CXX_COMPILER}" "${MPI_Fortran_COMPILER}" - + # Check required commands check_required_commands git cmake make curl tar - + echo "Configuration validation complete." -} \ No newline at end of file +} diff --git a/scripts/install/configs/cpu_intel_config.sh b/scripts/install/configs/cpu_intel_config.sh index a0dfb046..51f4fe60 100644 --- a/scripts/install/configs/cpu_intel_config.sh +++ b/scripts/install/configs/cpu_intel_config.sh @@ -71,7 +71,7 @@ export CMAKE_GPU_ARCHITECTURES="" export CMAKE_GPU_FLAGS="" ########################################### -# CHAI Options (Not Used in CPU Build) +# CHAI Options (CPU build: GPU-specific features disabled) ########################################### export CHAI_DISABLE_RM="OFF" export CHAI_THIN_GPU_ALLOCATE="OFF" @@ -80,5 +80,7 @@ export CHAI_ENABLE_PICK="OFF" export CHAI_DEBUG="OFF" export CHAI_ENABLE_GPU_SIMULATION_MODE="OFF" export CHAI_ENABLE_UM="OFF" -export CHAI_ENABLE_MANAGED_PTR="OFF" +# managed_ptr is required on CPU too: SNLS (batch solvers) and ExaCMech +# use chai::managed_ptr whenever SNLS is built with the RAJA Portability Suite. +export CHAI_ENABLE_MANAGED_PTR="ON" export CHAI_ENABLE_MANAGED_PTR_ON_GPU="OFF" \ No newline at end of file diff --git a/scripts/install/configs/cpu_mac_config.sh b/scripts/install/configs/cpu_mac_config.sh index b2598c87..3119f1f7 100644 --- a/scripts/install/configs/cpu_mac_config.sh +++ b/scripts/install/configs/cpu_mac_config.sh @@ -97,7 +97,7 @@ export CMAKE_GPU_ARCHITECTURES="" export CMAKE_GPU_FLAGS="" ########################################### -# CHAI Options (Not Used in CPU Build) +# CHAI Options (CPU build: GPU-specific features disabled) ########################################### export CHAI_DISABLE_RM="OFF" export CHAI_THIN_GPU_ALLOCATE="OFF" @@ -106,7 +106,9 @@ export CHAI_ENABLE_PICK="OFF" export CHAI_DEBUG="OFF" export CHAI_ENABLE_GPU_SIMULATION_MODE="OFF" export CHAI_ENABLE_UM="OFF" -export CHAI_ENABLE_MANAGED_PTR="OFF" +# managed_ptr is required on CPU too: SNLS (batch solvers) and ExaCMech +# use chai::managed_ptr whenever SNLS is built with the RAJA Portability Suite. +export CHAI_ENABLE_MANAGED_PTR="ON" export CHAI_ENABLE_MANAGED_PTR_ON_GPU="OFF" ########################################### From 88f28902825fc8f5b793a419a1f548f59313fde6 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:36:56 -0700 Subject: [PATCH 02/12] [Claude] Make SimulationState's quadrature function registry lookups safe A few SimulationState accessors used std::map::operator[], which inserts an entry when the key is missing. On a rank that owns no elements of a region, that left nullptr entries behind in the quadrature function and quadrature space registries for any code that later loops over them: - GetQuadratureFunction() now uses find() and returns nullptr without inserting anything. - AddQuadratureFunction() no longer creates a function on a null quadrature space when the region has no elements on this rank; it returns false instead, as it already did for a duplicate name. - UpdateModel() and SetupModelVariables() use at(), since every registered begin/end pair must exist. PartialQuadratureFunction gets an explicit copy assignment that copies the data only, plus a defaulted copy constructor. The implicit copy assignment it relied on is deprecated (-Wdeprecated-copy), because mfem::QuadratureFunction has a user-provided copy constructor, and it also re-pointed the function's space members at the source's. SetupModelVariables() and ExaCMechModel::InitStateVars() only ever copy between functions on the same space, which the new assignment asserts. SimulationState::UpdateDeltaTime() named its second parameter `failure` with a default of false, but passed it on as TimeManagement's `success`, so a one-argument call would have been treated as a failed step. It is now `success` with no default. Every existing caller passes both arguments, so behavior is unchanged. --- scripts/install/common/dependency_versions.sh | 3 +- src/mfem_expt/partial_qfunc.hpp | 29 +++++++++++++++++++ src/sim_state/simulation_state.cpp | 20 ++++++++----- src/sim_state/simulation_state.hpp | 23 ++++++++++----- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/scripts/install/common/dependency_versions.sh b/scripts/install/common/dependency_versions.sh index 500b9c96..2bdb2025 100644 --- a/scripts/install/common/dependency_versions.sh +++ b/scripts/install/common/dependency_versions.sh @@ -35,7 +35,8 @@ export SNLS_VER="v0.4.4" # series has caused CMake configuration issues downstream. export HDF5_REPO="https://github.com/HDFGroup/hdf5.git" export HDF5_VER="hdf5_1.14.6" -# Conduit v0.9.7 is the version Axom v0.15.0 was released against. +# Conduit v0.9.7 is the version Axom v0.15.0 was released against +# even though we use v0.14.0 below. export CONDUIT_REPO="https://github.com/LLNL/conduit.git" export CONDUIT_VER="v0.9.7" diff --git a/src/mfem_expt/partial_qfunc.hpp b/src/mfem_expt/partial_qfunc.hpp index 5e6bb83c..d53d82de 100644 --- a/src/mfem_expt/partial_qfunc.hpp +++ b/src/mfem_expt/partial_qfunc.hpp @@ -165,6 +165,35 @@ class PartialQuadratureFunction : public QuadratureFunction { */ PartialQuadratureFunction& operator=(const QuadratureFunction& qf); + /** + * @brief Copy constructor: a new function on the same partial space with a copy of the data. + * + * @details Declared (defaulted) explicitly because the copy assignment below is + * user-provided; the behaviour is that of the former implicit copy constructor. + */ + PartialQuadratureFunction(const PartialQuadratureFunction&) = default; + + /** + * @brief Copy assignment with value semantics: copies the DATA only. + * @param rhs Function on an identically laid-out partial space (same vdim and size) + * @return Reference to this PartialQuadratureFunction + * + * @details The implicitly-declared version relied on mfem::QuadratureFunction's + * implicitly-declared copy assignment, which is deprecated because QuadratureFunction has a + * user-provided copy constructor (-Wdeprecated-copy), and it also re-pointed this function's + * space members at the source's. It is reached from SimulationState::SetupModelVariables + * (every registered begin/end pair) and ExaCMechModel::InitStateVars; both copy between + * functions on the same space, for which data-only copying is exactly the intended operation. + */ + PartialQuadratureFunction& operator=(const PartialQuadratureFunction& rhs) { + MFEM_ASSERT(rhs.GetVDim() == vdim && rhs.Size() == this->Size(), + "PartialQuadratureFunction copy assignment requires identical layouts"); + if (this != &rhs) { + QuadratureFunction::operator=(static_cast(rhs)); + } + return *this; + } + /** * @brief Fill a global QuadratureFunction with data from this partial function. * diff --git a/src/sim_state/simulation_state.cpp b/src/sim_state/simulation_state.cpp index d6a39145..41910065 100644 --- a/src/sim_state/simulation_state.cpp +++ b/src/sim_state/simulation_state.cpp @@ -642,14 +642,20 @@ bool SimulationState::AddQuadratureFunction(const std::string_view& qf_name, const int vdim, const int region) { std::string qf_name_mat = GetQuadratureFunctionMapName(qf_name, region); - if (m_map_qfs.find(qf_name_mat) == m_map_qfs.end()) { - std::string qspace_name = GetRegionName(region); - m_map_qfs.emplace(qf_name_mat, - std::make_shared( - m_map_qs[qspace_name], vdim, 0.0)); - return true; + if (m_map_qfs.find(qf_name_mat) != m_map_qfs.end()) { + return false; } - return false; + // find(), not operator[]: a region with no elements on this rank has no quadrature space + // here, and operator[] would insert a null entry into m_map_qs, which every loop over the + // spaces would then dereference. + const auto space = m_map_qs.find(GetRegionName(region)); + if (space == m_map_qs.end() || !space->second) { + return false; + } + m_map_qfs.emplace( + qf_name_mat, + std::make_shared(space->second, vdim, 0.0)); + return true; } std::pair diff --git a/src/sim_state/simulation_state.hpp b/src/sim_state/simulation_state.hpp index 18ba4102..bd88bedd 100644 --- a/src/sim_state/simulation_state.hpp +++ b/src/sim_state/simulation_state.hpp @@ -566,7 +566,8 @@ class SimulationState { * @param qf_name Quadrature function name * @param region Region index (-1 for global) * @return Shared pointer to the quadrature function - * @throws std::runtime_error if quadrature function doesn't exist + * @return The quadrature function, or nullptr if no function is registered under the + * resolved name (the lookup never inserts into the registry). * * @details Primary interface for accessing simulation data. Automatically resolves * region-specific names and returns the appropriate quadrature function. @@ -585,7 +586,10 @@ class SimulationState { */ std::shared_ptr GetQuadratureFunction(const std::string_view& qf_name, const int region = -1) { - return m_map_qfs[GetQuadratureFunctionMapName(qf_name, region)]; + // find(), not operator[]: a lookup of an unregistered name (e.g. a region inactive on + // this rank) must not insert a nullptr entry into m_map_qfs. + const auto it = m_map_qfs.find(GetQuadratureFunctionMapName(qf_name, region)); + return (it != m_map_qfs.end()) ? it->second : nullptr; } /** @@ -594,7 +598,9 @@ class SimulationState { * @param qf_name Base quadrature function name * @param vdim Vector dimension of the quadrature function * @param region Region index (-1 for global) - * @return True if successfully added, false if already exists + * @return True if added; false if a function of that name is already registered, or if + * the region has no quadrature space on this rank (it owns none of the region's + * elements) * * @details Creates a new quadrature function with the specified vector dimension * and associates it with the given region. The function is initialized with zeros @@ -680,7 +686,7 @@ class SimulationState { */ void UpdateModel() { for (auto [name_prev, name_cur] : m_model_update_qf_pairs) { - m_map_qfs[name_prev]->Swap(*m_map_qfs[name_cur]); + m_map_qfs.at(name_prev)->Swap(*m_map_qfs.at(name_cur)); } } @@ -693,7 +699,8 @@ class SimulationState { */ void SetupModelVariables() { for (auto [name_prev, name_cur] : m_model_update_qf_pairs) { - m_map_qfs[name_cur]->operator=(*m_map_qfs[name_prev]); + // Value copy (PartialQuadratureFunction's explicit copy assignment). + *m_map_qfs.at(name_cur) = *m_map_qfs.at(name_prev); } } @@ -1093,14 +1100,14 @@ class SimulationState { * @brief Update time step based on solver performance * * @param nr_steps Number of Newton-Raphson iterations required - * @param failure Whether the time step failed to converge + * @param success Whether the time step converged (true) or failed (false) * @return Updated time step status * * @details Delegates to TimeManagement for comprehensive adaptive time step control. * See TimeManagement::UpdateDeltaTime() for detailed algorithm description. */ - TimeStep UpdateDeltaTime(const int nr_steps, const bool failure = false) { - return m_time_manager.UpdateDeltaTime(nr_steps, failure); + TimeStep UpdateDeltaTime(const int nr_steps, const bool success) { + return m_time_manager.UpdateDeltaTime(nr_steps, success); } /** From b6f637f63687c49f015aabed3507c00dcc360a9b Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:38:57 -0700 Subject: [PATCH 03/12] [Claude] Add the [Restart] options table Adds RestartOptions, parsed from a new [Restart] table: - write, interval, write_final: write a checkpoint every `interval` completed steps and/or when the run finishes - keep_last: how many complete checkpoints to keep (0 keeps all) - directory: where checkpoints are written, one cycle_/ directory per checkpoint - read, read_cycle, read_directory: restart from the newest (or a given) checkpoint, optionally from another run's directory Validation rejects negative intervals and keep_last values, empty directories, and a restart whose read directory doesn't exist, and warns about settings that are ignored or would never write a checkpoint. Writing or reading checkpoints requires ExaConstit built with Axom; until the checkpoint code lands in a later commit, any [Restart] table that turns on write or read is rejected with that message. print_options() prints the restart settings when they're used. On a restart (read = true), the mesh, grain, region-mapping, orientation and state-variable files are replaced by the checkpoint and may have been deleted, so validation no longer checks them in that case. MeshOptions, GrainInfo, StateVariables and MaterialOptions::validate() gain a check_input_files flag (default true) for this. The region-mapping check for multiple materials was duplicated in ExaOptions::validate(); one copy is removed. The old [Time] restart, restart_time and restart_cycle keys are removed. They were parsed and printed, but nothing ever used them. src/options.toml documents the new table, and drops the old commented-out keys (also from the calibration workflow example) and a line claiming restart files go to the output directory. --- developers_guide.md | 1 + src/CMakeLists.txt | 1 + src/options.toml | 56 +++++++-- src/options/option_material.cpp | 21 ++-- src/options/option_mesh.cpp | 7 +- src/options/option_parser_v2.cpp | 97 +++++++------- src/options/option_parser_v2.hpp | 118 ++++++++++++++---- src/options/option_restart.cpp | 107 ++++++++++++++++ src/options/option_time.cpp | 13 -- .../examples/options.toml | 8 -- 10 files changed, 322 insertions(+), 107 deletions(-) create mode 100644 src/options/option_restart.cpp diff --git a/developers_guide.md b/developers_guide.md index 239763ae..afe3a402 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -218,6 +218,7 @@ The `src/` directory contains the core ExaConstit implementation organized into - **`option_solvers.cpp`**: Linear and nonlinear solver settings - **`option_time.cpp`**: Time-stepping parameters - **`option_post_processing.cpp`**: Post-processing configuration +- **`option_restart.cpp`**: Checkpoint/restart configuration - **`option_enum.cpp`**: Enumeration type conversions - **`option_util.hpp`**: Utility functions for option parsing diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af8cd1ab..042b9e0e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -53,6 +53,7 @@ set(EXACONSTIT_SOURCES options/option_material.cpp options/option_mesh.cpp options/option_post_processing.cpp + options/option_restart.cpp options/option_solvers.cpp options/option_time.cpp postprocessing/postprocessing_driver.cpp diff --git a/src/options.toml b/src/options.toml index b44ab69c..fd473d1c 100644 --- a/src/options.toml +++ b/src/options.toml @@ -335,13 +335,6 @@ grain_file = "grains.txt" # 3. Fixed (if [Time.Fixed] exists or as default) time_type = "custom" - # ===== Restart Options (Optional) ===== - # Continue from a previous simulation - # Currently ignored but will be coming in a near-future release - # restart = true - # restart_time = 0.5 # Time to restart from - # restart_cycle = 100 # Cycle number to restart from - # ===== Automatic Time Stepping ===== # Adjusts time step based on convergence behavior # After each converged step dt is multiplied by @@ -642,7 +635,6 @@ grain_file = "grains.txt" # [output_directory]/[basename]/ <- Main simulation folder # [output_directory]/[basename]/avg_*.txt <- Volume average files # [output_directory]/[basename]/visualizations/ <- Visualization files - # [output_directory]/[basename]/restart/ <- Restart files (if enabled) # # Example: If output_directory = "./results" and basename = "tensile_test": # ./results/tensile_test/ <- All outputs go here @@ -900,4 +892,50 @@ grain_file = "grains.txt" # Number of elements [X_elements, Y_elements, Z_elements] # Total elements = product of these numbers - nxyz = [10, 10, 10] \ No newline at end of file + nxyz = [10, 10, 10] + +# ===================================== +# CHECKPOINT / RESTART +# ===================================== +# Writes binary checkpoints during a run and restarts later runs from them. +# Requires ExaConstit built with Axom (Axom with Sidre, Conduit and HDF5; the +# install scripts build all three). See doc/restart.md for the full details. +# +# A restart needs only the checkpoint and this options file: the mesh, grain, +# region-mapping, orientation and state-variable files are not read. +# A restart must use the same number of MPI ranks, the same mesh order and the +# same materials (names, model types and state variable counts) as the run that +# wrote the checkpoint. The final time, time steps, solver settings, output +# settings and material properties may change; boundary conditions for steps +# already completed must stay the same. + +[Restart] + # Write checkpoints during this run + write = false + + # Write a checkpoint every this many completed time steps + # 0 = only write the final checkpoint (if write_final = true) + interval = 0 + + # Also write a checkpoint when the simulation finishes + write_final = true + + # Number of checkpoints to keep on disk; older ones are removed + # 0 = keep every checkpoint + keep_last = 2 + + # Directory checkpoints are written to, one subdirectory per checkpoint + # (cycle_) + directory = "restart" + + # Restart this run from a checkpoint + read = false + + # Which checkpoint to restart from, by number of completed steps + # If omitted, the newest complete checkpoint is used + # read_cycle = 100 + + # Read the checkpoint from this directory instead of `directory` + # Useful for restarting from another run's checkpoints without touching + # them; this run still writes its own checkpoints to `directory` + # read_directory = "other_run/restart" diff --git a/src/options/option_material.cpp b/src/options/option_material.cpp index d93a06f4..cabef2eb 100644 --- a/src/options/option_material.cpp +++ b/src/options/option_material.cpp @@ -297,15 +297,16 @@ std::vector MaterialOptions::from_toml_array(const toml::value& return materials; } -bool GrainInfo::validate() const { - // Implement validation logic - if (!orientation_file) { +bool GrainInfo::validate(bool check_input_files) const { + // On a restart the orientations come from the checkpointed state variables, so the + // orientation file is neither required nor read. + if (check_input_files && !orientation_file) { WARNING_0_OPT("Error: Grain table was provided without providing an orientation file this " "is required"); return false; } - if (orientation_file) { + if (check_input_files && orientation_file) { if (!std::filesystem::exists(*orientation_file)) { std::ostringstream err; err << "Error: Orientation file does not exist provided value: " << *orientation_file; @@ -354,8 +355,10 @@ bool MaterialProperties::validate() const { return true; } -bool StateVariables::validate() const { - if (static_cast(num_vars) != initial_values.size()) { +bool StateVariables::validate(bool check_input_files) const { + // initial_values only seed a fresh run; on a restart the state comes from the checkpoint + // (a deleted state file leaves initial_values empty after StateVariables::from_toml). + if (check_input_files && static_cast(num_vars) != initial_values.size()) { WARNING_0_OPT("Error: StateVariables num_vars != initial_values.size()"); return false; } @@ -422,7 +425,7 @@ bool MaterialModelOptions::validate() const { return true; } -bool MaterialOptions::validate() const { +bool MaterialOptions::validate(bool check_input_files) const { std::string mat_name = material_name + "_" + std::to_string(region_id); if (mech_type == MechType::NOTYPE) { @@ -448,7 +451,7 @@ bool MaterialOptions::validate() const { WARNING_0_OPT(err.str()); return false; } - if (!state_vars.validate()) { + if (!state_vars.validate(check_input_files)) { std::ostringstream err; err << "Error: Material table for material_name_region# " << mat_name << " the State_Vars table had errors"; @@ -464,7 +467,7 @@ bool MaterialOptions::validate() const { } if (grain_info) { - if (!grain_info->validate()) { + if (!grain_info->validate(check_input_files)) { std::ostringstream err; err << "Error: Material table for material_name_region# " << mat_name << " the Grain table had errors"; diff --git a/src/options/option_mesh.cpp b/src/options/option_mesh.cpp index 684f1384..d6cfece6 100644 --- a/src/options/option_mesh.cpp +++ b/src/options/option_mesh.cpp @@ -61,7 +61,7 @@ MeshOptions MeshOptions::from_toml(const toml::value& toml_input) { return options; } -bool MeshOptions::validate() const { +bool MeshOptions::validate(bool check_input_files) const { if (mesh_type == MeshType::NOTYPE) { WARNING_0_OPT("Error: Mesh table was not provided an appropriate mesh type"); return false; @@ -87,8 +87,9 @@ bool MeshOptions::validate() const { } } - // Check that mesh file exists for CUBIT or OTHER mesh types - if ((mesh_type == MeshType::FILE) && !mesh_file.empty()) { + // Check that mesh file exists for CUBIT or OTHER mesh types. A restart rebuilds the mesh + // from the checkpoint, so the file is not needed (or checked) then. + if (check_input_files && (mesh_type == MeshType::FILE) && !mesh_file.empty()) { if (!fs::exists(mesh_file)) { std::ostringstream err; err << "Error: Mesh file '" << mesh_file << "' does not exist." << std::endl; diff --git a/src/options/option_parser_v2.cpp b/src/options/option_parser_v2.cpp index 4f030d74..d8dee223 100644 --- a/src/options/option_parser_v2.cpp +++ b/src/options/option_parser_v2.cpp @@ -242,6 +242,7 @@ void ExaOptions::parse_from_toml(const toml::value& toml_input) { // Parse component sections parse_mesh_options(toml_input); parse_time_options(toml_input); + parse_restart_options(toml_input); parse_solver_options(toml_input); parse_boundary_options(toml_input); parse_visualization_options(toml_input); @@ -277,6 +278,12 @@ void ExaOptions::parse_time_options(const toml::value& toml_input) { time = TimeOptions::from_toml(toml::find(toml_input, "Time")); } +void ExaOptions::parse_restart_options(const toml::value& toml_input) { + if (toml_input.contains("Restart")) { + restart = RestartOptions::from_toml(toml::find(toml_input, "Restart")); + } +} + void ExaOptions::parse_solver_options(const toml::value& toml_input) { if (toml_input.contains("Solvers")) { solvers = SolverOptions::from_toml(toml::find(toml_input, "Solvers")); @@ -466,7 +473,11 @@ void ExaOptions::load_post_processing_file() { bool ExaOptions::validate() { // Basic validation - could be expanded with more comprehensive checks - if (!mesh.validate()) + // On a restart, the mesh, grain, region-mapping, orientation and state-variable inputs + // are replaced by the checkpoint and may no longer exist. + const bool inputs_from_checkpoint = restart.read; + + if (!mesh.validate(!inputs_from_checkpoint)) return false; if (!time.validate()) return false; @@ -476,6 +487,8 @@ bool ExaOptions::validate() { return false; if (!boundary_conditions.validate()) return false; + if (!restart.validate()) + return false; // Check that we have at least one material if (materials.empty()) { @@ -483,19 +496,7 @@ bool ExaOptions::validate() { return false; } - if (materials.size() > 1) { - if (!region_mapping_file) { - WARNING_0_OPT("Error: region_mapping_file was not provided even though multiple " - "materials were asked for."); - return false; - } else if (mesh.mesh_type == MeshType::AUTO && !grain_file) { - WARNING_0_OPT("Error: region_mapping_file was provided but no grain_file was provided " - "when using auto mesh."); - return false; - } - } - - if (materials.size() > 1) { + if (materials.size() > 1 && !inputs_from_checkpoint) { if (!region_mapping_file) { WARNING_0_OPT("Error: region_mapping_file was not provided even though multiple " "materials were asked for."); @@ -509,7 +510,7 @@ bool ExaOptions::validate() { size_t index = 1; for (auto& mat : materials) { - if (!mat.validate()) + if (!mat.validate(!inputs_from_checkpoint)) return false; // Update the region_id value after validating // everything so to make it easier for users to @@ -630,31 +631,34 @@ bool ExaOptions::validate() { if (!post_processing.validate()) return false; - if (region_mapping_file) { - if (!std::filesystem::exists(*region_mapping_file)) { - std::ostringstream err; - err << "Error: Region mapping file: " << *region_mapping_file << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (!inputs_from_checkpoint) { + if (region_mapping_file) { + if (!std::filesystem::exists(*region_mapping_file)) { + std::ostringstream err; + err << "Error: Region mapping file: " << *region_mapping_file << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } - } - if (grain_file) { - if (!std::filesystem::exists(*grain_file)) { - std::ostringstream err; - err << "Error: Grain file provided at top level: " << *grain_file << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (grain_file) { + if (!std::filesystem::exists(*grain_file)) { + std::ostringstream err; + err << "Error: Grain file provided at top level: " << *grain_file + << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } - } - if (orientation_file) { - if (!std::filesystem::exists(*orientation_file)) { - std::ostringstream err; - err << "Error: Orientation file provided at top level: " << *orientation_file - << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (orientation_file) { + if (!std::filesystem::exists(*orientation_file)) { + std::ostringstream err; + err << "Error: Orientation file provided at top level: " << *orientation_file + << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } } @@ -678,6 +682,21 @@ void ExaOptions::print_options() const { std::cout << "\nSimulation Information:\n"; std::cout << " Base name: " << basename << "\n"; std::cout << " Version: " << version << "\n"; + if (restart.write || restart.read) { + std::cout << " Restart: write=" << (restart.write ? "yes" : "no") + << " interval=" << restart.interval + << " write_final=" << (restart.write_final ? "yes" : "no") + << " keep_last=" << restart.keep_last + << " read=" << (restart.read ? "yes" : "no"); + if (restart.read_cycle) { + std::cout << " read_cycle=" << *restart.read_cycle; + } + std::cout << " directory=" << restart.directory.string(); + if (restart.read && restart.read_directory) { + std::cout << " read_directory=" << restart.read_directory->string(); + } + std::cout << "\n"; + } // Print each component print_mesh_options(); @@ -773,12 +792,6 @@ void ExaOptions::print_time_options() const { } else { std::cout << " Type: none (time_type has no matching [Time.*] section)\n"; } - - if (time.restart) { - std::cout << " Restart enabled:\n"; - std::cout << " Restart time: " << time.restart_time << "\n"; - std::cout << " Restart cycle: " << time.restart_cycle << "\n"; - } } void ExaOptions::print_solver_options() const { diff --git a/src/options/option_parser_v2.hpp b/src/options/option_parser_v2.hpp index 1f312a42..866b2a30 100644 --- a/src/options/option_parser_v2.hpp +++ b/src/options/option_parser_v2.hpp @@ -181,8 +181,9 @@ struct MeshOptions { */ bool periodicity = false; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static MeshOptions from_toml(const toml::value& toml_input); @@ -222,8 +223,9 @@ struct GrainInfo { */ int num_grains = 0; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static GrainInfo from_toml(const toml::value& toml_input); @@ -274,8 +276,9 @@ struct StateVariables { */ std::vector initial_values; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static StateVariables from_toml(const toml::value& toml_input); @@ -441,8 +444,9 @@ struct MaterialOptions { */ double temperature = 298.0; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static MaterialOptions from_toml(const toml::value& toml_input); @@ -562,21 +566,6 @@ struct TimeOptions { */ std::optional custom_time; - /** - * @brief Whether this is a restart simulation - */ - bool restart = false; - - /** - * @brief Time to restart from (if restart is enabled) - */ - double restart_time = 0.0; - - /** - * @brief Cycle number to restart from (if restart is enabled) - */ - size_t restart_cycle = 0; - /** * @brief Determine which time stepping mode is active * @@ -1352,6 +1341,78 @@ struct PostProcessingOptions { const LightUpOptions* get_light_up_config_for_region(int region_id) const; }; +/** + * @brief Checkpoint and restart configuration ([Restart] table) + * + * @details Checkpoints are binary Axom Sidre files, one per MPI rank, written to + * `directory/cycle_/`. A restart needs only the checkpoint and the options + * file; the mesh, grain, region-mapping, orientation and state-variable files are not read. + * Writing or reading checkpoints requires ExaConstit built with Axom (EXACONSTIT_USE_AXOM). + * See doc/restart.md for the full description. + */ +struct RestartOptions { + /** + * @brief Whether to write checkpoints during the run + */ + bool write = false; + + /** + * @brief Number of completed cycles between checkpoints (0 disables interval checkpoints) + */ + int interval = 0; + + /** + * @brief Whether to also write a checkpoint when the simulation finishes + */ + bool write_final = true; + + /** + * @brief Number of complete checkpoints to keep on disk; older ones are removed (0 keeps + * all of them) + */ + int keep_last = 2; + + /** + * @brief Whether to restart this run from a checkpoint + */ + bool read = false; + + /** + * @brief Completed-cycle count of the checkpoint to restart from; the newest complete + * checkpoint if unset + */ + std::optional read_cycle; + + /** + * @brief Directory checkpoints are written to (and read from unless read_directory is set) + */ + std::filesystem::path directory = "restart"; + + /** + * @brief Directory to read the restart checkpoint from instead of directory + * + * @details Lets a continuation restart from another run's checkpoints while writing its own + * to directory. Writing, superseding and keep_last pruning only ever act on directory, so + * the other run's checkpoints are never renamed or removed. + */ + std::optional read_directory; + + /** + * @brief Directory a restart reads from + * + * @return read_directory if it is set, otherwise directory + */ + const std::filesystem::path& GetReadDirectory() const { + return read_directory ? *read_directory : directory; + } + + // Validation + bool validate() const; + + // Conversion from toml + static RestartOptions from_toml(const toml::value& toml_input); +}; + /** * @brief Main options class for ExaConstit simulation configuration */ @@ -1402,6 +1463,11 @@ class ExaOptions { */ PostProcessingOptions post_processing; + /** + * @brief Checkpoint/restart options + */ + RestartOptions restart; + /** * @brief Paths to external material configuration files */ @@ -1464,6 +1530,12 @@ class ExaOptions { */ void parse_time_options(const toml::value& toml_input); + /** + * @brief Parse checkpoint and restart options from TOML input + * @param toml_input TOML value containing the [Restart] table + */ + void parse_restart_options(const toml::value& toml_input); + /** * @brief Parse solver options from TOML input * @param toml_input TOML value containing solver configuration diff --git a/src/options/option_restart.cpp b/src/options/option_restart.cpp new file mode 100644 index 00000000..7196fd48 --- /dev/null +++ b/src/options/option_restart.cpp @@ -0,0 +1,107 @@ +#include "options/option_parser_v2.hpp" +#include "options/option_util.hpp" + +#include +#include + +namespace fs = std::filesystem; + +RestartOptions RestartOptions::from_toml(const toml::value& toml_input) { + RestartOptions options; + + if (toml_input.contains("write")) { + options.write = toml::find(toml_input, "write"); + } + + if (toml_input.contains("interval")) { + options.interval = toml::find(toml_input, "interval"); + } + + if (toml_input.contains("write_final")) { + options.write_final = toml::find(toml_input, "write_final"); + } + + if (toml_input.contains("keep_last")) { + options.keep_last = toml::find(toml_input, "keep_last"); + } + + if (toml_input.contains("read")) { + options.read = toml::find(toml_input, "read"); + } + + if (toml_input.contains("read_cycle")) { + options.read_cycle = toml::find(toml_input, "read_cycle"); + } + + if (toml_input.contains("directory")) { + options.directory = toml::find(toml_input, "directory"); + } + + if (toml_input.contains("read_directory")) { + options.read_directory = toml::find(toml_input, "read_directory"); + } + + return options; +} + +bool RestartOptions::validate() const { + // Checkpoints are Axom Sidre files, so a build without Axom can neither write nor read + // them. Without this check the request would be silently ignored. +#if !defined(EXACONSTIT_USE_AXOM) + if (write || read) { + WARNING_0_OPT("Error: The Restart table asks to write or read checkpoints, but ExaConstit " + "was built without Axom (configure with AXOM_DIR pointing at an Axom " + "built with HDF5)."); + return false; + } +#endif + + if (interval < 0) { + WARNING_0_OPT("Error: Restart table has `interval` set to a value less than 0 (use 0 to " + "disable interval checkpoints)."); + return false; + } + + if (keep_last < 0) { + WARNING_0_OPT("Error: Restart table has `keep_last` set to a value less than 0 (use 0 to " + "keep every checkpoint)."); + return false; + } + + if ((write || read) && directory.empty()) { + WARNING_0_OPT("Error: Restart table has an empty `directory`."); + return false; + } + + if (read_directory.has_value() && read_directory->empty()) { + WARNING_0_OPT("Error: Restart table has an empty `read_directory`."); + return false; + } + + // Checking the directory here gives a clear message before any setup work is done. Which + // checkpoint inside it is used, and whether it is complete, is decided when the + // simulation starts. + if (read && !fs::is_directory(GetReadDirectory())) { + std::ostringstream err; + err << "Error: Restart table has `read` = true, but the checkpoint directory '" + << GetReadDirectory().string() << "' does not exist."; + WARNING_0_OPT(err.str()); + return false; + } + + // The remaining cases are allowed but probably not what was intended. + if (read_directory.has_value() && !read) { + WARNING_0_OPT("Restart table sets `read_directory` but `read` = false, so it is ignored."); + } + + if (read_cycle.has_value() && !read) { + WARNING_0_OPT("Restart table sets `read_cycle` but `read` = false, so it is ignored."); + } + + if (write && interval == 0 && !write_final) { + WARNING_0_OPT("Restart table has `write` = true, but `interval` = 0 and `write_final` = " + "false, so no checkpoints will be written."); + } + + return true; +} diff --git a/src/options/option_time.cpp b/src/options/option_time.cpp index cf2a760b..7a614296 100644 --- a/src/options/option_time.cpp +++ b/src/options/option_time.cpp @@ -125,19 +125,6 @@ void TimeOptions::determine_time_type(const std::optional& request TimeOptions TimeOptions::from_toml(const toml::value& toml_input) { TimeOptions options; - // Check for restart options - if (toml_input.contains("restart")) { - options.restart = toml::find(toml_input, "restart"); - } - - if (toml_input.contains("restart_time")) { - options.restart_time = toml::find(toml_input, "restart_time"); - } - - if (toml_input.contains("restart_cycle")) { - options.restart_cycle = toml::find(toml_input, "restart_cycle"); - } - // Check for nested time stepping sections if (toml_input.contains("Auto")) { options.auto_time = AutoTimeOptions::from_toml(toml::find(toml_input, "Auto")); diff --git a/workflows/exaconstit-calibrate/examples/options.toml b/workflows/exaconstit-calibrate/examples/options.toml index a68c8cb0..aae9a4ae 100644 --- a/workflows/exaconstit-calibrate/examples/options.toml +++ b/workflows/exaconstit-calibrate/examples/options.toml @@ -315,13 +315,6 @@ grain_file = "grains.txt" # 3. Fixed (if [Time.Fixed] exists or as default) time_type = "custom" - # ===== Restart Options (Optional) ===== - # Continue from a previous simulation - # Currently ignored but will be coming in a near-future release - # restart = true - # restart_time = 0.5 # Time to restart from - # restart_cycle = 100 # Cycle number to restart from - # ===== Automatic Time Stepping ===== # Adjusts time step based on convergence behavior [Time.Auto] @@ -560,7 +553,6 @@ grain_file = "grains.txt" # [output_directory]/[basename]/ <- Main simulation folder # [output_directory]/[basename]/avg_*.txt <- Volume average files # [output_directory]/[basename]/visualizations/ <- Visualization files - # [output_directory]/[basename]/restart/ <- Restart files (if enabled) # # Example: If output_directory = "./results" and basename = "tensile_test": # ./results/tensile_test/ <- All outputs go here From 72850b0408cf58ead106d3617b298f5d773096b5 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:39:46 -0700 Subject: [PATCH 04/12] [Claude] Write and rebuild a ParMesh exactly for checkpoints Adds ExaSidreParMesh (sim_state/exa_sidre_parmesh.hpp/cpp), which writes one rank's piece of an mfem::ParMesh into an Axom Sidre group and rebuilds the identical ParMesh from it in a later run with the same number of ranks. Identical means the same local element and vertex numbering, element vertex order, attributes, communication groups, shared vertices, edges and faces, and shared-entity orientation, so every finite element space built on the rebuilt mesh numbers its DOFs exactly as the original did. That is what lets a checkpoint store DOF vectors and quadrature data as plain arrays. axom::sidre::MFEMSidreDataCollection is not used. It records shared edges and faces in each rank's local orientation, which matches shared DOFs incorrectly once a shared face has more than one interior DOF (H1 order 3 and up on hexes), and it references the mesh's memory rather than copying it. The layout follows Conduit's mesh Blueprint (coordsets, topologies and adjacency sets). Each group's local index is stored explicitly, so the rebuilt group order doesn't depend on the order Sidre or HDF5 iterate over groups. The rebuild checks the stored counts and group indices and reports problems on the ranks that find them before every rank aborts together. Ranks that own no elements write and read empty arrays and take part in every collective step. Conforming meshes with one element geometry and one boundary geometry are supported (segments, triangles, quadrilaterals, tetrahedra, hexahedra). restart_sidre_utils.hpp holds small helpers shared with the checkpoint code in the next commit. ExaConstit now links Axom and defines EXACONSTIT_USE_AXOM when configured with AXOM_DIR; the unused Axom placeholder member in SimulationState is removed, since it would no longer compile with that define. test_restart_mesh_roundtrip writes and rebuilds hex and tet meshes, with and without parallel refinement, plus a mesh where half of the ranks own no elements. It checks the topology, groups, shared-entity order and orientation, and that H1 spaces of order 1 to 4 have identical DOF layouts. It runs on 1 to 4 ranks, but never on more ranks than EXACONSTIT_TEST_MAX_MPI_TASKS (default: the logical core count detected at configure time), so it doesn't oversubscribe the CI runners. --- developers_guide.md | 1 + src/CMakeLists.txt | 9 + src/sim_state/exa_sidre_parmesh.cpp | 1127 +++++++++++++++++++++++++ src/sim_state/exa_sidre_parmesh.hpp | 157 ++++ src/sim_state/restart_sidre_utils.hpp | 94 +++ src/sim_state/simulation_state.hpp | 8 - test/CMakeLists.txt | 30 + test/test_restart_mesh_roundtrip.cpp | 536 ++++++++++++ 8 files changed, 1954 insertions(+), 8 deletions(-) create mode 100644 src/sim_state/exa_sidre_parmesh.cpp create mode 100644 src/sim_state/exa_sidre_parmesh.hpp create mode 100644 src/sim_state/restart_sidre_utils.hpp create mode 100644 test/test_restart_mesh_roundtrip.cpp diff --git a/developers_guide.md b/developers_guide.md index afe3a402..d5fac282 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -243,6 +243,7 @@ The `src/` directory contains the core ExaConstit implementation organized into #### `sim_state/` **Purpose**: Simulation state management and field storage - **`simulation_state.hpp/cpp`**: Central state container class +- **`exa_sidre_parmesh.hpp/cpp`**: Writing and rebuilding the parallel mesh for checkpoints **Manages**: - Finite element spaces and mesh data diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 042b9e0e..cb9c825b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,7 @@ set(EXACONSTIT_HEADERS postprocessing/postprocessing_driver.hpp postprocessing/mechanics_lightup.hpp sim_state/simulation_state.hpp + sim_state/exa_sidre_parmesh.hpp solvers/mechanics_solver.hpp solvers/trust_region_solver.hpp utilities/dynamic_function_loader.hpp @@ -60,6 +61,7 @@ set(EXACONSTIT_SOURCES postprocessing/projection_class.cpp postprocessing/mechanics_lightup.cpp sim_state/simulation_state.cpp + sim_state/exa_sidre_parmesh.cpp solvers/mechanics_solver.cpp solvers/trust_region_solver.cpp utilities/mechanics_kernels.cpp @@ -115,6 +117,10 @@ if(ENABLE_CALIPER) list(APPEND EXACONSTIT_DEPENDS caliper) endif() +if(ENABLE_AXOM) + list(APPEND EXACONSTIT_DEPENDS axom) +endif() + if(UNIX AND NOT APPLE AND TARGET Threads::Threads) # Check if we determined explicit linking is needed if(EXACONSTIT_THREADS_EXPLICIT_LINK) @@ -133,6 +139,9 @@ set(EXACONSTIT_DEFINES HAVE_EXACONSTIT) if(ENABLE_CALIPER) list(APPEND EXACONSTIT_DEFINES HAVE_CALIPER) endif() +if(ENABLE_AXOM) + list(APPEND EXACONSTIT_DEFINES EXACONSTIT_USE_AXOM) +endif() #------------------------------------------------------------------------------ # Includes #------------------------------------------------------------------------------ diff --git a/src/sim_state/exa_sidre_parmesh.cpp b/src/sim_state/exa_sidre_parmesh.cpp new file mode 100644 index 00000000..3a896823 --- /dev/null +++ b/src/sim_state/exa_sidre_parmesh.cpp @@ -0,0 +1,1127 @@ +// Compiled only when ExaConstit is built with Axom, so this file can always be listed in +// CMake; without EXACONSTIT_USE_AXOM it is an empty translation unit. +#if defined(EXACONSTIT_USE_AXOM) + +#include "sim_state/exa_sidre_parmesh.hpp" + +#include "sim_state/restart_sidre_utils.hpp" +#include "utilities/mechanics_log.hpp" +#include "utilities/unified_logger.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +// ========================================================================= +// BACKGROUND: HOW AN mfem::ParMesh DESCRIBES ITS PARALLEL STRUCTURE +// ========================================================================= +// +// Everything in this file writes or rebuilds the pieces listed here, so it is worth reading +// once before changing anything below. References are to MFEM's mesh/pmesh.cpp, +// general/communication.cpp and fem/pfespace.cpp. +// +// Local mesh. Each rank holds an ordinary serial mfem::Mesh for its own elements: local +// vertices, local elements, local boundary elements, and the local edges and faces MFEM +// derives from the element connectivity (FinalizeTopology()). Vertices, edges and faces on +// the interface between ranks exist on every rank that touches them, each rank with its own +// local index for them. +// +// Communication groups (ParMesh::gtopo, an mfem::GroupTopology). A group is a set of ranks +// that share at least one vertex, edge or face. Group 0 is always this rank on its own and +// shares nothing. Every other group has at least two members and a local index +// 1..NGroups()-1 on each member rank; the same group generally has different local indices +// on different ranks. Each group has a master rank (GroupTopology::Create() picks the +// smallest rank in the group), and the master owns the group's shared DOFs when the finite +// element spaces assign true DOFs. Inside a GroupTopology, ranks are referred to by "local +// processor" (lproc) numbers: this rank is lproc 0 and the other ranks are numbered in the +// order they first appear when walking the groups in local group order +// (GroupTopology::ProcToLProc()). The local group order therefore also fixes the neighbor +// numbering and the order of the messages the group communicators exchange. +// +// Shared entities. The shared vertices are stored as local vertex indices (svert_lvert). +// The shared edges and faces are stored as vertex tuples (shared_edges, shared_trias, +// shared_quads) in a "stored orientation" that every rank in the group agrees on, together +// with the local edge or face each one corresponds to (sedge_ledge, sface_lface). +// +// Group tables (group_svert, group_sedge, group_stria, group_squad). mfem::Table objects in +// compressed-row (CSR) form with one row per communication group 1..N: row r describes group +// r+1, since group 0 shares nothing. Row r lists the indices, into the shared-entity arrays +// above, of the entities group r+1 shares. +// +// How the finite element spaces use this. ParFiniteElementSpace builds its parallel DOF +// numbering from these structures. For each group it collects the DOFs on the group's shared +// entities, in the order the entities appear in the group's row, and the group's master +// assigns them true DOFs that the other members then receive by position. Two consequences +// drive the whole design of this file: +// 1. Entity i of a group on one rank is matched with entity i of the same group on every +// other member, so the order of the entities within each group must be identical on +// every member rank. +// 2. A shared edge or face with several interior DOFs lists them in a specific order, and +// GetSharedEdgeDofs() / GetSharedTriangleDofs() / GetSharedQuadrilateralDofs() permute +// each rank's local DOF order by the orientation that GroupEdge() / GroupTriangle() / +// GroupQuadrilateral() report, so that every rank lists them in the stored order. A +// wrong stored orientation silently matches the wrong DOFs across ranks. It only shows +// up once a shared entity has two or more interior DOFs (H1 order >= 3 on hex faces), +// and the mesh otherwise looks perfectly valid. +// +// MFEM normally builds all of this in the partitioning ParMesh constructor, which walks the +// global serial mesh in global entity order; that is what makes the per-group entity order +// and the stored orientations agree across ranks. A restarted run has no global mesh, so the +// writer below records exactly what each rank holds and the reader rebuilds it unchanged. + +namespace exaconstit { + +namespace { + +using axom::sidre::Group; +using axom::sidre::IndexType; +using axom::sidre::View; +using restart_detail::ReadSidreScalar; +using restart_detail::VerifyOnAllRanks; + +static_assert(std::is_same::value, + "Restart mesh I/O stores coordinates as FLOAT64; build MFEM in double precision."); + +/** + * @brief Conduit Blueprint shape name for an MFEM geometry + * + * @param geometry MFEM geometry type (mfem::Geometry::Type) + * @return Blueprint shape name, or nullptr if the restart mesh I/O does not support the + * geometry + */ +const char* ShapeNameFromGeometry(const int geometry) { + switch (geometry) { + case mfem::Geometry::SEGMENT: + return "line"; + case mfem::Geometry::TRIANGLE: + return "tri"; + case mfem::Geometry::SQUARE: + return "quad"; + case mfem::Geometry::TETRAHEDRON: + return "tet"; + case mfem::Geometry::CUBE: + return "hex"; + default: + return nullptr; + } +} + +/** + * @brief Create an integer array view owned by Sidre + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param count Number of entries + * @return Pointer to the allocated entries, or nullptr when count is 0 + * + * @details The arrays are owned by Sidre, never external views of the mesh's memory, because + * Group::load() only restores the description of an external view and not its data, so the + * reader would get nothing back. Zero-length arrays are described but not allocated, which + * is why callers must not dereference the returned pointer when count is 0. Ranks with no + * boundary elements, or groups with no shared edges, produce such arrays all the time. + */ +int* CreateOwnedIntArray(Group* group, const std::string& path, const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::INT_ID, 0); + return nullptr; + } + return group->createViewAndAllocate(path, axom::sidre::INT_ID, count)->getData(); +} + +/** + * @brief Create a double array view owned by Sidre + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param count Number of entries + * @return Pointer to the allocated entries, or nullptr when count is 0 + * + * @details See CreateOwnedIntArray() for why the arrays are owned and why zero-length arrays + * are not allocated. + */ +double* CreateOwnedDoubleArray(Group* group, const std::string& path, const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return nullptr; + } + return group->createViewAndAllocate(path, axom::sidre::FLOAT64_ID, count)->getData(); +} + +/** + * @brief Look up an array written by CreateOwnedIntArray() or CreateOwnedDoubleArray() + * + * @tparam ValueType int or double + * @param group Group that holds the view + * @param path Path of the view relative to the group + * @param expected_count Number of entries the array must have + * @return Pointer to the entries (nullptr when the array is empty) and whether the view + * exists, is owned by Sidre, has the expected type and has expected_count entries + * + * @details The type and ownership checks catch a checkpoint written by a different layout + * version or a corrupted file. Without them a wrong-typed array would be reinterpreted + * silently, and an external view would come back with a null data pointer. + */ +template +std::pair +ReadOwnedArray(Group* group, const std::string& path, const IndexType expected_count) { + const axom::sidre::TypeID expected_type = std::is_same::value + ? axom::sidre::FLOAT64_ID + : axom::sidre::INT_ID; + if (!group->hasView(path)) { + return {nullptr, false}; + } + View* view = group->getView(path); + if (view->getNumElements() != expected_count) { + return {nullptr, false}; + } + if (expected_count == 0) { + return {nullptr, true}; + } + if (view->getTypeID() != expected_type || view->isExternal()) { + return {nullptr, false}; + } + return {view->getData(), true}; +} + +/** + * @brief Look up an owned integer array of unknown length that holds fixed-size tuples + * + * @param group Group that holds the view + * @param path Path of the view relative to the group + * @param components Number of integers per tuple (for example 2 for an edge) + * @param ok Set to false if the view is missing, malformed, or its length is not a multiple + * of components; left unchanged otherwise, so one flag can collect several checks + * @return Pointer to the entries (nullptr when empty or malformed) and the number of tuples + * + * @details Used for the adjacency-set arrays, whose lengths are not recorded separately. + */ +std::pair +ReadIntTuples(Group* group, const std::string& path, const int components, bool& ok) { + if (!group->hasView(path)) { + ok = false; + return {nullptr, 0}; + } + View* view = group->getView(path); + const IndexType scalar_count = view->getNumElements(); + if (scalar_count % components != 0) { + ok = false; + return {nullptr, 0}; + } + if (scalar_count == 0) { + return {nullptr, 0}; + } + if (view->getTypeID() != axom::sidre::INT_ID || view->isExternal()) { + ok = false; + return {nullptr, 0}; + } + return {view->getData(), scalar_count / components}; +} + +/** + * @brief Agree across ranks on the element and boundary element geometries + * + * @param mesh Parallel mesh being written + * @return Mesh-wide element geometry and boundary element geometry; the boundary geometry is + * -1 if no rank has boundary elements + * + * @details Collective. The reader needs the geometry to size the connectivity arrays and to + * create elements, and it has to be the same on every rank. A rank deep inside the domain may + * own no boundary elements, though, and a rank could in principle own no elements at all, so + * a rank cannot simply look at its own first element. Each rank reports what it has, using -1 + * for "none", and an MPI_Allreduce with MPI_MAX picks the mesh-wide value (valid geometry + * values are all >= 0). + * + * The same pass also checks that no rank mixes geometries, that every rank agrees with the + * mesh-wide value, and that the geometry is one the Blueprint layout here supports. Any + * violation aborts on every rank, since a single connectivity array per topology cannot + * describe mixed meshes. + */ +std::pair AgreeOnMeshGeometries(const mfem::ParMesh& mesh) { + int local_element_geometry = -1; + int local_boundary_geometry = -1; + bool local_mixed_geometry = false; + for (int element_index = 0; element_index < mesh.GetNE(); ++element_index) { + const int geometry = mesh.GetElementGeometry(element_index); + if (local_element_geometry < 0) { + local_element_geometry = geometry; + } else if (geometry != local_element_geometry) { + local_mixed_geometry = true; + } + } + for (int boundary_index = 0; boundary_index < mesh.GetNBE(); ++boundary_index) { + const int geometry = mesh.GetBdrElementGeometry(boundary_index); + if (local_boundary_geometry < 0) { + local_boundary_geometry = geometry; + } else if (geometry != local_boundary_geometry) { + local_mixed_geometry = true; + } + } + + std::array local_geometries = {local_element_geometry, local_boundary_geometry}; + std::array global_geometries = {-1, -1}; + MPI_Allreduce( + local_geometries.data(), global_geometries.data(), 2, MPI_INT, MPI_MAX, mesh.GetComm()); + const int element_geometry = global_geometries[0]; + const int boundary_geometry = global_geometries[1]; + + const bool local_consistent = + !local_mixed_geometry && + (local_element_geometry < 0 || local_element_geometry == element_geometry) && + (local_boundary_geometry < 0 || local_boundary_geometry == boundary_geometry) && + ShapeNameFromGeometry(element_geometry) != nullptr && + (boundary_geometry < 0 || ShapeNameFromGeometry(boundary_geometry) != nullptr); + VerifyOnAllRanks(local_consistent, + mesh.GetComm(), + "the mesh mixes element geometries or uses a geometry other than segment, " + "triangle, quadrilateral, tetrahedron or hexahedron", + "Unable to write the mesh to the restart checkpoint"); + return {element_geometry, boundary_geometry}; +} + +/** + * @brief Write the vertex coordinates under `coordsets/coords` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * + * @details One array per axis (x, y and, in 3D, z), each indexed by local vertex number, as + * Blueprint's explicit coordsets expect. The local vertex numbering has to be preserved + * exactly: element connectivity, the shared vertices and the shared edge and face tuples all + * refer to vertices by these indices. + * + * These are the mesh's vertex positions, not its Nodes. For a high-order or deformed mesh the + * vertices only carry the element corners. SimulationState's coordinate fields are restored + * separately from the checkpoint, so nothing is lost. + */ +void WriteCoordinates(const mfem::ParMesh& mesh, Group* mesh_group) { + const std::array axis_names = {"x", "y", "z"}; + mesh_group->createViewString("coordsets/coords/type", "explicit"); + for (int axis = 0; axis < mesh.SpaceDimension(); ++axis) { + double* axis_values = CreateOwnedDoubleArray( + mesh_group, std::string("coordsets/coords/values/") + axis_names[axis], mesh.GetNV()); + for (int vertex_index = 0; vertex_index < mesh.GetNV(); ++vertex_index) { + axis_values[vertex_index] = mesh.GetVertex(vertex_index)[axis]; + } + } +} + +/** + * @brief Write the element connectivity and element attributes + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param element_geometry Mesh-wide element geometry from AgreeOnMeshGeometries() + * + * @details The connectivity is one flat array, vertices_per_element entries per element, in + * local element order and in each element's own vertex order. Both orders matter. The finite + * element spaces number their local DOFs element by element following each element's vertex + * order, so a different element order or a rotated vertex list would put restored values on + * the wrong DOFs even though the mesh looks the same. That is also why the reader must not + * let MFEM "fix" element orientations (see the reader's Finalize() call). + * + * In ExaConstit the element attributes hold the region IDs once SimulationState has been set + * up (it overwrites the grain IDs read from the mesh file), and that is what a restarted run + * uses to rebuild its regions. The grain IDs are checkpointed separately by the caller. + */ +void WriteVolumeTopology(const mfem::ParMesh& mesh, Group* mesh_group, const int element_geometry) { + const int num_elements = mesh.GetNE(); + const int vertices_per_element = mfem::Geometry::NumVerts[element_geometry]; + mesh_group->createViewString("topologies/mesh/type", "unstructured"); + mesh_group->createViewString("topologies/mesh/coordset", "coords"); + mesh_group->createViewString("topologies/mesh/elements/shape", + ShapeNameFromGeometry(element_geometry)); + + int* connectivity = CreateOwnedIntArray(mesh_group, + "topologies/mesh/elements/connectivity", + static_cast(num_elements) * + vertices_per_element); + for (int element_index = 0; element_index < num_elements; ++element_index) { + const int* element_vertices = mesh.GetElement(element_index)->GetVertices(); + for (int local_vertex = 0; local_vertex < vertices_per_element; ++local_vertex) { + connectivity[static_cast(element_index) * vertices_per_element + + local_vertex] = element_vertices[local_vertex]; + } + } + + mesh_group->createViewString("fields/mesh_material_attribute/association", "element"); + mesh_group->createViewString("fields/mesh_material_attribute/topology", "mesh"); + int* element_attributes = CreateOwnedIntArray( + mesh_group, "fields/mesh_material_attribute/values", num_elements); + for (int element_index = 0; element_index < num_elements; ++element_index) { + element_attributes[element_index] = mesh.GetAttribute(element_index); + } +} + +/** + * @brief Write the boundary element connectivity and boundary attributes + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param boundary_geometry Mesh-wide boundary geometry from AgreeOnMeshGeometries(); nothing + * is written when it is -1 (no rank has boundary elements) + * + * @details Boundary elements are stored explicitly rather than regenerated on read, because + * their attributes are what the boundary conditions refer to, and MFEM's generated boundary + * would carry default attributes. A rank with no boundary elements still writes the + * (zero-length) arrays whenever some other rank has boundary elements, so every rank's group + * has the same structure. + */ +void WriteBoundaryTopology(const mfem::ParMesh& mesh, + Group* mesh_group, + const int boundary_geometry) { + if (boundary_geometry < 0) { + return; + } + const int num_boundary_elements = mesh.GetNBE(); + const int vertices_per_boundary_element = mfem::Geometry::NumVerts[boundary_geometry]; + mesh_group->createViewString("topologies/mesh/boundary_topology", "boundary"); + mesh_group->createViewString("topologies/boundary/type", "unstructured"); + mesh_group->createViewString("topologies/boundary/coordset", "coords"); + mesh_group->createViewString("topologies/boundary/elements/shape", + ShapeNameFromGeometry(boundary_geometry)); + + int* boundary_connectivity = CreateOwnedIntArray(mesh_group, + "topologies/boundary/elements/connectivity", + static_cast(num_boundary_elements) * + vertices_per_boundary_element); + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + const int* boundary_vertices = mesh.GetBdrElement(boundary_index)->GetVertices(); + for (int local_vertex = 0; local_vertex < vertices_per_boundary_element; ++local_vertex) { + boundary_connectivity[static_cast(boundary_index) * + vertices_per_boundary_element + + local_vertex] = boundary_vertices[local_vertex]; + } + } + + mesh_group->createViewString("fields/boundary_material_attribute/association", "element"); + mesh_group->createViewString("fields/boundary_material_attribute/topology", "boundary"); + int* boundary_attributes = CreateOwnedIntArray( + mesh_group, "fields/boundary_material_attribute/values", num_boundary_elements); + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + boundary_attributes[boundary_index] = mesh.GetBdrAttribute(boundary_index); + } +} + +/** + * @brief Write the communication groups and their shared entities under `adjsets/mesh` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * + * @details Writes one Sidre group per communication group g = 1..NGroups()-1 (see the + * background notes at the top of this file for what a communication group is). Each holds: + * - `local_group_index`: g itself. The reader puts the groups back in this order. The order + * determines the lproc numbering and the message order of the group communicators, so + * rebuilding it exactly keeps the restarted run's parallel communication, and therefore + * its floating-point reduction order, identical to the original run's. The reader does not + * rely on Sidre or HDF5 returning the groups in creation order. + * - `neighbors`: the MPI ranks of the other group members. + * - `values`: the shared vertices, as local vertex indices. + * - `edges` (2D and 3D): the shared edges as vertex pairs, in the stored orientation. + * - `triangles`, `quadrilaterals` (3D): the shared faces as vertex tuples, in the stored + * orientation. + * + * The shared entities are written in the order the group's table row lists them, which the + * partitioning ParMesh constructor made identical on every member rank (it walks the global + * entities in global order). The reader rebuilds each row in that same order, so the + * per-group order agreement survives the round trip. + * + * Every group is written, including groups that share only vertices, so the rebuilt mesh has + * exactly the original set of groups. The group name `g_` is the + * same on every member rank, which makes the files easier to inspect; nothing depends on it. + * + * **Recovering the stored orientation.** ParMesh does not expose the stored edge and face + * tuples directly. GroupEdge(), GroupTriangle() and GroupQuadrilateral() return the local + * edge or face index and the orientation o of the stored entity relative to that local + * entity, which is what ParFiniteElementSpace consumes. Writing the local entity's vertices + * instead of the stored ones would be wrong whenever o is not the identity. Every rank would + * rebuild its shared entity in its own local orientation, every rank would then report o as + * the identity, and ranks whose local orientations differ would list the interior DOFs of + * that edge or face in different orders. So the stored tuple is rebuilt from the local one: + * - Edges: GetEdgeVertices() always returns a local edge as (smaller, larger), and + * GroupEdge() reports o = +1 exactly when the stored edge's first vertex is the smaller one + * (`o = (v[0] < v[1]) ? +1 : -1`). Hence stored = (o > 0) ? (smaller, larger) : (larger, + * smaller). + * - Faces: GroupTriangle() / GroupQuadrilateral() compute + * o = GetTriOrientation() / GetQuadOrientation(base = local face, test = stored face), and + * those return the o for which test[Orient[o][i]] == base[i] for every i + * (Geometry::Constants<...>::Orient in fem/geom.hpp). So the stored face is obtained by + * scattering: stored[Orient[o][i]] = local_face[i]. + * On read, the rebuilt mesh holds these stored tuples, and GroupEdge(), GroupTriangle() and + * GroupQuadrilateral() return the same orientations as in the writing run. + */ +void WriteAdjacencySets(const mfem::ParMesh& mesh, Group* mesh_group) { + const int dimension = mesh.Dimension(); + Group* adjset_group = mesh_group->createGroup("adjsets/mesh"); + adjset_group->createViewString("association", "vertex"); + adjset_group->createViewString("topology", "mesh"); + Group* communication_groups = adjset_group->createGroup("groups"); + + mfem::Array local_entity_vertices; + const auto& triangle_orient = mfem::Geometry::Constants::Orient; + const auto& quad_orient = mfem::Geometry::Constants::Orient; + + // Group 0 is this rank on its own and shares nothing, so it is not written. + for (int group_index = 1; group_index < mesh.GetNGroups(); ++group_index) { + const std::string group_name = "g" + + std::to_string(mesh.gtopo.GetGroupMasterRank(group_index)) + + "_" + + std::to_string(mesh.gtopo.GetGroupMasterGroup(group_index)); + Group* shared_group = communication_groups->createGroup(group_name); + shared_group->createViewScalar("local_group_index", group_index); + + // Neighbor ranks. GetGroup() lists the members as lproc numbers; this rank is always + // lproc 0, and every other lproc is translated back to its MPI rank, since lproc + // numbers mean nothing outside this GroupTopology. + const int member_count = mesh.gtopo.GetGroupSize(group_index); + const int* member_lprocs = mesh.gtopo.GetGroup(group_index); + int* neighbor_ranks = CreateOwnedIntArray(shared_group, "neighbors", member_count - 1); + int neighbor_slot = 0; + for (int member = 0; member < member_count; ++member) { + if (member_lprocs[member] != 0) { + neighbor_ranks[neighbor_slot++] = mesh.gtopo.GetNeighborRank(member_lprocs[member]); + } + } + + // Shared vertices: a vertex has no orientation, so its local index is all there is. + const int shared_vertex_count = mesh.GroupNVertices(group_index); + int* shared_vertices = CreateOwnedIntArray(shared_group, "values", shared_vertex_count); + for (int i = 0; i < shared_vertex_count; ++i) { + shared_vertices[i] = mesh.GroupVertex(group_index, i); + } + + if (dimension >= 2) { + // Shared edges, converted from the local (smaller, larger) order to the stored + // order using GroupEdge()'s orientation (see the function documentation). + const int shared_edge_count = mesh.GroupNEdges(group_index); + int* shared_edges_out = CreateOwnedIntArray( + shared_group, "edges", 2 * static_cast(shared_edge_count)); + for (int i = 0; i < shared_edge_count; ++i) { + int local_edge = -1; + int orientation = 0; + mesh.GroupEdge(group_index, i, local_edge, orientation); + mesh.GetEdgeVertices(local_edge, local_entity_vertices); + const int smaller = local_entity_vertices[0]; + const int larger = local_entity_vertices[1]; + shared_edges_out[2 * i + 0] = (orientation > 0) ? smaller : larger; + shared_edges_out[2 * i + 1] = (orientation > 0) ? larger : smaller; + } + } + + if (dimension >= 3) { + // Shared faces, converted from the local face's vertex order to the stored order + // by scattering through Orient[o] (see the function documentation). A group can + // share both triangles and quadrilaterals only in mixed meshes, which are not + // supported, but both lists are always written so the layout does not depend on + // the element type. + const int shared_triangle_count = mesh.GroupNTriangles(group_index); + int* shared_triangles_out = CreateOwnedIntArray( + shared_group, "triangles", 3 * static_cast(shared_triangle_count)); + for (int i = 0; i < shared_triangle_count; ++i) { + int local_face = -1; + int orientation = 0; + mesh.GroupTriangle(group_index, i, local_face, orientation); + mesh.GetFaceVertices(local_face, local_entity_vertices); + for (int j = 0; j < 3; ++j) { + shared_triangles_out[3 * i + triangle_orient[orientation][j]] = + local_entity_vertices[j]; + } + } + + const int shared_quad_count = mesh.GroupNQuadrilaterals(group_index); + int* shared_quads_out = CreateOwnedIntArray( + shared_group, "quadrilaterals", 4 * static_cast(shared_quad_count)); + for (int i = 0; i < shared_quad_count; ++i) { + int local_face = -1; + int orientation = 0; + mesh.GroupQuadrilateral(group_index, i, local_face, orientation); + mesh.GetFaceVertices(local_face, local_entity_vertices); + for (int j = 0; j < 4; ++j) { + shared_quads_out[4 * i + quad_orient[orientation][j]] = + local_entity_vertices[j]; + } + } + } + } +} + +/** + * @brief Write the ExaConstit metadata and validation counts under `exaconstit/` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param element_geometry Mesh-wide element geometry + * @param boundary_geometry Mesh-wide boundary geometry (-1 if there are no boundary elements) + * + * @details The format version lets the reader reject layouts it does not understand. The + * dimensions and geometries are what the reader needs to size and create the mesh, and the + * topology counts are compared after the rebuild (see MeshTopologyCounts). The vertex, + * element and boundary element counts also give the expected lengths of the stored arrays. + */ +void WriteMeshMetadata(const mfem::ParMesh& mesh, + Group* mesh_group, + const int element_geometry, + const int boundary_geometry) { + Group* metadata = mesh_group->createGroup("exaconstit"); + metadata->createViewScalar("format_version", MESH_CHECKPOINT_FORMAT_VERSION); + metadata->createViewScalar("dimension", mesh.Dimension()); + metadata->createViewScalar("space_dimension", mesh.SpaceDimension()); + metadata->createViewScalar("element_geometry", element_geometry); + metadata->createViewScalar("boundary_geometry", boundary_geometry); + const MeshTopologyCounts counts = CountMeshTopology(mesh); + metadata->createViewScalar("num_vertices", counts.num_vertices); + metadata->createViewScalar("num_elements", counts.num_elements); + metadata->createViewScalar("num_boundary_elements", counts.num_boundary_elements); + metadata->createViewScalar("num_edges", counts.num_edges); + metadata->createViewScalar("num_faces", counts.num_faces); + metadata->createViewScalar("num_communication_groups", counts.num_communication_groups); +} + +} // namespace + +// ========================================================================= +// TOPOLOGY COUNTS +// ========================================================================= + +bool MeshTopologyCounts::Matches(const MeshTopologyCounts& other) const { + return num_vertices == other.num_vertices && num_elements == other.num_elements && + num_boundary_elements == other.num_boundary_elements && num_edges == other.num_edges && + num_faces == other.num_faces && + num_communication_groups == other.num_communication_groups; +} + +MeshTopologyCounts CountMeshTopology(const mfem::ParMesh& mesh) { + MeshTopologyCounts counts; + counts.num_vertices = mesh.GetNV(); + counts.num_elements = mesh.GetNE(); + counts.num_boundary_elements = mesh.GetNBE(); + counts.num_edges = mesh.GetNEdges(); + counts.num_faces = mesh.GetNFaces(); + // GetNGroups() includes group 0 (this rank alone), which shares nothing. + counts.num_communication_groups = mesh.GetNGroups() - 1; + return counts; +} + +MeshTopologyCounts ReadStoredMeshTopologyCounts(Group* mesh_group) { + Group* metadata = mesh_group->getGroup("exaconstit"); + MeshTopologyCounts counts; + counts.num_vertices = ReadSidreScalar(metadata, "num_vertices"); + counts.num_elements = ReadSidreScalar(metadata, "num_elements"); + counts.num_boundary_elements = ReadSidreScalar(metadata, "num_boundary_elements"); + counts.num_edges = ReadSidreScalar(metadata, "num_edges"); + counts.num_faces = ReadSidreScalar(metadata, "num_faces"); + counts.num_communication_groups = ReadSidreScalar(metadata, + "num_communication_groups"); + return counts; +} + +// ========================================================================= +// WRITER +// ========================================================================= + +/** + * @details Implementation notes. The writer records exactly what this rank's ParMesh holds, + * with no renumbering of any kind, because the reader has to reproduce every numbering and + * ordering listed in the background notes at the top of this file. Each piece is written by + * its own helper, documented above: + * - AgreeOnMeshGeometries(): the only collective step (MPI_Allreduce). + * - WriteCoordinates(): local vertex positions. + * - WriteVolumeTopology(): element connectivity and attributes (region IDs). + * - WriteBoundaryTopology(): boundary element connectivity and attributes. + * - WriteAdjacencySets(): communication groups and shared entities in stored orientation. + * - WriteMeshMetadata(): format version, dimensions, geometries and validation counts. + * + * Nonconforming and NURBS meshes keep extra structures (NCMesh, NURBSExtension) that this + * layout does not describe, so they are rejected up front rather than written incompletely. + */ +void WriteParMeshBlueprint(const mfem::ParMesh& mesh, Group* mesh_group) { + CALI_CXX_MARK_SCOPE("exaconstit::WriteParMeshBlueprint"); + + MFEM_VERIFY(mesh_group != nullptr, "WriteParMeshBlueprint was given a null mesh group"); + MFEM_VERIFY_0(!mesh.Nonconforming() && mesh.NURBSext == nullptr, + "Restart checkpoints do not support nonconforming or NURBS meshes"); + + const auto [element_geometry, boundary_geometry] = AgreeOnMeshGeometries(mesh); + WriteCoordinates(mesh, mesh_group); + WriteVolumeTopology(mesh, mesh_group, element_geometry); + WriteBoundaryTopology(mesh, mesh_group, boundary_geometry); + WriteAdjacencySets(mesh, mesh_group); + WriteMeshMetadata(mesh, mesh_group, element_geometry, boundary_geometry); +} + +// ========================================================================= +// READER +// ========================================================================= + +/** + * @details Implementation notes. This replaces MFEM's partitioning ParMesh constructor for a + * restarted run. That constructor builds the local mesh piece and the parallel structures + * from the global serial mesh. Here they are rebuilt from what each rank wrote, using the same + * sequence of protected Mesh / ParMesh calls the MFEM constructors use, in the same order. + * Read the background notes at the top of this file first; the steps below refer to them. + * + * The order of the steps is not arbitrary: + * - The communicator has to be set before the first collective call. + * - Every array is checked before anything is built, so a bad checkpoint aborts on every rank + * instead of crashing one rank halfway through construction while the others wait in a + * collective call. + * - FinalizeTopology() has to run before the shared structures are rebuilt, and Finalize() + * after, because Finalize() looks up the local edge and face of every shared entity in the + * edge and face tables FinalizeTopology() built. + * + * Any change here should be checked with test_restart_mesh_roundtrip, which compares the + * rebuilt mesh with the original on 1 to 4 ranks, including shared-entity orientations and the + * true-DOF layout of H1 spaces up to order 4. + */ +ExaSidreParMesh::ExaSidreParMesh(Group* mesh_group, MPI_Comm comm) { + CALI_CXX_MARK_SCOPE("exaconstit::ExaSidreParMesh"); + + MFEM_VERIFY(mesh_group != nullptr, "ExaSidreParMesh was given a null mesh group"); + MFEM_VERIFY(comm != MPI_COMM_NULL, "ExaSidreParMesh was given a null communicator"); + const std::string abort_summary = "Unable to rebuild the mesh from the restart checkpoint"; + + // --------------------------------------------------------------------- + // Step 0: communicator + // + // The ParMesh base class was default constructed, which leaves MyComm, NRanks, MyRank + // and gtopo's communicator unset. They must be set before the first collective call: + // FinalizeTopology() calls ParMesh::HasBoundaryElements() and ParMesh::SetAttributes(), + // both of which use MyComm, and BuildSharedStructures() uses gtopo's communicator. The + // MFEM constructors do the same thing first. + // --------------------------------------------------------------------- + MyComm = comm; + gtopo.SetComm(comm); + MPI_Comm_size(MyComm, &NRanks); + MPI_Comm_rank(MyComm, &MyRank); + + // --------------------------------------------------------------------- + // Step 1: metadata + // + // The format version comes first, since nothing else can be interpreted without it. The + // dimensions, geometries and local sizes then determine how many entries each array must + // have. A boundary geometry of -1 means no rank had boundary elements when the checkpoint + // was written, which is only consistent if this rank has none either. + // --------------------------------------------------------------------- + const bool has_metadata = mesh_group->hasGroup("exaconstit"); + int format_version = -1; + if (has_metadata) { + format_version = ReadSidreScalar(mesh_group->getGroup("exaconstit"), "format_version"); + } + VerifyOnAllRanks(has_metadata && format_version == MESH_CHECKPOINT_FORMAT_VERSION, + comm, + "the mesh group has no exaconstit/ metadata or an unsupported " + "format_version (" + + std::to_string(format_version) + ", expected " + + std::to_string(MESH_CHECKPOINT_FORMAT_VERSION) + ")", + abort_summary); + + Group* metadata = mesh_group->getGroup("exaconstit"); + const int dimension = ReadSidreScalar(metadata, "dimension"); + const int space_dimension = ReadSidreScalar(metadata, "space_dimension"); + const int element_geometry = ReadSidreScalar(metadata, "element_geometry"); + const int boundary_geometry = ReadSidreScalar(metadata, "boundary_geometry"); + const MeshTopologyCounts stored_counts = ReadStoredMeshTopologyCounts(mesh_group); + const int num_vertices = static_cast(stored_counts.num_vertices); + const int num_elements = static_cast(stored_counts.num_elements); + const int num_boundary_elements = static_cast(stored_counts.num_boundary_elements); + + const bool geometry_supported = element_geometry >= 0 && + element_geometry < mfem::Geometry::NUM_GEOMETRIES && + (boundary_geometry < 0 || + boundary_geometry < mfem::Geometry::NUM_GEOMETRIES) && + (boundary_geometry >= 0 || num_boundary_elements == 0); + VerifyOnAllRanks(geometry_supported, + comm, + "the mesh metadata has an invalid element or boundary geometry", + abort_summary); + + const int vertices_per_element = mfem::Geometry::NumVerts[element_geometry]; + const int vertices_per_boundary_element = boundary_geometry >= 0 + ? mfem::Geometry::NumVerts[boundary_geometry] + : 0; + + // --------------------------------------------------------------------- + // Step 2: check every array before building anything + // + // Each array must exist, be owned by Sidre (an external view would have no data after + // load), have the expected type, and have exactly the length the metadata implies. The + // adjacency-set arrays are checked in BuildSharedStructures(); only the presence of their + // parent group is checked here. All results are combined into one flag and agreed on by + // every rank, so either every rank goes on to build the mesh or every rank aborts. + // --------------------------------------------------------------------- + bool arrays_ok = true; + std::array axis_values = {nullptr, nullptr, nullptr}; + const std::array axis_names = {"x", "y", "z"}; + for (int axis = 0; axis < space_dimension; ++axis) { + const auto [values, ok] = ReadOwnedArray( + mesh_group, std::string("coordsets/coords/values/") + axis_names[axis], num_vertices); + axis_values[axis] = values; + arrays_ok = arrays_ok && ok; + } + const auto [element_connectivity, connectivity_ok] = ReadOwnedArray( + mesh_group, + "topologies/mesh/elements/connectivity", + static_cast(num_elements) * vertices_per_element); + const auto [element_attributes, element_attributes_ok] = ReadOwnedArray( + mesh_group, "fields/mesh_material_attribute/values", num_elements); + arrays_ok = arrays_ok && connectivity_ok && element_attributes_ok; + + const int* boundary_connectivity = nullptr; + const int* boundary_attributes = nullptr; + if (boundary_geometry >= 0) { + const auto [connectivity, boundary_connectivity_ok] = ReadOwnedArray( + mesh_group, + "topologies/boundary/elements/connectivity", + static_cast(num_boundary_elements) * vertices_per_boundary_element); + const auto [attributes, boundary_attributes_ok] = ReadOwnedArray( + mesh_group, "fields/boundary_material_attribute/values", num_boundary_elements); + boundary_connectivity = connectivity; + boundary_attributes = attributes; + arrays_ok = arrays_ok && boundary_connectivity_ok && boundary_attributes_ok; + } + arrays_ok = arrays_ok && mesh_group->hasGroup("adjsets/mesh/groups"); + VerifyOnAllRanks(arrays_ok, + comm, + "the mesh arrays are missing, external, or have unexpected sizes or types", + abort_summary); + + // --------------------------------------------------------------------- + // Step 3: build the local (serial) mesh piece + // + // InitMesh() only reserves space; AddVertex(), AddElement() and AddBdrElement() then + // append in order, so local vertex i, element i and boundary element i get exactly the + // indices they had in the writing run. Every value is copied into storage MFEM owns: + // AddVertex() copies the coordinates, and SetVertices() copies the vertex indices into + // the element MFEM allocated with NewElement(). Nothing points into the Sidre buffers + // afterwards. The Element objects are owned by the mesh once added and are deleted with + // it. + // + // InitMesh() takes the counts from the metadata, but AddVertex() and AddElement() grow + // the arrays as needed, so the counts are only a size hint; the array checks in step 2 + // are what guarantee the loops below stay in bounds. + // --------------------------------------------------------------------- + InitMesh(dimension, space_dimension, num_vertices, num_elements, num_boundary_elements); + for (int vertex_index = 0; vertex_index < num_vertices; ++vertex_index) { + mfem::real_t coordinates[3] = {0.0, 0.0, 0.0}; + for (int axis = 0; axis < space_dimension; ++axis) { + coordinates[axis] = axis_values[axis][vertex_index]; + } + AddVertex(coordinates); + } + for (int element_index = 0; element_index < num_elements; ++element_index) { + mfem::Element* element = NewElement(element_geometry); + element->SetVertices(element_connectivity + + static_cast(element_index) * vertices_per_element); + element->SetAttribute(element_attributes[element_index]); + AddElement(element); + } + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + mfem::Element* boundary_element = NewElement(boundary_geometry); + boundary_element->SetVertices(boundary_connectivity + + static_cast(boundary_index) * + vertices_per_boundary_element); + boundary_element->SetAttribute(boundary_attributes[boundary_index]); + AddBdrElement(boundary_element); + } + + // --------------------------------------------------------------------- + // Step 4: local topology + // + // FinalizeTopology() sets the local mesh type (SetMeshGen()), builds the element-to-face + // and element-to-edge tables, generates the local faces and edges with their local + // numbering, and sets the attribute lists. The shared-entity lookups in Finalize() (step + // 6) need these edge and face tables. + // + // generate_bdr is false because the boundary was read from the checkpoint. Generating it + // would add boundary elements with default attributes, which the boundary conditions + // would not recognize. Two calls inside are collective in a ParMesh and are reached on + // every rank: HasBoundaryElements() is an MPI_Allreduce and is evaluated before + // generate_bdr in `!HasBoundaryElements() && generate_bdr`, and SetAttributes() is + // ParMesh::SetAttributes(), which gathers the attribute lists from every rank so that + // `attributes` and `bdr_attributes` are mesh-wide. + // --------------------------------------------------------------------- + const bool generate_boundary_elements = false; + FinalizeTopology(generate_boundary_elements); + + // --------------------------------------------------------------------- + // Step 5: parallel structure + // + // FinalizeTopology() computed `meshgen` (which element families are present) from this + // rank's elements only. ReduceMeshGen() combines it across ranks with a bitwise OR, so + // every rank, including one with no elements, sees the mesh-wide value that MFEM's + // parallel code expects. The MFEM constructors do the same. + // + // BuildSharedStructures() then rebuilds the communication groups and the shared-entity + // tables; see its implementation notes. + // --------------------------------------------------------------------- + ReduceMeshGen(); + BuildSharedStructures(mesh_group->getGroup("adjsets/mesh/groups"), dimension); + + // --------------------------------------------------------------------- + // Step 6: finalize + // + // ParMesh::Finalize() calls Mesh::Finalize() and then FinalizeParTopo(), which fills in + // sedge_ledge and sface_lface (the local edge and face of every shared edge and face) by + // looking up each stored vertex tuple in the local vertex-to-vertex and face tables. + // ParMesh::Finalize() also saves and restores the mesh-wide meshgen around + // Mesh::Finalize(), which may recompute it from the local elements. + // + // Both flags must stay false. Either one would rewrite element vertex lists that were + // stored exactly as the writing run had them, and that run's mesh was already finalized: + // - fix_orientation = true lets CheckElementOrientation() reorder the vertices of any + // element it considers inverted. + // - refine = true runs MarkForRefinement(), which for tetrahedral meshes reorders every + // tet's vertices to mark its refinement edge and can rotate the shared triangles. + // Either change renumbers local DOFs relative to the writing run and can break the stored + // shared-face orientations, so the restored fields would land on the wrong DOFs. + // --------------------------------------------------------------------- + const bool refine = false; + const bool fix_orientation = false; + Finalize(refine, fix_orientation); +} + +/** + * @details Implementation notes. This rebuilds everything listed under "Communication + * groups", "Shared entities" and "Group tables" in the background notes at the top of this + * file, from the per-group arrays WriteAdjacencySets() wrote. It does the same job as the + * group-building part of MFEM's partitioning ParMesh constructor (FindSharedFaces(), + * FindSharedEdges(), FindSharedVertices(), gtopo.Create(), BuildFaceGroup(), BuildEdgeGroup(), + * BuildVertexGroup() and the shared-element builders), except that the groups and entities + * are read rather than discovered. + * + * Three properties carry over from the writing run, and the rest of the parallel machinery + * depends on each of them: + * - Local group order: group g is rebuilt with local index g (from `local_group_index`). This + * fixes the lproc numbering and the communication pattern, so the restarted run exchanges + * data exactly as the original run did. + * - Per-group entity order: each table row lists its entities in the stored order, which is + * the order every member rank agrees on. + * - Stored orientation: the shared edge and face tuples are taken verbatim, so GroupEdge(), + * GroupTriangle() and GroupQuadrilateral() report the original orientations. + * + * `sedge_ledge` and `sface_lface` are only sized here. Their values depend on the local edge + * and face numbering, and Finalize() computes them from the stored tuples. + */ +void ExaSidreParMesh::BuildSharedStructures(Group* adjset_groups, const int dimension) { + CALI_CXX_MARK_SCOPE("exaconstit::ExaSidreParMesh::BuildSharedStructures"); + + // Pointers into the Sidre arrays of one communication group, plus their tuple counts. + struct SharedGroupArrays { + const int* neighbor_ranks = nullptr; + IndexType neighbor_count = 0; + const int* vertices = nullptr; + IndexType vertex_count = 0; + const int* edges = nullptr; + IndexType edge_count = 0; + const int* triangles = nullptr; + IndexType triangle_count = 0; + const int* quadrilaterals = nullptr; + IndexType quadrilateral_count = 0; + }; + + // --------------------------------------------------------------------- + // Read the stored groups into their original local order + // + // shared_groups[g - 1] receives the group whose stored local_group_index is g, no matter + // in what order Sidre returns the groups. Each index 1..group_count has to appear exactly + // once; a missing, duplicated or out-of-range index means the adjacency sets are corrupt. + // The tuple sizes (1 for vertices and neighbors, 2 for edges, 3 for triangles, 4 for + // quadrilaterals) are checked by ReadIntTuples(). + // --------------------------------------------------------------------- + const int group_count = static_cast(adjset_groups->getNumGroups()); + std::vector shared_groups(static_cast(group_count)); + std::vector group_filled(static_cast(group_count), false); + bool arrays_ok = true; + for (Group& group : adjset_groups->groups()) { + const int local_group_index = group.hasView("local_group_index") + ? ReadSidreScalar(&group, "local_group_index") + : -1; + if (local_group_index < 1 || local_group_index > group_count || + group_filled[static_cast(local_group_index - 1)]) { + arrays_ok = false; + continue; + } + group_filled[static_cast(local_group_index - 1)] = true; + SharedGroupArrays& arrays = shared_groups[static_cast(local_group_index - 1)]; + std::tie(arrays.neighbor_ranks, + arrays.neighbor_count) = ReadIntTuples(&group, "neighbors", 1, arrays_ok); + std::tie(arrays.vertices, + arrays.vertex_count) = ReadIntTuples(&group, "values", 1, arrays_ok); + if (dimension >= 2) { + std::tie(arrays.edges, + arrays.edge_count) = ReadIntTuples(&group, "edges", 2, arrays_ok); + } + if (dimension >= 3) { + std::tie(arrays.triangles, + arrays.triangle_count) = ReadIntTuples(&group, "triangles", 3, arrays_ok); + std::tie(arrays.quadrilaterals, arrays.quadrilateral_count) = ReadIntTuples( + &group, "quadrilaterals", 4, arrays_ok); + } + } + + // --------------------------------------------------------------------- + // Rebuild the group topology + // + // GroupTopology::Create() takes the groups as a list of rank sets, in local group order, + // with this rank alone first (group 0). Each IntegerSet must contain MPI ranks in sorted + // order: Create() picks each group's master with ListOfIntegerSets::PickElementInSet(), + // which returns the set's first entry, so sorting is what makes the master the smallest + // rank, the same choice MFEM's partitioning constructor makes on every member rank. + // + // ListOfIntegerSets::Insert() returns the index of an equal set if one is already in the + // list instead of adding a new one. Two stored groups with the same members would + // therefore collapse into one and shift every later group's index, so the returned index + // is checked against the expected one. A valid mesh never has duplicate groups. + // + // Create() exchanges group information with each neighbor rank (each non-master member + // learns the group's index on its master), so every rank must reach it. That is why the + // array checks above are agreed on across ranks first. The MPI tag is the one MFEM's + // partitioning constructor uses for the same call. + // --------------------------------------------------------------------- + mfem::ListOfIntegerSets communication_sets; + { + mfem::IntegerSet local_only_set; + mfem::Array& local_only = local_only_set; + local_only.Append(MyRank); + communication_sets.Insert(local_only_set); + } + for (int group_offset = 0; group_offset < group_count; ++group_offset) { + const SharedGroupArrays& arrays = shared_groups[static_cast(group_offset)]; + mfem::IntegerSet member_set; + mfem::Array& members = member_set; + members.Append(MyRank); + members.Append(arrays.neighbor_ranks, static_cast(arrays.neighbor_count)); + members.Sort(); + const int inserted_index = communication_sets.Insert(member_set); + arrays_ok = arrays_ok && (inserted_index == group_offset + 1); + } + VerifyOnAllRanks(arrays_ok, + MyComm, + "the adjacency-set arrays are malformed, have missing or duplicate local " + "group indices, or contain duplicate communication groups", + "Unable to rebuild the mesh from the restart checkpoint"); + const int group_topology_mpi_tag = 822; + gtopo.Create(communication_sets, group_topology_mpi_tag); + + // --------------------------------------------------------------------- + // Size the shared-entity arrays and group tables + // + // The shared-entity arrays (svert_lvert, shared_edges, shared_trias, shared_quads) hold + // every group's entities back to back, group 1 first. The group tables have one row per + // group 1..N (row r is group r+1), and each row lists the indices of that group's + // entities in those arrays. Since the arrays are filled group by group in row order, each + // row's indices are simply consecutive. + // + // ParFiniteElementSpace::GetGroupComm() calls GroupNTriangles() and GroupNQuadrilaterals() + // for every group in every dimension, so below 3D the triangle and quadrilateral tables + // still need group_count (empty) rows instead of being left unsized. The partitioning + // constructor also sizes them in every dimension (BuildFaceGroup()). + // --------------------------------------------------------------------- + IndexType total_shared_vertices = 0; + IndexType total_shared_edges = 0; + IndexType total_shared_faces = 0; + for (const SharedGroupArrays& arrays : shared_groups) { + total_shared_vertices += arrays.vertex_count; + total_shared_edges += arrays.edge_count; + total_shared_faces += arrays.triangle_count + arrays.quadrilateral_count; + } + + svert_lvert.SetSize(static_cast(total_shared_vertices)); + group_svert.SetDims(group_count, static_cast(total_shared_vertices)); + if (dimension >= 2) { + sedge_ledge.SetSize(static_cast(total_shared_edges)); + shared_edges.SetSize(static_cast(total_shared_edges)); + group_sedge.SetDims(group_count, static_cast(total_shared_edges)); + } + if (dimension >= 3) { + sface_lface.SetSize(static_cast(total_shared_faces)); + group_stria.MakeI(group_count); + group_squad.MakeI(group_count); + } else { + group_stria.SetSize(group_count, 0); + group_squad.SetSize(group_count, 0); + } + + // --------------------------------------------------------------------- + // Fill the shared-entity arrays and group tables, group by group in local order + // + // Vertex and edge tables are written directly in compressed-row form. Table::SetDims() + // allocated I (row offsets, size group_count + 1) and J (column entries) and set I[0] = 0; + // after row r is filled, I[r + 1] is set to the running total, which is both the end of + // row r and the start of row r + 1. J receives consecutive indices into svert_lvert and + // shared_edges. + // + // The face tables use mfem::Table's two-pass construction instead, because triangles and + // quadrilaterals are counted per row first: MakeI() above zeroed the per-row counts, + // AddColumnsInRow() records each row's count here, and MakeJ() below turns the counts + // into offsets. + // + // Within each group the entities are appended in stored order, which is the order every + // member rank agrees on, and the edge and face tuples are copied verbatim, keeping their + // stored orientation. Each shared edge becomes an mfem::Segment owned by the mesh, + // with attribute 1 as in MFEM's own constructors. Shared faces are plain vertex tuples + // (Vert3 / Vert4). + // --------------------------------------------------------------------- + int vertices_added = 0; + int edges_added = 0; + for (int row = 0; row < group_count; ++row) { + const SharedGroupArrays& arrays = shared_groups[static_cast(row)]; + + for (IndexType i = 0; i < arrays.vertex_count; ++i) { + group_svert.GetJ()[vertices_added] = vertices_added; + svert_lvert[vertices_added] = arrays.vertices[i]; + ++vertices_added; + } + group_svert.GetI()[row + 1] = vertices_added; + + if (dimension >= 2) { + for (IndexType i = 0; i < arrays.edge_count; ++i) { + group_sedge.GetJ()[edges_added] = edges_added; + shared_edges[edges_added] = new mfem::Segment(arrays.edges + 2 * i, 1); + ++edges_added; + } + group_sedge.GetI()[row + 1] = edges_added; + } + + if (dimension >= 3) { + for (IndexType i = 0; i < arrays.triangle_count; ++i) { + const int* v = arrays.triangles + 3 * i; + shared_trias.Append(Vert3(v[0], v[1], v[2])); + } + for (IndexType i = 0; i < arrays.quadrilateral_count; ++i) { + const int* v = arrays.quadrilaterals + 4 * i; + shared_quads.Append(Vert4(v[0], v[1], v[2], v[3])); + } + group_stria.AddColumnsInRow(row, static_cast(arrays.triangle_count)); + group_squad.AddColumnsInRow(row, static_cast(arrays.quadrilateral_count)); + } + } + + if (dimension >= 3) { + // sface_lface holds the triangles first and then the quadrilaterals, so its size has + // to equal the two lists combined for FinalizeParTopo() to index it correctly. + MFEM_VERIFY(shared_trias.Size() + shared_quads.Size() == sface_lface.Size(), + "Shared face count mismatch while rebuilding the mesh from the restart " + "checkpoint"); + // Row offsets from the per-row counts, then consecutive column entries: triangle + // (or quadrilateral) j of the whole list is entry j of the table. + group_stria.MakeJ(); + for (int i = 0; i < shared_trias.Size(); ++i) { + group_stria.GetJ()[i] = i; + } + group_squad.MakeJ(); + for (int i = 0; i < shared_quads.Size(); ++i) { + group_squad.GetJ()[i] = i; + } + } +} + +} // namespace exaconstit + +#endif // EXACONSTIT_USE_AXOM diff --git a/src/sim_state/exa_sidre_parmesh.hpp b/src/sim_state/exa_sidre_parmesh.hpp new file mode 100644 index 00000000..54e986f5 --- /dev/null +++ b/src/sim_state/exa_sidre_parmesh.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include "mfem.hpp" + +#include +#include +#include + +namespace axom { +namespace sidre { +class Group; +} // namespace sidre +} // namespace axom + +namespace exaconstit { + +/** + * @brief Version of the mesh subtree layout written by WriteParMeshBlueprint() + * + * Stored in `mesh/exaconstit/format_version` of every checkpoint rank file. Bump it whenever + * the layout written by WriteParMeshBlueprint() changes, so older checkpoints are rejected + * with a clear message instead of being misread. + */ +inline constexpr int MESH_CHECKPOINT_FORMAT_VERSION = 1; + +/** + * @brief Per-rank topology counts used to validate a reconstructed mesh + * + * @details Edges and faces are not stored in a checkpoint; MFEM regenerates them from the + * element connectivity. Comparing these counts with the ones recorded at write time checks + * that the regenerated local topology matches the original. + */ +struct MeshTopologyCounts { + /** @brief Number of local vertices (Mesh::GetNV) */ + std::int64_t num_vertices = 0; + /** @brief Number of local elements (Mesh::GetNE) */ + std::int64_t num_elements = 0; + /** @brief Number of local boundary elements (Mesh::GetNBE) */ + std::int64_t num_boundary_elements = 0; + /** @brief Number of local edges (Mesh::GetNEdges) */ + std::int64_t num_edges = 0; + /** @brief Number of local faces (Mesh::GetNFaces) */ + std::int64_t num_faces = 0; + /** @brief Number of communication groups, not counting group 0 (this rank alone) */ + std::int64_t num_communication_groups = 0; + + /** + * @brief Check whether every count agrees with another set of counts + * + * @param other Counts to compare against + * @return True if all six counts are equal + */ + bool Matches(const MeshTopologyCounts& other) const; +}; + +/** + * @brief Count the local topology of a parallel mesh + * + * @param mesh Parallel mesh to count + * @return Topology counts of this rank's piece of the mesh + * + * @details Purely local; no MPI communication takes place. + */ +MeshTopologyCounts CountMeshTopology(const mfem::ParMesh& mesh); + +/** + * @brief Read the topology counts recorded by WriteParMeshBlueprint() + * + * @param mesh_group Sidre group that WriteParMeshBlueprint() wrote into + * @return Topology counts stored under `exaconstit/` in the group + */ +MeshTopologyCounts ReadStoredMeshTopologyCounts(axom::sidre::Group* mesh_group); + +/** + * @brief Write this rank's piece of a parallel mesh into a Sidre group + * + * @param mesh Parallel mesh to write + * @param mesh_group Empty Sidre group to write into + * + * @details Writes the local vertices, elements, boundary elements and attributes, plus the + * description of how this piece connects to the other ranks' pieces (communication groups + * and shared vertices, edges and faces), so that ExaSidreParMesh can rebuild the identical + * ParMesh on the same rank. The layout follows the Conduit Blueprint mesh conventions, with + * ExaConstit additions under `adjsets/` and `exaconstit/`. All arrays are owned by Sidre. + * + * Collective over mesh.GetComm(). + * + * @pre The mesh is conforming and not NURBS, all elements share one geometry and all boundary + * elements share one geometry (segment, triangle, quadrilateral, tetrahedron or hexahedron), + * and mfem::real_t is double. Violations abort on every rank. + */ +void WriteParMeshBlueprint(const mfem::ParMesh& mesh, axom::sidre::Group* mesh_group); + +/** + * @brief Parallel mesh rebuilt from the output of WriteParMeshBlueprint() + * + * @details Restarted runs use this in place of reading the original mesh file, which may no + * longer exist. The result is a finalized, conforming mfem::ParMesh identical to the one that + * was written: the same local vertex and element numbering, element vertex order, + * attributes, communication groups, shared entities and shared-entity orientation. Finite + * element spaces built on it therefore number their local and true DOFs exactly as in the + * writing run. + * + * Invariants callers must respect: + * - The run must use the same number of MPI ranks as the writing run, and rank r must read + * the group that rank r wrote. + * - The mesh has no Nodes grid function. Code that needs high-order or deformed coordinates + * must supply them (SimulationState overlays the checkpointed coordinates). + * + * This is a subclass only because the members needed to rebuild a ParMesh from its pieces + * are protected; it adds nothing beyond its constructor. + */ +class ExaSidreParMesh : public mfem::ParMesh { +public: + /** + * @brief Rebuild a parallel mesh from a checkpoint's mesh group + * + * @param mesh_group Group written by WriteParMeshBlueprint(). Non-const because Sidre's + * typed accessor View::getData() is non-const. + * @param comm Communicator with the same number of ranks as the writing run + * + * @details Collective over comm. Every array is checked on every rank before anything is + * built, and a bad checkpoint aborts on every rank. Nothing points into the Sidre buffers + * once the constructor returns, so the DataStore holding mesh_group may be destroyed + * right away. + */ + ExaSidreParMesh(axom::sidre::Group* mesh_group, MPI_Comm comm); + + ExaSidreParMesh(const ExaSidreParMesh&) = delete; + ExaSidreParMesh& operator=(const ExaSidreParMesh&) = delete; + ~ExaSidreParMesh() override = default; + + /** + * @brief Rebuild a parallel mesh and return it as a shared pointer + * + * @param mesh_group Group written by WriteParMeshBlueprint() + * @param comm Communicator with the same number of ranks as the writing run + * @return Shared pointer to the rebuilt mesh, ready to assign to SimulationState's mesh + */ + static std::shared_ptr Create(axom::sidre::Group* mesh_group, MPI_Comm comm) { + return std::make_shared(mesh_group, comm); + } + +private: + /** + * @brief Rebuild the communication groups and the shared-entity tables + * + * @param adjset_groups The `adjsets/mesh/groups` group written by WriteParMeshBlueprint() + * @param dimension Topological dimension of the mesh + * + * @details Collective over the mesh communicator. Must be called after ReduceMeshGen() + * and before Finalize(). + */ + void BuildSharedStructures(axom::sidre::Group* adjset_groups, int dimension); +}; + +} // namespace exaconstit diff --git a/src/sim_state/restart_sidre_utils.hpp b/src/sim_state/restart_sidre_utils.hpp new file mode 100644 index 00000000..7fe77b9a --- /dev/null +++ b/src/sim_state/restart_sidre_utils.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include "utilities/unified_logger.hpp" + +#include "mfem.hpp" + +#include +#include +#include + +#include "axom/sidre.hpp" + +/** + * @brief Small helpers shared by the checkpoint/restart implementation files + * + * @details Only included from exa_sidre_parmesh.cpp and restart_checkpoint.cpp, which are + * compiled when ExaConstit is built with Axom (EXACONSTIT_USE_AXOM). Not part of the public + * restart interface. + */ +namespace exaconstit::restart_detail { + +/** + * @brief Read a scalar view, converting from whatever numeric type it was stored as + * + * @tparam ScalarType Type to return (integer, unsigned integer or floating point) + * @param group Group that holds the view + * @param path Path of the scalar view relative to the group + * @return The stored value converted to ScalarType + * + * @details View::getScalar() does not convert between integer widths, and the JSON protocol + * does not preserve them: Conduit's JSON reader brings every integer back as int64, so an int + * written to `commit.json` as 1 reads back as 0 through getScalar() into an int. Going through + * conduit::Node::to_int64(), to_uint64() and to_float64() makes every read independent of the + * protocol and of the exact type the writer used. + */ +template +ScalarType ReadSidreScalar(axom::sidre::Group* group, const std::string& path) { + const auto& node = group->getView(path)->getNode(); + if constexpr (std::is_floating_point::value) { + return static_cast(node.to_float64()); + } else if constexpr (std::is_unsigned::value) { + return static_cast(node.to_uint64()); + } else { + return static_cast(node.to_int64()); + } +} + +/** + * @brief Check whether a condition holds on every rank + * + * @param local_value This rank's result + * @param comm Communicator to reduce over + * @return True on every rank if local_value is true on every rank, false on every rank + * otherwise + * + * @details Collective over comm. Used to turn rank-local checks into a decision every rank + * agrees on, so that either all ranks continue or all ranks abort together. Aborting on only + * the ranks that failed would leave the others waiting forever in their next collective call. + */ +inline bool AllRanksTrue(const bool local_value, MPI_Comm comm) { + int local_flag = local_value ? 1 : 0; + int global_flag = 0; + MPI_Allreduce(&local_flag, &global_flag, 1, MPI_INT, MPI_MIN, comm); + return global_flag == 1; +} + +/** + * @brief Abort on every rank if a rank-local check failed on any rank + * + * @param local_ok This rank's result + * @param comm Communicator to reduce over + * @param local_message Description of the problem, reported by each rank where it occurred + * @param summary Message rank 0 aborts with, shared by every rank + * + * @details Collective over comm. Each failing rank prints its own warning, since only that + * rank knows the details, and then every rank aborts with the same summary through + * MFEM_ABORT_0, so the summary appears once. + */ +inline void VerifyOnAllRanks(const bool local_ok, + MPI_Comm comm, + const std::string& local_message, + const std::string& summary) { + if (AllRanksTrue(local_ok, comm)) { + return; + } + if (!local_ok) { + int rank = 0; + MPI_Comm_rank(comm, &rank); + MFEM_WARNING("Rank " << rank << ": " << local_message); + } + MFEM_ABORT_0(summary << " (see the per-rank warnings above)"); +} + +} // namespace exaconstit::restart_detail diff --git a/src/sim_state/simulation_state.hpp b/src/sim_state/simulation_state.hpp index bd88bedd..6d9eb905 100644 --- a/src/sim_state/simulation_state.hpp +++ b/src/sim_state/simulation_state.hpp @@ -459,14 +459,6 @@ class SimulationState { /** @brief Reference to simulation options */ ExaOptions& m_options; -#if defined(EXACONSTIT_USE_AXOM) - // We want this to be something akin to a axom::sidre::MFEMSidreDataCollection - // However, we need it flexible enough to handle multiple different mesh topologies in it that - // we might due to different mfem::SubMesh objects that correspond to each - // PartialQuadraturePoint - /** @brief Simulation restart data store (optional Axom/Sidre support) */ - std::unique_ptr m_simulation_restart; -#endif /** @brief MPI rank identifier */ int my_id; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c521415d..63a29eb4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,6 +47,36 @@ blt_add_executable(NAME test_grad_oper blt_add_test(NAME test_gradient_operation COMMAND test_grad_oper) +#################################### +# Checkpoint/restart tests (Axom Sidre) +#################################### +if(ENABLE_AXOM) + # Mesh round trip through the checkpoint mesh I/O, including a case with empty ranks + blt_add_executable(NAME test_restart_mesh_roundtrip + SOURCES test_restart_mesh_roundtrip.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${EXACONSTIT_TEST_DEPENDS} gtest) + + # MPI tests never use more ranks than the machine has cores, so the suite does not + # oversubscribe small machines such as the 4-core CI runners (Open MPI refuses to start more + # ranks than slots by default). The default is the number of logical cores CMake detects; + # set EXACONSTIT_TEST_MAX_MPI_TASKS to override it. Rank counts above the cap are skipped. + cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_LOGICAL_CORES) + if(NOT _detected_cores OR _detected_cores LESS 1) + set(_detected_cores 1) + endif() + set(EXACONSTIT_TEST_MAX_MPI_TASKS ${_detected_cores} CACHE STRING + "Largest number of MPI ranks any test may use (default: detected logical cores)") + + foreach(_np 1 2 3 4) + if(NOT _np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + blt_add_test(NAME test_restart_mesh_roundtrip_np${_np} + COMMAND test_restart_mesh_roundtrip + NUM_MPI_TASKS ${_np}) + endif() + endforeach() +endif() + if (NOT DEFINED PYTHON_EXECUTABLE) message(FATAL_ERROR "cmake: PYTHON_EXECUTABLE must be defined if running test suite current value: ${PYTHON_EXECUTABLE}") endif() diff --git a/test/test_restart_mesh_roundtrip.cpp b/test/test_restart_mesh_roundtrip.cpp new file mode 100644 index 00000000..fa7a0c3f --- /dev/null +++ b/test/test_restart_mesh_roundtrip.cpp @@ -0,0 +1,536 @@ +#include "sim_state/exa_sidre_parmesh.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" +#include + +// Mesh round-trip test for the restart checkpoint mesh I/O (WriteParMeshBlueprint and +// ExaSidreParMesh, src/sim_state/exa_sidre_parmesh.*). +// +// Each case partitions a small perturbed serial mesh, writes every rank's piece with +// WriteParMeshBlueprint(), saves it to HDF5, loads it back into a fresh DataStore, rebuilds the +// parallel mesh with ExaSidreParMesh, and then checks that the rebuilt mesh is identical to the +// original in every way the finite element spaces depend on: +// - local topology counts and the global element count; +// - every communication group: its master and, entity by entity in group order, its shared +// vertices, edges and faces, including each shared edge's and face's orientation; +// - for H1 spaces of order 1 to 4: identical local, true and global true sizes, bitwise +// identical local and true-DOF vectors of a projected nonlinear field, identical +// element-local DOF hashes, and a field that survives restriction to true DOFs and +// prolongation back unchanged (shared DOFs matched consistently across ranks). +// +// Order 4 matters: a shared face then carries several interior DOFs, so a wrong shared-face +// orientation changes which DOFs are matched across ranks (see the background notes in +// exa_sidre_parmesh.cpp). The `empty_ranks` case partitions the mesh so that half of the ranks +// own no elements, covering zero-length arrays in the checkpoint and empty rebuilt meshes. +// +// Every case also runs a negative control that rewrites the shared edges and faces in each +// rank's local vertex order before saving (what axom::sidre::MFEMSidreDataCollection writes) +// and reports how far the shared-DOF round trip is then off. It is printed for information +// rather than checked, since whether a given partition happens to have mismatched +// orientations is not guaranteed; on 2 or more ranks it is typically 0.1 or larger for hexes at +// order >= 3, against ~1e-15 for the real writer. +// +// Run with any number of ranks, for example `mpirun -np 3 ./test_restart_mesh_roundtrip`. +// Scratch files go to restart_roundtrip_np_/ in the working directory. + +using axom::sidre::DataStore; +using axom::sidre::Group; + +namespace { + +/** + * @brief 64-bit FNV-1a hash, the same as the one used by restart_checkpoint.cpp + */ +class Fnv1a64 { +public: + /** + * @brief Add raw bytes to the hash + * + * @param data Bytes to add + * @param byte_count Number of bytes + */ + void AddBytes(const void* data, const std::size_t byte_count) { + const auto* bytes = static_cast(data); + for (std::size_t i = 0; i < byte_count; ++i) { + m_state ^= bytes[i]; + m_state *= 1099511628211ULL; + } + } + + /** + * @brief Current hash value + * + * @return The hash of every byte added so far + */ + std::uint64_t Value() const { + return m_state; + } + +private: + /** @brief Running hash state, starting from the FNV-1a 64-bit offset basis */ + std::uint64_t m_state = 1469598103934665603ULL; +}; + +/** + * @brief Hash of a grid function visited element by element in local DOF order + * + * @param field Grid function to hash + * @return FNV-1a hash of the values in GetElementVDofs() order (sign-flipped for reversed DOFs) + * + * @details Same definition as ElementLocalFieldHash() in restart_checkpoint.cpp. Equal hashes + * on the original and rebuilt meshes mean both number their local DOFs the same way. + */ +std::uint64_t ElementLocalFieldHash(const mfem::ParGridFunction& field) { + const mfem::ParFiniteElementSpace* space = field.ParFESpace(); + const double* values = field.HostRead(); + mfem::Array element_vdofs; + Fnv1a64 hasher; + for (int element_index = 0; element_index < space->GetNE(); ++element_index) { + space->GetElementVDofs(element_index, element_vdofs); + for (const int signed_vdof : element_vdofs) { + const int vdof = signed_vdof >= 0 ? signed_vdof : -1 - signed_vdof; + const double value = signed_vdof >= 0 ? values[vdof] : -values[vdof]; + hasher.AddBytes(&value, sizeof(value)); + } + } + return hasher.Value(); +} + +/** + * @brief Largest change of a grid function under restriction to true DOFs and prolongation + * + * @param field Grid function to test + * @return max |field - P(R(field))| over this rank's local DOFs + * + * @details Collective. Zero (up to rounding) exactly when every shared DOF is matched with + * the same true DOF on every rank; see SharedDofInconsistency() in restart_checkpoint.cpp. + */ +double SharedDofInconsistency(const mfem::ParGridFunction& field) { + mfem::Vector true_values; + field.GetTrueDofs(true_values); + mfem::ParGridFunction round_trip(field.ParFESpace()); + round_trip.SetFromTrueDofs(true_values); + double max_difference = 0.0; + for (int i = 0; i < field.Size(); ++i) { + max_difference = std::max(max_difference, std::abs(field(i) - round_trip(i))); + } + return max_difference; +} + +/** + * @brief Smooth nonlinear vector field projected onto the H1 spaces + * + * @param x Point + * @param y Field value at x + * + * @details Nonlinear in every coordinate, so every DOF of a high-order space gets a distinct + * value and any DOF permutation changes the projected vector. + */ +void TestField(const mfem::Vector& x, mfem::Vector& y) { + y(0) = std::sin(2.0 * x(0) + x(1)) + 0.3 * x(2) * x(2); + y(1) = std::cos(x(0) - 3.0 * x(2)) + x(1) * x(1) * x(0); + y(2) = x(0) * x(1) * x(2) + std::sin(1.3 * x(2) + 0.7 * x(1)); +} + +/** + * @brief Move every vertex by a small smooth offset + * + * @param mesh Serial mesh to perturb + * @param amplitude Size of the offset + * + * @details Makes the coordinates distinct and non-axis-aligned, so a mismatched vertex or DOF + * always changes a value instead of possibly landing on an equal one. + */ +void PerturbVertices(mfem::Mesh& mesh, const double amplitude) { + for (int vertex = 0; vertex < mesh.GetNV(); ++vertex) { + double* x = mesh.GetVertex(vertex); + const double x0 = x[0]; + const double x1 = x[1]; + const double x2 = x[2]; + x[0] += amplitude * std::sin(3.1 * x1 + 1.7 * x2 + 0.3); + x[1] += amplitude * std::sin(2.3 * x2 + 0.9 * x0 + 1.1); + x[2] += amplitude * std::sin(1.9 * x0 + 2.9 * x1 + 0.7); + } +} + +/** + * @brief Largest value of a double over all ranks + * + * @param value This rank's value + * @return The maximum over MPI_COMM_WORLD + */ +double GlobalMax(const double value) { + double result = 0.0; + MPI_Allreduce(&value, &result, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return result; +} + +/** + * @brief Sorted MPI ranks of a communication group + * + * @param mesh Parallel mesh + * @param group Local group index + * @return The member ranks in increasing order + * + * @details Groups are matched between the original and rebuilt meshes by their member ranks, + * so the comparison does not assume anything about local group indices. + */ +std::vector GroupRanks(const mfem::ParMesh& mesh, const int group) { + std::vector ranks; + for (int member = 0; member < mesh.gtopo.GetGroupSize(group); ++member) { + ranks.push_back(mesh.gtopo.GetNeighborRank(mesh.gtopo.GetGroup(group)[member])); + } + std::sort(ranks.begin(), ranks.end()); + return ranks; +} + +/** + * @brief Check that two meshes have identical communication groups and shared entities + * + * @param original Mesh that was written + * @param rebuilt Mesh rebuilt from the checkpoint + * + * @details For every group of the original mesh, finds the rebuilt group with the same member + * ranks and checks its master and, entity by entity in group order, its shared vertices, the + * local vertices and orientation of its shared edges, and the local vertices and orientation + * of its shared triangles and quadrilaterals. Both the order and the orientation matter (see + * the background notes in exa_sidre_parmesh.cpp). + */ +void ExpectSameSharedStructures(const mfem::ParMesh& original, const mfem::ParMesh& rebuilt) { + EXPECT_EQ(original.GetNGroups(), rebuilt.GetNGroups()) << "number of communication groups"; + std::map, int> rebuilt_group_index; + for (int group = 1; group < rebuilt.GetNGroups(); ++group) { + rebuilt_group_index[GroupRanks(rebuilt, group)] = group; + } + + mfem::Array original_vertices; + mfem::Array rebuilt_vertices; + for (int ga = 1; ga < original.GetNGroups(); ++ga) { + const auto it = rebuilt_group_index.find(GroupRanks(original, ga)); + if (it == rebuilt_group_index.end()) { + ADD_FAILURE() << "original group " << ga << " has no rebuilt group with the same ranks"; + continue; + } + const int gb = it->second; + EXPECT_EQ(original.gtopo.GetGroupMasterRank(ga), rebuilt.gtopo.GetGroupMasterRank(gb)) + << "group master of group " << ga; + + // Shared vertices, in group order. + EXPECT_EQ(original.GroupNVertices(ga), rebuilt.GroupNVertices(gb)); + for (int i = 0; i < std::min(original.GroupNVertices(ga), rebuilt.GroupNVertices(gb)); + ++i) { + EXPECT_EQ(original.GroupVertex(ga, i), rebuilt.GroupVertex(gb, i)) + << "shared vertex " << i << " of group " << ga; + } + + // Shared edges: same local edge (by its vertices) and same orientation. + EXPECT_EQ(original.GroupNEdges(ga), rebuilt.GroupNEdges(gb)); + for (int i = 0; i < std::min(original.GroupNEdges(ga), rebuilt.GroupNEdges(gb)); ++i) { + int edge_a = -1; + int orientation_a = 0; + int edge_b = -1; + int orientation_b = 0; + original.GroupEdge(ga, i, edge_a, orientation_a); + rebuilt.GroupEdge(gb, i, edge_b, orientation_b); + original.GetEdgeVertices(edge_a, original_vertices); + rebuilt.GetEdgeVertices(edge_b, rebuilt_vertices); + EXPECT_TRUE(original_vertices[0] == rebuilt_vertices[0] && + original_vertices[1] == rebuilt_vertices[1] && + orientation_a == orientation_b) + << "shared edge " << i << " of group " << ga; + } + + // Shared faces: same local face (by its vertices) and same orientation. + const auto expect_same_faces = + [&](const int count_a, const int count_b, const bool triangles, const char* label) { + EXPECT_EQ(count_a, count_b) << "number of shared " << label; + for (int i = 0; i < std::min(count_a, count_b); ++i) { + int face_a = -1; + int orientation_a = 0; + int face_b = -1; + int orientation_b = 0; + if (triangles) { + original.GroupTriangle(ga, i, face_a, orientation_a); + rebuilt.GroupTriangle(gb, i, face_b, orientation_b); + } else { + original.GroupQuadrilateral(ga, i, face_a, orientation_a); + rebuilt.GroupQuadrilateral(gb, i, face_b, orientation_b); + } + original.GetFaceVertices(face_a, original_vertices); + rebuilt.GetFaceVertices(face_b, rebuilt_vertices); + bool same = original_vertices.Size() == rebuilt_vertices.Size() && + orientation_a == orientation_b; + for (int k = 0; same && k < original_vertices.Size(); ++k) { + same = original_vertices[k] == rebuilt_vertices[k]; + } + EXPECT_TRUE(same) << "shared " << label << " " << i << " of group " << ga; + } + }; + expect_same_faces( + original.GroupNTriangles(ga), rebuilt.GroupNTriangles(gb), true, "triangle"); + expect_same_faces(original.GroupNQuadrilaterals(ga), + rebuilt.GroupNQuadrilaterals(gb), + false, + "quadrilateral"); + } +} + +/** + * @brief Overwrite the stored shared edges and faces with each rank's local vertex order + * + * @param mesh Mesh that was written + * @param mesh_group Mesh group written by WriteParMeshBlueprint() + * + * @details Reproduces what axom::sidre::MFEMSidreDataCollection writes (local order, no + * orientation), for the negative control. The arrays are edited in place, so everything else + * the writer stored, including each group's local_group_index, is unchanged. + */ +void RewriteSharedEntitiesInLocalOrder(const mfem::ParMesh& mesh, Group* mesh_group) { + Group* groups = mesh_group->getGroup("adjsets/mesh/groups"); + mfem::Array vertices; + for (int group = 1; group < mesh.GetNGroups(); ++group) { + const std::string name = "g" + std::to_string(mesh.gtopo.GetGroupMasterRank(group)) + "_" + + std::to_string(mesh.gtopo.GetGroupMasterGroup(group)); + Group* shared_group = groups->getGroup(name); + if (mesh.GroupNEdges(group) > 0) { + int* edges = shared_group->getView("edges")->getData(); + for (int i = 0; i < mesh.GroupNEdges(group); ++i) { + int edge = -1; + int orientation = 0; + mesh.GroupEdge(group, i, edge, orientation); + mesh.GetEdgeVertices(edge, vertices); + edges[2 * i] = vertices[0]; + edges[2 * i + 1] = vertices[1]; + } + } + if (mesh.GroupNTriangles(group) > 0) { + int* triangles = shared_group->getView("triangles")->getData(); + for (int i = 0; i < mesh.GroupNTriangles(group); ++i) { + int face = -1; + int orientation = 0; + mesh.GroupTriangle(group, i, face, orientation); + mesh.GetFaceVertices(face, vertices); + for (int k = 0; k < 3; ++k) { + triangles[3 * i + k] = vertices[k]; + } + } + } + if (mesh.GroupNQuadrilaterals(group) > 0) { + int* quads = shared_group->getView("quadrilaterals")->getData(); + for (int i = 0; i < mesh.GroupNQuadrilaterals(group); ++i) { + int face = -1; + int orientation = 0; + mesh.GroupQuadrilateral(group, i, face, orientation); + mesh.GetFaceVertices(face, vertices); + for (int k = 0; k < 4; ++k) { + quads[4 * i + k] = vertices[k]; + } + } + } + } +} + +/** + * @brief Write a mesh through HDF5 and rebuild it, exactly as a checkpoint and restart do + * + * @param mesh Mesh to write + * @param directory Directory for the rank files (created by rank 0) + * @param local_order_control Whether to rewrite the shared entities in local order first + * @return The rebuilt mesh + * + * @details The DataStore used for loading goes out of scope before the rebuilt mesh is used, + * which also checks that the rebuilt mesh does not point into Sidre memory. + */ +std::shared_ptr RoundTrip(const mfem::ParMesh& mesh, + const std::filesystem::path& directory, + const bool local_order_control) { + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) { + std::filesystem::create_directories(directory); + } + MPI_Barrier(MPI_COMM_WORLD); + + const std::string file = (directory / ("rank_" + std::to_string(rank) + ".hdf5")).string(); + { + DataStore store; + Group* mesh_group = store.getRoot()->createGroup("mesh"); + exaconstit::WriteParMeshBlueprint(mesh, mesh_group); + if (local_order_control) { + RewriteSharedEntitiesInLocalOrder(mesh, mesh_group); + } + MFEM_VERIFY(store.getRoot()->save(file, "sidre_hdf5"), "Unable to save " << file); + } + MPI_Barrier(MPI_COMM_WORLD); + + DataStore store; + MFEM_VERIFY(store.getRoot()->load(file, "sidre_hdf5"), "Unable to load " << file); + return exaconstit::ExaSidreParMesh::Create(store.getRoot()->getGroup("mesh"), MPI_COMM_WORLD); +} + +/** + * @brief Run one round-trip case + * + * @param label Case name, used for the scratch directory and the summary line + * @param type Element type of the serial mesh + * @param n Elements per cube edge of the serial mesh + * @param ref_par Number of uniform parallel refinements after partitioning + * @param leave_ranks_empty Whether to partition onto only half of the ranks (at least one), so + * the remaining ranks own no elements + * + * @details Collective. See the comment at the top of this file for what is checked. + */ +void RunRoundTripCase(const std::string& label, + const mfem::Element::Type type, + const int n, + const int ref_par, + const bool leave_ranks_empty) { + int rank = 0; + int num_ranks = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + + mfem::Mesh serial = mfem::Mesh::MakeCartesian3D(n, n, n, type, 1.0, 1.0, 1.0, false); + PerturbVertices(serial, 0.04 / n); + + // With leave_ranks_empty, elements are dealt out round-robin over the first half of the + // ranks only, so the other half own nothing. MFEM accepts a partitioning that leaves ranks + // empty; this is the situation the recent zero-element fixes in the solver guard against. + std::vector partitioning; + if (leave_ranks_empty) { + const int active_ranks = std::max(1, num_ranks / 2); + partitioning.resize(static_cast(serial.GetNE())); + for (int element = 0; element < serial.GetNE(); ++element) { + partitioning[static_cast(element)] = element % active_ranks; + } + } + mfem::ParMesh original( + MPI_COMM_WORLD, serial, leave_ranks_empty ? partitioning.data() : nullptr); + for (int refinement = 0; refinement < ref_par; ++refinement) { + original.UniformRefinement(); + } + + // The rank count is part of the directory name so tests with different rank counts can + // run at the same time. + const std::filesystem::path directory = std::filesystem::current_path() / + ("restart_roundtrip_np" + std::to_string(num_ranks) + + "_" + label); + const auto rebuilt = RoundTrip(original, directory, false); + + // Topology and parallel structure. + EXPECT_TRUE( + exaconstit::CountMeshTopology(original).Matches(exaconstit::CountMeshTopology(*rebuilt))) + << "local topology counts"; + EXPECT_EQ(original.GetGlobalNE(), rebuilt->GetGlobalNE()) << "global element count"; + EXPECT_EQ(original.bdr_attributes.Size(), rebuilt->bdr_attributes.Size()) + << "boundary attribute list"; + ExpectSameSharedStructures(original, *rebuilt); + + // H1 spaces of order 1 to 4 on both meshes must be indistinguishable. + mfem::VectorFunctionCoefficient coefficient(3, TestField); + double worst_inconsistency = 0.0; + for (int order = 1; order <= 4; ++order) { + const std::string context = label + ", order " + std::to_string(order); + mfem::H1_FECollection fec(order, 3); + mfem::ParFiniteElementSpace space_a(&original, &fec, 3); + mfem::ParFiniteElementSpace space_b(rebuilt.get(), &fec, 3); + EXPECT_EQ(space_a.GetVSize(), space_b.GetVSize()) << context; + EXPECT_EQ(space_a.TrueVSize(), space_b.TrueVSize()) << context; + EXPECT_EQ(space_a.GlobalTrueVSize(), space_b.GlobalTrueVSize()) << context; + + mfem::ParGridFunction field_a(&space_a); + mfem::ParGridFunction field_b(&space_b); + field_a.ProjectCoefficient(coefficient); + field_b.ProjectCoefficient(coefficient); + double local_difference = 0.0; + for (int i = 0; i < std::min(field_a.Size(), field_b.Size()); ++i) { + local_difference = std::max(local_difference, std::abs(field_a(i) - field_b(i))); + } + EXPECT_EQ(local_difference, 0.0) << "local DOF vectors differ, " << context; + EXPECT_EQ(ElementLocalFieldHash(field_a), ElementLocalFieldHash(field_b)) + << "element-local DOF hash differs, " << context; + + mfem::Vector true_a; + mfem::Vector true_b; + field_a.GetTrueDofs(true_a); + field_b.GetTrueDofs(true_b); + double true_difference = 0.0; + for (int i = 0; i < std::min(true_a.Size(), true_b.Size()); ++i) { + true_difference = std::max(true_difference, std::abs(true_a(i) - true_b(i))); + } + EXPECT_EQ(true_difference, 0.0) << "true DOF vectors differ, " << context; + + const double inconsistency = GlobalMax(SharedDofInconsistency(field_b)); + worst_inconsistency = std::max(worst_inconsistency, inconsistency); + EXPECT_LT(inconsistency, 1e-12) << "shared DOFs matched inconsistently, " << context; + } + + // Negative control (reported, not checked; see the comment at the top of this file). + const auto control = RoundTrip(original, directory.string() + "_control", true); + std::array control_inconsistency{}; + for (int order = 1; order <= 4; ++order) { + mfem::H1_FECollection fec(order, 3); + mfem::ParFiniteElementSpace space_c(control.get(), &fec, 3); + mfem::ParGridFunction field_c(&space_c); + field_c.ProjectCoefficient(coefficient); + control_inconsistency[static_cast(order)] = GlobalMax( + SharedDofInconsistency(field_c)); + } + + // GetGlobalNE() is collective, so it is called on every rank before printing on rank 0. + const long long global_elements = original.GetGlobalNE(); + int local_empty = original.GetNE() == 0 ? 1 : 0; + int empty_ranks = 0; + MPI_Allreduce(&local_empty, &empty_ranks, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + if (rank == 0) { + std::cout << label << ": " << num_ranks << " ranks (" << empty_ranks << " empty), " + << global_elements << " elements, max |x - P R x| = " << worst_inconsistency + << "; negative control at orders 1-4: " << control_inconsistency[1] << ", " + << control_inconsistency[2] << ", " << control_inconsistency[3] << ", " + << control_inconsistency[4] << std::endl; + } +} + +} // namespace + +TEST(restart_mesh_roundtrip, hex) { + RunRoundTripCase("hex", mfem::Element::HEXAHEDRON, 4, 0, false); +} + +TEST(restart_mesh_roundtrip, hex_parallel_refinement) { + RunRoundTripCase("hex_refpar", mfem::Element::HEXAHEDRON, 3, 1, false); +} + +TEST(restart_mesh_roundtrip, tet) { + RunRoundTripCase("tet", mfem::Element::TETRAHEDRON, 3, 0, false); +} + +TEST(restart_mesh_roundtrip, tet_parallel_refinement) { + RunRoundTripCase("tet_refpar", mfem::Element::TETRAHEDRON, 2, 1, false); +} + +TEST(restart_mesh_roundtrip, empty_ranks) { + RunRoundTripCase("empty_ranks", mfem::Element::HEXAHEDRON, 3, 0, true); +} + +int main(int argc, char* argv[]) { + MPI_Init(&argc, &argv); + mfem::Hypre::Init(); + ::testing::InitGoogleTest(&argc, argv); + const int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} From 961ab449205f3e9d6ab61a491b6c473fcee99d95 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:42:08 -0700 Subject: [PATCH 05/12] [Claude] Add checkpoint writing and restart Adds exaconstit::RestartCheckpoint (sim_state/restart_checkpoint.hpp/cpp), driven by the [Restart] options. A run can write binary checkpoints, one Sidre HDF5 file per rank, and a later run with the same number of ranks restarts from one and continues bitwise identically to a run that never stopped. A restart needs only the checkpoint and the options file; the mesh, grain, region-mapping, orientation and state-variable files are never read again. A checkpoint holds the mesh (through ExaSidreParMesh), the grain IDs, the reference and current coordinates, the velocity, every registered quadrature function except the tangent stiffness (recomputed before use), the time state and the material list. An end-of-step function identical to its beginning-of-step partner is stored as a reference to it. Rank 0 writes commit.json only once every rank has saved its file, so a checkpoint interrupted mid-write is never read. A failed write is reported as a warning and the run continues. keep_last prunes old checkpoints, and the first write after a restart supersedes checkpoints from the abandoned timeline. Restarting has two phases, "build first, overlay last": 1. In the SimulationState constructor, the mesh, reference coordinates and grain IDs come from the checkpoint instead of the input files, and the state-variable and orientation initialization is skipped. The region IDs come from the checkpointed mesh, checked against this run's material list. 2. Once SystemDriver and PostProcessingDriver exist, the driver calls RestoreFromCheckpoint(), which overwrites the evolving state, resumes the time state and recomputes the derived fields. It then replays the boundary-condition update steps already completed, so the active BC segment matches. Before anything is overwritten, the restart checks the format version, rank count, mesh, DOF layout, material identities and every quadrature space and function's shape. A problem on any rank is reported by that rank and aborts every rank together. Material properties may change (with a warning), as may the final time, time stepping, solver and output settings. TimeManagement gains CheckpointState and ResumeFromCheckpoint(), which use this run's time-step settings so a continuation can extend t_final. The previous and original time steps are stored too, so the restarted run logs exactly what the uninterrupted run would have. TimeManagement::RestartTimeState(), which nothing used, is removed. The AddQuadratureFunction() docs, the RestartCheckpoint class docs and a new developers_guide.md section ("Keeping New State Restartable") spell out what is checkpointed automatically and what new features must add: registered quadrature data is saved automatically, but new grid functions and meshes built on top of the volume mesh are not. doc/restart.md describes usage, what may change between runs, the design and the limitations. test_restart_e2e builds a real SimulationState with two UMAT regions and orientations, gives it a synthetic evolved state under AUTO time stepping, writes a checkpoint, deletes the input files, restarts, and compares everything bitwise, including the logged time-step line. It runs on 2 and 3 ranks within EXACONSTIT_TEST_MAX_MPI_TASKS. Two diagnostic tools are built with the tests: inspect_restart_rank_file and compare_restart_checkpoints (bitwise by default, --rtol optional). --- developers_guide.md | 43 +- doc/restart.md | 190 ++ .../restart/compare_restart_checkpoints.cpp | 530 +++++ scripts/restart/inspect_restart_rank_file.cpp | 54 + src/CMakeLists.txt | 2 + src/mechanics_driver.cpp | 34 + src/sim_state/restart_checkpoint.cpp | 1974 +++++++++++++++++ src/sim_state/restart_checkpoint.hpp | 306 +++ src/sim_state/simulation_state.cpp | 222 +- src/sim_state/simulation_state.hpp | 165 +- test/CMakeLists.txt | 26 + test/test_restart_e2e.cpp | 519 +++++ 12 files changed, 4040 insertions(+), 25 deletions(-) create mode 100644 doc/restart.md create mode 100644 scripts/restart/compare_restart_checkpoints.cpp create mode 100644 scripts/restart/inspect_restart_rank_file.cpp create mode 100644 src/sim_state/restart_checkpoint.cpp create mode 100644 src/sim_state/restart_checkpoint.hpp create mode 100644 test/test_restart_e2e.cpp diff --git a/developers_guide.md b/developers_guide.md index d5fac282..0bf32985 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -67,7 +67,7 @@ For detailed installation instructions, refer to the build scripts in `scripts/i **Optional Dependencies:** - **ADIOS2**: (MFEM-based) High-performance I/O for visualization - **Caliper**: Performance profiling -- **Axom** (with Sidre, Conduit and HDF5): Checkpoint/restart support, enabled with `-DAXOM_DIR=/lib/cmake` +- **Axom** (with Sidre, Conduit and HDF5): Checkpoint/restart support, enabled with `-DAXOM_DIR=/lib/cmake` (see `doc/restart.md`) ### Basic Build Process ```bash @@ -243,6 +243,7 @@ The `src/` directory contains the core ExaConstit implementation organized into #### `sim_state/` **Purpose**: Simulation state management and field storage - **`simulation_state.hpp/cpp`**: Central state container class +- **`restart_checkpoint.hpp/cpp`**: Writing checkpoints and restarting from them (requires Axom; see `doc/restart.md`) - **`exa_sidre_parmesh.hpp/cpp`**: Writing and rebuilding the parallel mesh for checkpoints **Manages**: @@ -617,14 +618,50 @@ The `test/data/` directory contains various example cases: 1. Create header/source in `models/mechanics_newmodel.hpp/cpp` 2. Inherit from `ExaModel` base class 3. Implement required virtual methods -4. Add configuration parsing support -5. Update `CMakeLists.txt` +4. Register every per-point quantity that carries history between steps with `SimulationState::AddQuadratureFunction()` (see [Keeping New State Restartable](#keeping-new-state-restartable)) +5. Add configuration parsing support +6. Update `CMakeLists.txt` #### New Boundary Condition Type 1. Extend `BCManager` class 2. Add parsing support in `option_boundary_conditions.cpp` 3. Update documentation and examples +#### Keeping New State Restartable +Checkpoints (`src/sim_state/restart_checkpoint.*`, described in `doc/restart.md`) save some +state automatically and some only because it is listed explicitly. Any new feature that +carries state from one step to the next must fit one of the cases below, or a restarted run +will silently continue from the wrong state. + +**Quadrature point data is saved automatically if it is registered.** Every function in +SimulationState's quadrature-function registry is written and restored, whichever component +registered it; the only exception is `tangent_stiffness`, which is recomputed before use. +- Create history data with `SimulationState::AddQuadratureFunction()`. Data kept in a model's + own `mfem::Vector` or `PartialQuadratureFunction` is not saved, and nothing warns about it. + Scratch data recomputed every step can stay private. +- Register it while the objects are being constructed (SimulationState, SystemDriver, the + models, PostProcessingDriver). The driver restores the checkpoint after all of them exist; + anything registered later keeps its initial value. +- Register beginning/end-of-step pairs with `AddUpdateVariablePairNames()` as usual; the + checkpoint then stores an identical end-of-step function as a reference to its partner. +- Data computed once in a constructor is fine if it depends only on the reference + configuration, the mesh and the options, since a restarted run reproduces those exactly. + +**Nodal data is not automatic.** Only the reference and current coordinates and the velocity +are saved; the other SimulationState grid functions are derived from them after a restart. A +new grid function or solution vector that carries history (an additional primal variable, +Lagrange multipliers, and so on) must be added explicitly to `WriteNodalFields()` and to the +binding in `BindFieldsAndQuadratureFunctions()`, with a check that the rebuilt space has the +same size. Store true DOFs where possible. + +**Only the volume mesh is saved.** Submeshes, surface meshes and other topologies built on top +of it must be rebuilt in the restarted run from the restored mesh, which is identical to the +original, local numbering included. Anything that cannot be rebuilt that way needs its own +checkpoint data. + +**Test it.** `test/test_restart_e2e.cpp` compares a restarted SimulationState with the +original bitwise; add any new state to it. + ### Performance Considerations - **GPU kernels**: Use RAJA for performance portability - **Memory management**: Follow MFEM memory patterns diff --git a/doc/restart.md b/doc/restart.md new file mode 100644 index 00000000..2caec526 --- /dev/null +++ b/doc/restart.md @@ -0,0 +1,190 @@ +# Checkpoint and Restart + +ExaConstit can write binary checkpoints during a run and restart a later run from one of them. +A restarted run continues exactly where the checkpoint left off: with the same number of MPI +ranks it produces bitwise identical results to a run that never stopped. + +A restart needs only the checkpoint and the options file. The mesh, grain, region-mapping, +orientation and state-variable files of the original run are not read again and may have been +deleted. + +## Contents +1. [Requirements](#requirements) +2. [Using checkpoints](#using-checkpoints) +3. [What may change between runs](#what-may-change-between-runs) +4. [How it works](#how-it-works) +5. [Checkpoint layout](#checkpoint-layout) +6. [Testing](#testing) +7. [Limitations](#limitations) + +## Requirements + +Checkpoints are written with Axom's Sidre library, one HDF5 file per MPI rank. ExaConstit must +be configured with `-DAXOM_DIR=/lib/cmake`, where Axom was built with Sidre and +its Conduit and HDF5 dependencies. The install scripts in `scripts/install/` build all three +(Axom v0.14.0, Conduit v0.9.7, HDF5 1.14.6). Without Axom, a `[Restart]` table that asks to +write or read checkpoints is rejected when the options are validated. + +## Using checkpoints + +Everything is controlled by the `[Restart]` table of the options file (documented with every +key in `src/options.toml`): + +```toml +[Restart] + write = true # write checkpoints during this run (default false) + interval = 50 # every 50 completed steps; 0 = only the final one (default 0) + write_final = true # also write one when the run finishes (default true) + keep_last = 2 # keep the newest 2 checkpoints; 0 = keep all (default 2) + directory = "restart" # where checkpoints are written (default "restart") + read = false # restart this run from a checkpoint (default false) + read_cycle = 1200 # which checkpoint to read; newest complete one if omitted + read_directory = "other" # read from here instead of `directory` (optional) +``` + +Each checkpoint is a directory `/cycle_/`. To continue a run, set +`read = true`, keep `directory` pointing at the same place, and increase `t_final` (or extend +the custom time steps). The restarted run picks the newest complete checkpoint unless +`read_cycle` names one, and keeps writing checkpoints to `directory` if `write` is set. + +`read_directory` restarts from another run's checkpoints while writing this run's own +checkpoints to `directory`. The other run's checkpoints are never renamed or removed, which is +useful for trying several continuations of one run, and is how the restart tests reuse a +single reference run. + +A checkpoint written at the final time is complete but has nothing left to run; restarting +from it prints a warning unless `t_final` has been increased. + +**Output files.** Volume-average output files are opened in append mode, so a continuation +that keeps the same `basename` appends to the original run's files and they read as one +uninterrupted run. Use a different `basename` to keep the continuation's output separate. + +## What may change between runs + +**Must stay the same** (checked; a mismatch aborts with a message saying what changed): +- the number of MPI ranks, since each rank holds its own piece of the mesh and state; +- the mesh order; +- the material list: the number of materials, their order, names, model types and state + variable counts (including orientations). + +**May change:** +- `t_final` and the time-step settings; +- solver, preconditioner and assembly settings; +- the device (`rtmodel`); +- visualization and post-processing settings. A quadrature function enabled only in the + continuation starts from its initial value, and one dropped by the continuation is skipped; + both are reported as warnings; +- material properties, with a warning; the continuation uses its own values. + +**Boundary conditions** for the steps already completed must stay the same; later entries of +the schedule may change. **Ignored on restart:** `ref_ser`, `ref_par` and the mesh file and +auto-mesh settings, since the mesh comes from the checkpoint. + +## How it works + +The implementation is documented in detail in the code. This section is an overview. + +| File | Contents | +|---|---| +| `src/sim_state/restart_checkpoint.hpp/.cpp` | `exaconstit::RestartCheckpoint`: writing, selecting, validating and restoring checkpoints | +| `src/sim_state/exa_sidre_parmesh.hpp/.cpp` | Writing a rank's piece of an `mfem::ParMesh` and rebuilding the identical `ParMesh` from it | +| `src/sim_state/restart_sidre_utils.hpp` | Small helpers shared by the two | +| `src/options/option_restart.cpp` | The `[Restart]` table | +| `src/sim_state/simulation_state.cpp` | The restart branches of the `SimulationState` constructor and `TimeManagement::ResumeFromCheckpoint()` | +| `src/mechanics_driver.cpp` | Restoring the state and boundary conditions before the time loop, and writing checkpoints in it | + +The background notes at the top of `restart_checkpoint.cpp` and `exa_sidre_parmesh.cpp` are +the place to start before changing anything. + +**Build first, overlay last.** A restarted run constructs `SimulationState`, `SystemDriver` +and `PostProcessingDriver` exactly as a fresh run does, then overwrites the evolving state +(coordinates, velocity, state variables, stresses and time) with the checkpointed values. +Anything a constructor initializes is simply replaced, and any quadrature function a component +registers is restored, without the restart code needing to know about it. + +The one exception is that the `SimulationState` constructor would read input files that may +no longer exist. On restart runs it takes those few things from the checkpoint instead +("phase 1"): the mesh, the reference coordinates, and the grain and region IDs. It also skips +the state-variable and orientation initialization. Everything else is restored from the driver +once every object exists ("phase 2"). + +**The mesh** is rebuilt from what each rank wrote, with the same local numbering, element +vertex order, communication groups, shared entities and shared-entity orientation as the +original, so the finite element spaces number their DOFs identically. The standard +`axom::sidre::MFEMSidreDataCollection` is not used: it records shared edges and faces in each +rank's local orientation, which matches shared DOFs incorrectly as soon as a shared face has +more than one interior DOF (H1 order 3 and up on hexes). It also references the mesh's memory +instead of copying it, and does not hand ownership of what it rebuilds to the caller. + +**Checks before anything is overwritten.** The restarted run verifies the checkpoint's format +version and rank count, the rebuilt mesh topology, the coordinate space sizes, that every DOF +is numbered as in the original (an element-local hash and a shared-DOF round trip of the +reference coordinates), the material identities, and every quadrature space and function's +shape. Problems found on any rank abort every rank together, with each failing rank reporting +its own details. + +**Writing is safe to interrupt.** Each rank writes its own file, and rank 0 writes `commit.json` +only once every rank has succeeded. A checkpoint without it is never read, so a crash during +a write cannot leave a checkpoint that looks complete. A failed write is reported as a warning +and the run continues. + +**Ranks without elements** are fully supported: they write and read empty checkpoint files and +take part in every collective step. + +**Adding new state.** Quadrature point data registered with +`SimulationState::AddQuadratureFunction()` is checkpointed automatically, including data a new +material model registers. Nodal data is not: only the coordinates and velocity are saved, and +a new grid function or solution vector has to be added to the checkpoint explicitly. Only the +volume mesh is saved; submeshes and other topologies must be rebuilt from it. The developer's +guide section "Keeping New State Restartable" lists the rules. + +## Checkpoint layout + +``` +/cycle_/ + rank_.hdf5 one per rank + commit.json written last by rank 0; marks the checkpoint complete +``` + +Each rank file holds: + +| Group | Contents | +|---|---| +| `restart_info/` | format version, rank count, rank, completed steps, driver step, completed time, next, previous and original time steps, time-step type, mesh order, ExaConstit version | +| `mesh/` | the rank's piece of the mesh (Conduit Blueprint layout) and each element's grain ID | +| `fields/` | reference and current coordinates, velocity, and the coordinate space sizes and DOF-layout hash | +| `qspaces/` | validation data for every quadrature space | +| `qfuncs/` | every quadrature function except the tangent stiffness (recomputed before use); an end-of-step function identical to its beginning-of-step partner is stored as `copy_of` instead of repeating its values | +| `materials/` | each region's material name, model type, state variable count and properties | + +Two diagnostic tools are built alongside the tests: `inspect_restart_rank_file` lists how each +quadrature function is stored in a rank file, and `compare_restart_checkpoints` compares two +checkpoints view by view (bitwise by default). + +## Testing + +The restart tests are built when ExaConstit is configured with Axom and tests are enabled. + +**Default suite** (fast; runs with `ctest`, including on CI): +- `test_restart_mesh_roundtrip_np{1..4}`: writes and rebuilds hex and tet meshes, with and + without parallel refinement, and a case where half of the ranks own no elements. Checks the + rebuilt mesh's topology, communication groups, shared-entity order and orientation, and that + H1 spaces of order 1 to 4 have identical DOF layouts. +- `test_restart_e2e_np{2,3}`: builds a real `SimulationState` from input files, checkpoints a + synthetic evolved state, deletes the input files, restarts, and compares everything bitwise. + +No test uses more MPI ranks than the machine has cores: rank counts above +`EXACONSTIT_TEST_MAX_MPI_TASKS` (default: the logical core count detected at configure time) +are skipped. + +## Limitations + +- **Same rank count.** Restarting on a different number of ranks would need a + partition-independent layout and redistribution of the state; it is not supported. +- **One file per rank per checkpoint.** Very large runs produce many files. Axom's `IOManager` + can group them later without changing the commit protocol. +- **Conforming meshes only**, with one element geometry and one boundary geometry + (segment, triangle, quadrilateral, tetrahedron or hexahedron). Nonconforming and NURBS meshes + are rejected. +- **Not yet tested on GPUs.** The restored data is written on the host and marked valid there, + so it should be copied to the device on first use, but no GPU run has been tested yet. diff --git a/scripts/restart/compare_restart_checkpoints.cpp b/scripts/restart/compare_restart_checkpoints.cpp new file mode 100644 index 00000000..95c1db77 --- /dev/null +++ b/scripts/restart/compare_restart_checkpoints.cpp @@ -0,0 +1,530 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +// Diagnostic and test tool: compares two ExaConstit checkpoint directories +// (/cycle_/, see doc/restart.md) group by group and view by view. +// +// Usage: compare_restart_checkpoints [--rtol ] +// +// Both directories must hold the same set of rank files. The commit file and every rank file +// are loaded in full, including the arrays stored as external views, and every group and view +// is compared: names, types, sizes and contents. A quadrature function stored as `copy_of` +// another one (see RestartCheckpoint::WriteQuadratureFunctions()) is resolved to that +// function's values first, so a begin/end pair stored as a copy in one checkpoint and as full +// values in the other still compares equal, as it should, since both restore the same state. +// +// Without --rtol the comparison is bitwise. With --rtol, double arrays may differ by +// rtol * max(|a|, |b|) per entry; everything else must still match exactly. +// +// Exit status: 0 = the checkpoints match, 1 = they differ, 2 = usage or I/O error. +// test/test_restart_runs.py relies on the exit status and on the last line of the output, +// which is always a one-line summary. + +namespace fs = std::filesystem; + +namespace { + +using axom::sidre::DataStore; +using axom::sidre::Group; +using axom::sidre::View; + +/** + * @brief Running totals of a comparison + */ +struct ComparisonTally { + /** @brief Relative tolerance for double arrays (0 = bitwise) */ + double rtol = 0.0; + /** @brief Number of views (or resolved quadrature functions) compared */ + long views = 0; + /** @brief Number of mismatches found */ + long mismatches = 0; + /** @brief Largest absolute difference seen between double entries */ + double max_abs = 0.0; + /** @brief Largest relative difference seen between double entries */ + double max_rel = 0.0; +}; + +/** + * @brief Encode a registry name the same way restart_checkpoint.cpp does + * + * @param name Registry name + * @return The name with '%' encoded as "%25" and '/' as "%2F" + * + * @details Needed to find the group a `copy_of` entry refers to, since `copy_of` holds the + * plain registry name while the groups are named with the encoded form. + */ +std::string SidreSafeName(const std::string& name) { + std::string encoded; + for (const char character : name) { + if (character == '%') { + encoded += "%25"; + } else if (character == '/') { + encoded += "%2F"; + } else { + encoded += character; + } + } + return encoded; +} + +/** + * @brief A Sidre file loaded in full, including the data of its external views + * + * @details Group::load() restores an external view as a description only, so this allocates + * a buffer for every external view, points the view at it, and then reads the data with + * Group::loadExternalData(), the same two-step read restart_checkpoint.cpp uses. Zero-length + * external views are described but have no data, so they are removed before the read and + * remembered by path, so the sets of empty arrays in two files can still be compared. + */ +class LoadedFile { +public: + /** + * @brief Load a Sidre file and all of its external data + * + * @param path File to load + * @param protocol Sidre protocol of the file ("sidre_hdf5" or "sidre_json") + * @return True if the file and its external data were read + */ + bool Load(const fs::path& path, const char* protocol) { + Group* root = m_store.getRoot(); + if (!root->load(path.string(), protocol)) { + return false; + } + std::vector> empty_views; + BindExternalViews(root, empty_views); + for (auto& [group, name] : empty_views) { + group->destroyView(name); + } + return m_external_buffers.empty() || root->loadExternalData(path.string()); + } + + /** + * @brief Root group of the loaded file + * + * @return The root group + */ + Group* Root() { + return m_store.getRoot(); + } + + /** + * @brief Paths of the zero-length external views removed during loading + * + * @return The set of paths + */ + const std::set& EmptyExternalViews() const { + return m_empty_paths; + } + +private: + /** + * @brief Point every non-empty external view at a newly allocated buffer + * + * @param group Group to process, recursively + * @param empty_views Receives the zero-length external views, to be removed by the caller + */ + void BindExternalViews(Group* group, std::vector>& empty_views) { + for (View& view : group->views()) { + if (!view.isExternal()) { + continue; + } + const std::size_t bytes = static_cast(view.getTotalBytes()); + if (bytes == 0) { + empty_views.emplace_back(group, view.getName()); + m_empty_paths.insert(view.getPathName()); + continue; + } + m_external_buffers.emplace_back(bytes); + view.setExternalDataPtr(m_external_buffers.back().data()); + } + for (Group& child : group->groups()) { + BindExternalViews(&child, empty_views); + } + } + + /** @brief DataStore holding the loaded file */ + DataStore m_store; + /** @brief Buffers the external views were pointed at */ + std::vector> m_external_buffers; + /** @brief Paths of the zero-length external views */ + std::set m_empty_paths; +}; + +/** + * @brief Print one mismatch and count it + * + * @param tally Running totals + * @param file File the mismatch was found in + * @param path Path of the group or view within the file + * @param message Description of the mismatch + */ +void ReportMismatch(ComparisonTally& tally, + const std::string& file, + const std::string& path, + const std::string& message) { + std::cout << " MISMATCH [" << file << "] " << path << ": " << message << std::endl; + ++tally.mismatches; +} + +/** + * @brief Compare two double arrays + * + * @param a First array + * @param b Second array + * @param count Number of entries in each + * @param tally Running totals (tolerance in, largest differences updated) + * @param message Receives a description of the differences when the arrays do not match + * @return True if the arrays match: bitwise when tally.rtol is 0, otherwise every entry within + * rtol * max(|a|, |b|) + * + * @details Entries with identical bits always match, which also makes two identical NaNs + * match. The description gives the number of differing entries, the largest absolute and + * relative differences, and the first differing entry, printed with 17 significant digits so + * the two values can be told apart. + */ +bool CompareDoubles(const double* a, + const double* b, + const long count, + ComparisonTally& tally, + std::string& message) { + if (count == 0) { + return true; + } + if (tally.rtol == 0.0 && + std::memcmp(a, b, static_cast(count) * sizeof(double)) == 0) { + return true; + } + long differing = 0; + long first_differing = -1; + double max_abs = 0.0; + double max_rel = 0.0; + for (long i = 0; i < count; ++i) { + if (std::memcmp(&a[i], &b[i], sizeof(double)) == 0) { + continue; + } + const double difference = std::abs(a[i] - b[i]); + const double scale = std::max({std::abs(a[i]), std::abs(b[i]), 1e-300}); + max_abs = std::max(max_abs, difference); + max_rel = std::max(max_rel, difference / scale); + if (tally.rtol == 0.0 || !(difference <= tally.rtol * scale)) { + if (first_differing < 0) { + first_differing = i; + } + ++differing; + } + } + tally.max_abs = std::max(tally.max_abs, max_abs); + tally.max_rel = std::max(tally.max_rel, max_rel); + if (differing == 0) { + return true; + } + std::ostringstream description; + description << differing << " of " << count << " entries differ (max abs " << std::scientific + << std::setprecision(3) << max_abs << ", max rel " << max_rel << "; first at " + << first_differing << ": " << std::defaultfloat << std::setprecision(17) + << a[first_differing] << " vs " << b[first_differing] << ")"; + message = description.str(); + return false; +} + +/** + * @brief Compare two views with the same path + * + * @param a View from the first file + * @param b View from the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Strings must be equal, other views must have the same type and number of entries, + * double arrays are compared with CompareDoubles(), and everything else byte for byte. + */ +void CompareViews(View* a, View* b, const std::string& file, ComparisonTally& tally) { + ++tally.views; + const std::string path = a->getPathName(); + if (a->getTypeID() != b->getTypeID()) { + ReportMismatch(tally, file, path, "the types differ"); + return; + } + if (a->isString() || b->isString()) { + if (!a->isString() || !b->isString() || std::string(a->getString()) != b->getString()) { + ReportMismatch(tally, + file, + path, + std::string("the strings differ: '") + + (a->isString() ? a->getString() : "?") + "' vs '" + + (b->isString() ? b->getString() : "?") + "'"); + } + return; + } + if (a->getNumElements() != b->getNumElements()) { + ReportMismatch(tally, + file, + path, + "the entry counts differ: " + std::to_string(a->getNumElements()) + " vs " + + std::to_string(b->getNumElements())); + return; + } + const long count = static_cast(a->getNumElements()); + if (count == 0) { + return; + } + const void* data_a = a->getVoidPtr(); + const void* data_b = b->getVoidPtr(); + if (data_a == nullptr || data_b == nullptr) { + ReportMismatch(tally, file, path, "no data was loaded"); + return; + } + if (a->getTypeID() == axom::sidre::FLOAT64_ID) { + std::string message; + if (!CompareDoubles(static_cast(data_a), + static_cast(data_b), + count, + tally, + message)) { + ReportMismatch(tally, file, path, message); + } + return; + } + if (std::memcmp(data_a, data_b, static_cast(a->getTotalBytes())) != 0) { + ReportMismatch(tally, file, path, "the contents differ"); + } +} + +/** + * @brief The values view of a quadrature function entry, following `copy_of` if present + * + * @param qfuncs The `qfuncs/` group + * @param entry One quadrature function's group + * @return The view holding the function's values, or nullptr if it cannot be found + */ +View* ResolveQuadratureFunctionValues(Group* qfuncs, Group* entry) { + if (entry->hasView("copy_of")) { + const std::string source = SidreSafeName(entry->getView("copy_of")->getString()); + if (!qfuncs->hasGroup(source)) { + return nullptr; + } + Group* source_entry = qfuncs->getGroup(source); + return source_entry->hasView("values") ? source_entry->getView("values") : nullptr; + } + return entry->hasView("values") ? entry->getView("values") : nullptr; +} + +/** + * @brief Compare the `qfuncs/` groups of two rank files + * + * @param a `qfuncs/` group of the first file + * @param b `qfuncs/` group of the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Both files must have the same quadrature functions with the same space name, vdim + * and value count. The values are compared after resolving `copy_of`, so how each file chose + * to store a function does not matter, only what it restores. A function with no values and + * no `copy_of` in both files is an empty function whose zero-length view was removed while + * loading, and has nothing to compare. + */ +void CompareQuadratureFunctions(Group* a, + Group* b, + const std::string& file, + ComparisonTally& tally) { + std::set names; + for (Group& group : a->groups()) { + names.insert(group.getName()); + } + for (Group& group : b->groups()) { + names.insert(group.getName()); + } + for (const std::string& name : names) { + const std::string path = a->getPathName() + "/" + name; + if (!a->hasGroup(name) || !b->hasGroup(name)) { + ReportMismatch(tally, + file, + path, + std::string("only in the ") + (a->hasGroup(name) ? "first" : "second") + + " checkpoint"); + continue; + } + Group* entry_a = a->getGroup(name); + Group* entry_b = b->getGroup(name); + for (const char* metadata : {"qspace_name", "vdim", "value_count"}) { + if (entry_a->hasView(metadata) != entry_b->hasView(metadata)) { + ReportMismatch( + tally, file, path + "/" + metadata, "present in only one checkpoint"); + } else if (entry_a->hasView(metadata)) { + CompareViews(entry_a->getView(metadata), entry_b->getView(metadata), file, tally); + } + } + + const bool empty_a = !entry_a->hasView("copy_of") && !entry_a->hasView("values"); + const bool empty_b = !entry_b->hasView("copy_of") && !entry_b->hasView("values"); + if (empty_a && empty_b) { + continue; + } + View* values_a = ResolveQuadratureFunctionValues(a, entry_a); + View* values_b = ResolveQuadratureFunctionValues(b, entry_b); + if (values_a == nullptr || values_b == nullptr) { + ReportMismatch(tally, file, path, "the values could not be resolved"); + continue; + } + ++tally.views; + if (values_a->getNumElements() != values_b->getNumElements()) { + ReportMismatch(tally, file, path, "the value counts differ"); + continue; + } + std::string message; + if (!CompareDoubles(static_cast(values_a->getVoidPtr()), + static_cast(values_b->getVoidPtr()), + static_cast(values_a->getNumElements()), + tally, + message)) { + ReportMismatch(tally, file, path, message); + } + } +} + +/** + * @brief Compare two groups recursively + * + * @param a Group from the first file + * @param b Group from the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Both groups must have the same views and child groups. The top-level `qfuncs/` + * group is compared with CompareQuadratureFunctions() so that `copy_of` entries are resolved. + */ +void CompareGroups(Group* a, Group* b, const std::string& file, ComparisonTally& tally) { + std::set views; + for (View& view : a->views()) { + views.insert(view.getName()); + } + for (View& view : b->views()) { + views.insert(view.getName()); + } + for (const std::string& name : views) { + if (!a->hasView(name) || !b->hasView(name)) { + ReportMismatch(tally, + file, + a->getPathName() + "/" + name, + std::string("view only in the ") + + (a->hasView(name) ? "first" : "second") + " checkpoint"); + continue; + } + CompareViews(a->getView(name), b->getView(name), file, tally); + } + + std::set groups; + for (Group& group : a->groups()) { + groups.insert(group.getName()); + } + for (Group& group : b->groups()) { + groups.insert(group.getName()); + } + for (const std::string& name : groups) { + if (!a->hasGroup(name) || !b->hasGroup(name)) { + ReportMismatch(tally, + file, + a->getPathName() + "/" + name, + std::string("group only in the ") + + (a->hasGroup(name) ? "first" : "second") + " checkpoint"); + continue; + } + if (name == "qfuncs" && a->isRoot()) { + CompareQuadratureFunctions(a->getGroup(name), b->getGroup(name), file, tally); + } else { + CompareGroups(a->getGroup(name), b->getGroup(name), file, tally); + } + } +} + +/** + * @brief Rank files in a checkpoint directory + * + * @param directory Checkpoint directory + * @return Names of the rank_*.hdf5 files, sorted + */ +std::vector RankFiles(const fs::path& directory) { + std::vector files; + std::error_code error; + for (const auto& item : fs::directory_iterator(directory, error)) { + const std::string name = item.path().filename().string(); + if (name.rfind("rank_", 0) == 0 && item.path().extension() == ".hdf5") { + files.push_back(name); + } + } + std::sort(files.begin(), files.end()); + return files; +} + +} // namespace + +int main(int argc, char** argv) { + ComparisonTally tally; + std::vector directories; + for (int i = 1; i < argc; ++i) { + const std::string argument = argv[i]; + if (argument == "--rtol" && i + 1 < argc) { + tally.rtol = std::atof(argv[++i]); + } else { + directories.push_back(argument); + } + } + if (directories.size() != 2) { + std::cerr << "usage: " << argv[0] << " [--rtol ]" + << std::endl; + return 2; + } + const fs::path directory_a = directories[0]; + const fs::path directory_b = directories[1]; + + const std::vector files_a = RankFiles(directory_a); + const std::vector files_b = RankFiles(directory_b); + if (files_a.empty() || files_a != files_b) { + std::cout << "ERROR: the rank files differ or are missing (" << files_a.size() << " in " + << directory_a.string() << ", " << files_b.size() << " in " + << directory_b.string() << ")" << std::endl; + return 2; + } + + { + LoadedFile commit_a; + LoadedFile commit_b; + if (!commit_a.Load(directory_a / "commit.json", "sidre_json") || + !commit_b.Load(directory_b / "commit.json", "sidre_json")) { + std::cout << "ERROR: unable to load commit.json in both directories" << std::endl; + return 2; + } + CompareGroups(commit_a.Root(), commit_b.Root(), "commit.json", tally); + } + for (const std::string& file : files_a) { + LoadedFile file_a; + LoadedFile file_b; + if (!file_a.Load(directory_a / file, "sidre_hdf5") || + !file_b.Load(directory_b / file, "sidre_hdf5")) { + std::cout << "ERROR: unable to load " << file << " in both directories" << std::endl; + return 2; + } + if (file_a.EmptyExternalViews() != file_b.EmptyExternalViews()) { + ReportMismatch(tally, file, "/", "the sets of empty arrays differ"); + } + CompareGroups(file_a.Root(), file_b.Root(), file, tally); + } + + std::cout << (tally.mismatches == 0 ? "MATCH" : "DIFFER") << ": " << files_a.size() + << " rank files, " << tally.views << " views compared, " << tally.mismatches + << " mismatches (max abs " << std::scientific << std::setprecision(3) << tally.max_abs + << ", max rel " << tally.max_rel << ")" << (tally.rtol == 0.0 ? ", bitwise" : "") + << std::endl; + return tally.mismatches == 0 ? 0 : 1; +} diff --git a/scripts/restart/inspect_restart_rank_file.cpp b/scripts/restart/inspect_restart_rank_file.cpp new file mode 100644 index 00000000..761ee9f1 --- /dev/null +++ b/scripts/restart/inspect_restart_rank_file.cpp @@ -0,0 +1,54 @@ +#include +#include +#include + +#include "axom/sidre.hpp" + +// Diagnostic tool: lists the top-level groups of an ExaConstit checkpoint rank file and how +// each quadrature function is stored in it, either as values (with their count) or as +// `copy_of` another function (see RestartCheckpoint::WriteQuadratureFunctions()). +// +// Usage: inspect_restart_rank_file /cycle_/rank_.hdf5 +// +// Only the file's structure is read; the stored arrays themselves are not loaded. + +int main(int argc, char** argv) { + if (argc < 2) { + std::cerr << "usage: " << argv[0] << " " << std::endl; + return EXIT_FAILURE; + } + + axom::sidre::DataStore store; + axom::sidre::Group* root = store.getRoot(); + if (!root->load(argv[1], "sidre_hdf5")) { + std::cerr << "Unable to load " << argv[1] << " as a Sidre HDF5 file" << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Top-level groups:"; + for (axom::sidre::Group& group : root->groups()) { + std::cout << " " << group.getName(); + } + std::cout << std::endl; + + if (!root->hasGroup("qfuncs")) { + std::cout << "No qfuncs/ group; this is not an ExaConstit checkpoint rank file" + << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Quadrature functions:" << std::endl; + for (axom::sidre::Group& entry : root->getGroup("qfuncs")->groups()) { + std::string storage; + if (entry.hasView("copy_of")) { + storage = std::string("copy_of ") + entry.getView("copy_of")->getString(); + } else if (entry.hasView("values")) { + storage = "values " + std::to_string(entry.getView("values")->getNumElements()) + + " doubles"; + } else { + storage = "(no values)"; + } + std::cout << " " << entry.getName() << ": " << storage << std::endl; + } + return EXIT_SUCCESS; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb9c825b..ccf94fab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ set(EXACONSTIT_HEADERS postprocessing/mechanics_lightup.hpp sim_state/simulation_state.hpp sim_state/exa_sidre_parmesh.hpp + sim_state/restart_checkpoint.hpp solvers/mechanics_solver.hpp solvers/trust_region_solver.hpp utilities/dynamic_function_loader.hpp @@ -62,6 +63,7 @@ set(EXACONSTIT_SOURCES postprocessing/mechanics_lightup.cpp sim_state/simulation_state.cpp sim_state/exa_sidre_parmesh.cpp + sim_state/restart_checkpoint.cpp solvers/mechanics_solver.cpp solvers/trust_region_solver.cpp utilities/mechanics_kernels.cpp diff --git a/src/mechanics_driver.cpp b/src/mechanics_driver.cpp index 4f3efcad..40ea95a9 100644 --- a/src/mechanics_driver.cpp +++ b/src/mechanics_driver.cpp @@ -318,6 +318,39 @@ int main(int argc, char* argv[]) { * - Performs material state updates and post-processing at each step */ int ti = 0; + if (sim_state->IsRestartRun()) { + // Restart phase 2. Every object above has been built exactly as in a fresh run, so + // the checkpoint now overwrites the evolving state (coordinates, velocity, state + // variables, stresses and time). Whatever the constructors initialized, such as + // ExaCMechModel::InitStateVars(), is simply replaced. + sim_state->RestoreFromCheckpoint(); + ti = static_cast(sim_state->GetRestartDriverStep()); + + // Bring the boundary conditions to the segment that was active after step ti. + // + // SystemDriver's constructor set up the boundary conditions of step 1. In an + // uninterrupted run, each later update step s calls GetUpdateStep(s), which makes s + // BCManager's current step, and then UpdateEssBdr() rebuilds the boundary data and + // essential DOFs for it (followed by UpdateVelocity() and SolveInit()). + // GetUpdateStep(s) has no other side effects and only changes the current step when + // s is an update step, so replaying it for s = 1..ti leaves BCManager on the last + // update step <= ti, and UpdateEssBdr() then rebuilds that segment's boundary data. + // + // UpdateVelocity() and SolveInit() are not repeated. UpdateVelocity() is called by + // the loop before every Solve(). SolveInit() was the corrector applied at the start of + // that segment, and its effect is already part of the restored velocity. If ti + 1 + // is itself an update step, the loop below handles it exactly as an uninterrupted + // run would. + int last_bc_update_step = 0; + for (int step = 1; step <= ti; ++step) { + if (BCManager::GetInstance().GetUpdateStep(step)) { + last_bc_update_step = step; + } + } + if (last_bc_update_step > 0) { + oper.UpdateEssBdr(); + } + } auto v_sol = sim_state->GetPrimalField(); while (!sim_state->IsFinished()) { ti++; @@ -367,6 +400,7 @@ int main(int argc, char* argv[]) { sim_state->FinishCycle(); oper.UpdateModel(); post_process.Update(ti, sim_state->GetTrueCycleTime()); + sim_state->WriteCheckpointIfDue(static_cast(ti)); } // end loop over time steps /** diff --git a/src/sim_state/restart_checkpoint.cpp b/src/sim_state/restart_checkpoint.cpp new file mode 100644 index 00000000..f17a72d5 --- /dev/null +++ b/src/sim_state/restart_checkpoint.cpp @@ -0,0 +1,1974 @@ +// Compiled only when ExaConstit is built with Axom, so this file can always be listed in +// CMake; without EXACONSTIT_USE_AXOM it is an empty translation unit. +#if defined(EXACONSTIT_USE_AXOM) + +#include "sim_state/restart_checkpoint.hpp" + +#include "sim_state/exa_sidre_parmesh.hpp" +#include "sim_state/restart_sidre_utils.hpp" +#include "sim_state/simulation_state.hpp" +#include "utilities/mechanics_log.hpp" +#include "utilities/unified_logger.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +#if !defined(AXOM_USE_HDF5) +#error "ExaConstit restart uses the sidre_hdf5 protocol; build Axom with HDF5 (AXOM_USE_HDF5)." +#endif + +// ========================================================================= +// BACKGROUND: WHAT A CHECKPOINT CONTAINS AND HOW A RESTART USES IT +// ========================================================================= +// +// Read this once before changing anything below; every function in this file implements a +// piece of it. doc/restart.md has the user-facing description and the full design record. +// +// Goal. A restarted run must continue exactly as if the original run had never stopped, +// using only the checkpoint and the options file. The mesh, grain, region-mapping, +// orientation and state-variable files may no longer exist. For a run with the same number of +// ranks, the restarted run is bitwise identical to the uninterrupted one +// (test/test_restart_runs.py checks this on real runs). +// +// Rank file layout. Every rank writes one Sidre DataStore to +// /cycle_/rank_.hdf5 with these groups: +// +// restart_info/ format version, rank count, this rank, completed cycles, driver step, +// completed time, next dt, time-step type, mesh order, ExaConstit version +// mesh/ this rank's piece of the mesh (exa_sidre_parmesh.cpp), plus +// fields/grain_id (the grain ID of every local element) +// fields/ mesh_ref/values reference coordinates (owned copy, see below) +// mesh_current/values current coordinates +// primal_field/values velocity true DOFs +// mesh_space/ sizes and a DOF-layout hash of the coordinate space +// qspaces/ validation data for each quadrature space (order, size, element subset) +// qfuncs/ each quadrature function: its space, vdim and size, plus either its +// values or `copy_of = ` (see WriteQuadratureFunctions()) +// materials/ name, model type, state-variable count and properties of every region +// +// Registry names are made into single Sidre path segments with SidreSafeName(), because +// Sidre uses '/' as its path separator. +// +// Owned and external views. Sidre can either copy data into memory it owns ("owned" view) +// or record a pointer to memory someone else owns ("external" view). The large arrays +// (current coordinates, velocity, quadrature functions) are written as external views +// pointing straight at SimulationState's host data, so writing a checkpoint copies nothing in +// memory. On read, Group::load() restores an external view as a description only (type and +// size, no data). The reader then points each one at the SimulationState buffer it should +// fill (View::setExternalDataPtr()) and a single Group::loadExternalData() call reads all of +// them from the file straight into place. Small and metadata arrays are owned, and so is +// mesh_ref, because it is needed in phase 1, before the buffers phase 2 fills even exist. +// +// Commit protocol. Every rank saves its file, the ranks agree that all saves worked, and only +// then does rank 0 write commit.json (to a temporary name, then renamed, which is atomic on +// POSIX file systems). A checkpoint directory without commit.json is incomplete and is never +// selected for a restart, so a crash in the middle of a write can never be restarted from. +// +// Two-phase restart. A fresh run builds every object from the input files: SimulationState, +// then SystemDriver (the operator, the material models and the solvers), then +// PostProcessingDriver. A restarted run builds them in exactly the same way and then +// overwrites the evolving state with the checkpoint ("build first, overlay last"). Anything a +// constructor initializes (for example ExaCMechModel::InitStateVars()) is simply replaced, +// and quadrature functions registered by any component are restored as long as they exist +// by then. The one unavoidable exception is that SimulationState's constructor would read +// input files that may be gone, so on restart runs it takes those few things from the +// checkpoint instead (phase 1): +// - the mesh (ReconstructMesh()), since every space is built on it; +// - the reference coordinates (RestoreReferenceConfiguration()), see that function for why +// the exact bits matter even during construction; +// - the grain IDs and region IDs (ReadElementGrainIds() and the element attributes), since +// they decide which region spaces and functions the constructor creates; +// - the orientation and state-variable initialization is skipped, since phase 2 overwrites +// that state anyway. +// Phase 2 (RestoreState()) is the overlay itself, called from the driver once every object +// exists. +// +// Ranks with no elements. Every function here must work on a rank that owns no elements, and +// such a rank must still reach every collective call. On such a rank the mesh arrays, the +// coordinate and velocity vectors and the global quadrature functions all have zero length, +// and the region quadrature spaces and functions do not exist at all (SimulationState only +// creates them where the region has elements). Zero-length arrays are written as described +// but unallocated views and are never dereferenced; their data pointers may be null. Each +// rank only ever reads the file it wrote, so a region that does not exist on a rank is absent +// from both the checkpoint and the restarted run on that rank. test_restart_mesh_roundtrip +// and test_restart_runs both include cases with empty ranks. +// +// DataStore lifetimes. A DataStore that finds Axom's logger (SLIC) uninitialized initializes +// it and finalizes it again in its destructor. Two DataStores alive at the same time could +// therefore leave the second one logging through a finalized logger, so every DataStore in +// this file is destroyed before the next one is created: the write store before the commit +// store, and the commit store before the read store. + +namespace fs = std::filesystem; + +namespace exaconstit { + +namespace { + +using axom::sidre::DataStore; +using axom::sidre::Group; +using axom::sidre::IndexType; +using axom::sidre::View; +using restart_detail::AllRanksTrue; +using restart_detail::ReadSidreScalar; + +/** @brief File name of the commit marker that makes a checkpoint directory usable */ +constexpr const char* COMMIT_FILE_NAME = "commit.json"; +/** @brief Sidre protocol of the commit file (small, human readable) */ +constexpr const char* COMMIT_PROTOCOL = "sidre_json"; +/** @brief Sidre protocol of the rank files */ +constexpr const char* RANK_FILE_PROTOCOL = "sidre_hdf5"; +/** @brief Number of digits of the cycle count in a checkpoint directory name */ +constexpr std::size_t CYCLE_DIGITS = 10; + +/** + * @brief Name of the checkpoint directory for a completed-cycle count + * + * @param completed_cycles Number of completed cycles the checkpoint holds + * @return `cycle_` followed by the count, zero-padded to CYCLE_DIGITS digits + * + * @details Zero padding makes the directories sort in cycle order by name as well. + */ +std::string CycleDirectoryName(const std::size_t completed_cycles) { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "cycle_%010zu", completed_cycles); + return std::string(buffer); +} + +/** + * @brief Name of a rank's file inside a checkpoint directory + * + * @param rank MPI rank + * @return `rank_` followed by the rank, zero-padded to 6 digits, and `.hdf5` + */ +std::string RankFileName(const int rank) { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "rank_%06d.hdf5", rank); + return std::string(buffer); +} + +/** + * @brief Parse a checkpoint directory name + * + * @param name Directory name (not a path) + * @param completed_cycles Receives the cycle count when the name matches + * @return True if the name is exactly `cycle_` followed by CYCLE_DIGITS digits + * + * @details The match is exact, so renamed directories (`cycle_.superseded`) and anything + * else a user leaves in the checkpoint directory are never mistaken for checkpoints. + */ +bool ParseCycleDirectoryName(const std::string& name, std::size_t& completed_cycles) { + const std::string prefix = "cycle_"; + if (name.size() != prefix.size() + CYCLE_DIGITS || + name.compare(0, prefix.size(), prefix) != 0) { + return false; + } + for (std::size_t i = prefix.size(); i < name.size(); ++i) { + if (name[i] < '0' || name[i] > '9') { + return false; + } + } + completed_cycles = static_cast(std::stoull(name.substr(prefix.size()))); + return true; +} + +/** + * @brief A checkpoint directory found on disk + */ +struct CheckpointDirectoryEntry { + /** @brief Completed cycles the checkpoint holds (from its name) */ + std::size_t completed_cycles = 0; + /** @brief Path of the checkpoint directory */ + fs::path path; + /** @brief Whether the directory has a commit file, i.e. is complete */ + bool committed = false; +}; + +/** + * @brief List the checkpoint directories under a directory, oldest first + * + * @param directory Checkpoint root directory + * @return One entry per `cycle_` subdirectory, sorted by completed cycles; empty if + * the directory does not exist + * + * @details Only rank 0 calls this. File system errors are treated as "no checkpoints" rather + * than thrown, since the callers decide what a missing checkpoint means. + */ +std::vector ListCheckpointDirectories(const fs::path& directory) { + std::vector entries; + std::error_code error; + if (!fs::is_directory(directory, error)) { + return entries; + } + for (const auto& item : fs::directory_iterator(directory, error)) { + std::size_t completed_cycles = 0; + if (item.is_directory(error) && + ParseCycleDirectoryName(item.path().filename().string(), completed_cycles)) { + entries.push_back( + {completed_cycles, item.path(), fs::exists(item.path() / COMMIT_FILE_NAME, error)}); + } + } + std::sort(entries.begin(), entries.end(), [](const auto& lhs, const auto& rhs) { + return lhs.completed_cycles < rhs.completed_cycles; + }); + return entries; +} + +/** + * @brief Turn a SimulationState registry name into a single Sidre path segment + * + * @param name Registry name (quadrature space or quadrature function name) + * @return The name with '%' encoded as "%25" and '/' as "%2F" + * + * @details Sidre treats '/' as a path separator, so a name containing it would otherwise + * create nested groups. The same encoding is used on write and read, so the round trip only + * needs the two to agree; scripts/restart/compare_restart_checkpoints.cpp uses it too. + */ +std::string SidreSafeName(const std::string& name) { + std::string encoded; + encoded.reserve(name.size()); + for (const char character : name) { + if (character == '/') { + encoded += "%2F"; + } else if (character == '%') { + encoded += "%25"; + } else { + encoded += character; + } + } + return encoded; +} + +/** + * @brief Broadcast a string from rank 0 + * + * @param value On rank 0 the string to send; on every other rank receives it + * @param comm Communicator to broadcast over + * + * @details Collective. The length is broadcast first so the receivers can size the string. + */ +void BroadcastString(std::string& value, MPI_Comm comm) { + int length = static_cast(value.size()); + MPI_Bcast(&length, 1, MPI_INT, 0, comm); + value.resize(static_cast(length)); + if (length > 0) { + MPI_Bcast(value.data(), length, MPI_CHAR, 0, comm); + } +} + +/** + * @brief 64-bit FNV-1a hash + * + * @details A simple, deterministic, order-sensitive hash with no dependencies. It is used to + * fingerprint arrays that are too large to store just for validation (DOF layouts and element + * subsets). It is not cryptographic; it only has to make an accidental match between two + * different layouts very unlikely. + */ +class Fnv1a64 { +public: + /** + * @brief Add raw bytes to the hash + * + * @param data Bytes to add + * @param byte_count Number of bytes + */ + void AddBytes(const void* data, const std::size_t byte_count) { + const auto* bytes = static_cast(data); + for (std::size_t i = 0; i < byte_count; ++i) { + m_state ^= static_cast(bytes[i]); + m_state *= 1099511628211ULL; + } + } + + /** + * @brief Current hash value + * + * @return The hash of every byte added so far + */ + std::uint64_t Value() const { + return m_state; + } + +private: + /** @brief Running hash state, starting from the FNV-1a 64-bit offset basis */ + std::uint64_t m_state = 1469598103934665603ULL; +}; + +/** + * @brief Hash of an integer array's contents + * + * @param values Array to hash + * @return FNV-1a hash of the entries in order; the hash of nothing for an empty array + * + * @details Empty arrays (for example the element subset of the global quadrature space on a + * rank with no elements) are never read, since their host pointer may be null. + */ +std::uint64_t HashIntArray(const mfem::Array& values) { + Fnv1a64 hasher; + if (values.Size() > 0) { + hasher.AddBytes(values.HostRead(), static_cast(values.Size()) * sizeof(int)); + } + return hasher.Value(); +} + +/** + * @brief Hash of a grid function read element by element in each element's DOF order + * + * @param field Grid function to hash + * @return FNV-1a hash of the values visited element by element + * + * @details For each local element, the field's values are visited through + * GetElementVDofs(), i.e. in the order the element's DOFs are numbered locally. If the + * rebuilt space numbers the local DOFs differently from the writing run, the stored + * coordinate vector lands on different DOFs and the visited sequence, and so the hash, + * changes. Coordinates at distinct nodes are distinct, so a renumbering cannot reproduce the + * same sequence by accident. + * + * MFEM encodes a DOF with reversed orientation as a negative index -1 - dof whose value must + * be negated; that convention is followed so the hash matches what an element actually sees. + * On a rank with no elements nothing is visited and the hash is that of an empty sequence, + * the same in both runs. + */ +std::uint64_t ElementLocalFieldHash(const mfem::ParGridFunction& field) { + const mfem::ParFiniteElementSpace* space = field.ParFESpace(); + const double* values = field.HostRead(); + mfem::Array element_vdofs; + Fnv1a64 hasher; + for (int element_index = 0; element_index < space->GetNE(); ++element_index) { + space->GetElementVDofs(element_index, element_vdofs); + for (const int signed_vdof : element_vdofs) { + const int vdof = (signed_vdof >= 0) ? signed_vdof : (-1 - signed_vdof); + const double value = (signed_vdof >= 0) ? values[vdof] : -values[vdof]; + hasher.AddBytes(&value, sizeof(value)); + } + } + return hasher.Value(); +} + +/** + * @brief Largest change of a grid function when restricted to true DOFs and prolongated back + * + * @param field Grid function to test + * @return max |field - P(R(field))| over this rank's local DOFs + * + * @details R (GridFunction::GetTrueDofs()) keeps one value per true DOF, the one held by the + * owning rank, and P (ParGridFunction::SetFromTrueDofs()) copies it back to every rank that + * shares the DOF. A field that was continuous in the writing run survives this unchanged if, + * and only if, every shared DOF is matched with the same true DOF on every rank. A + * mismatched shared DOF, for example from a wrong shared-face orientation, receives another + * DOF's value and shows up as a jump of the order of the element size. + * + * Collective: the prolongation exchanges shared values between ranks, so every rank must + * call this, including ranks with no DOFs (which return 0). + */ +double SharedDofInconsistency(const mfem::ParGridFunction& field) { + mfem::Vector true_values; + field.GetTrueDofs(true_values); + mfem::ParGridFunction prolongated(field.ParFESpace()); + prolongated.SetFromTrueDofs(true_values); + const double* original = field.HostRead(); + const double* round_trip = prolongated.HostRead(); + double max_difference = 0.0; + for (int i = 0; i < field.Size(); ++i) { + max_difference = std::max(max_difference, std::abs(original[i] - round_trip[i])); + } + return max_difference; +} + +/** + * @brief Create an external double array view + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param data Host data the view points at; must stay valid until the DataStore is saved + * @param count Number of entries + * + * @details No data is copied; Group::save() reads straight from data. Zero-length arrays are + * only described, since their data pointer may be null. Sidre's external-view API takes a + * non-const pointer, but saving only reads through it. + */ +void CreateExternalDoubleView(Group* group, + const std::string& path, + const double* data, + const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return; + } + group->createView(path, axom::sidre::FLOAT64_ID, count, const_cast(data)); +} + +/** + * @brief Create an owned double array view holding a copy of some data + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param data Data to copy (may be null when count is 0) + * @param count Number of entries + * + * @details Zero-length arrays are only described, not allocated. + */ +void CreateOwnedDoubleCopy(Group* group, + const std::string& path, + const double* data, + const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return; + } + double* stored = + group->createViewAndAllocate(path, axom::sidre::FLOAT64_ID, count)->getData(); + std::copy(data, data + count, stored); +} + +/** + * @brief Name under which a quadrature space is registered in SimulationState + * + * @param spaces SimulationState's quadrature space registry + * @param space Space to look up + * @return The registry name, or an empty string if the space is not registered + * + * @details Quadrature functions only hold a pointer to their space, so the name is found by + * searching the registry. It is stored with each quadrature function so the reader can check + * that the function still lives on the same space. + */ +std::string FindQuadratureSpaceName( + const std::map>& spaces, + const mfem::expt::PartialQuadratureSpace* space) { + for (const auto& [name, candidate] : spaces) { + if (candidate.get() == space) { + return name; + } + } + return std::string(); +} + +/** + * @brief Quadrature functions that are recomputed before they are next read, and so are + * neither written nor restored + * + * @param sim_state Simulation state whose registry names to use + * @return Registry names of the global and every region's tangent stiffness + * + * @details The tangent stiffness is the largest quadrature function (36 values per point, + * both globally and per region) and it is always recomputed before any read: + * - Every NonlinearMechOperator::Setup() calls the material model's ModelSetup(), which + * recomputes the tangent from the converged state. Setup() runs at the start of every + * Newton residual evaluation (Mult()) and in SolveInit() after a boundary-condition change + * (GetUpdateBCsAction()). + * - The integrators read tangent values only while assembling the Jacobian + * (AssembleElementGrad(), AssembleGradPA(), AssembleEA(), AssembleGradDiagonalPA(), + * AddMult[Transpose]GradPA()), which always happens after such a Setup(). + * ICExaNLFIntegrator::AddMultPA() only uses the function's space (its integration rule), + * which the restarted run's constructor-built function provides. + * If a future change reads the tangent before the first Setup() of a step, it must be removed + * from this list so it is checkpointed again. + * + * The region names are generated for every region, including regions with no elements on + * this rank; looking them up in a set is harmless when they are absent from the registry. + */ +std::set RecomputedQuadratureFunctionNames(const SimulationState& sim_state) { + std::set names; + names.insert(sim_state.GetQuadratureFunctionMapName("tangent_stiffness", -1)); + for (std::size_t region = 0; region < sim_state.GetNumberOfRegions(); ++region) { + names.insert( + sim_state.GetQuadratureFunctionMapName("tangent_stiffness", static_cast(region))); + } + return names; +} + +/** + * @brief Collect the external views that have not been pointed at a buffer + * + * @param group Group to search, recursively + * @param out Receives (group, view name) for every external view with a null data pointer + * + * @details Group::loadExternalData() tries to read every external view described in the + * group, bound or not, so the unbound ones (zero-length arrays and quadrature functions + * present only in the checkpoint) have to be removed first; see RestoreState(). + */ +void CollectUnboundExternalViews(Group* group, std::vector>& out) { + for (View& view : group->views()) { + if (view.isExternal() && view.getVoidPtr() == nullptr) { + out.emplace_back(group, view.getName()); + } + } + for (Group& child : group->groups()) { + CollectUnboundExternalViews(&child, out); + } +} + +} // namespace + +// ========================================================================= +// CONSTRUCTION AND FAILURE REPORTING +// ========================================================================= + +RestartCheckpoint::RestartCheckpoint(const RestartOptions& options, MPI_Comm comm) + : m_options(options), m_comm(comm) { + MPI_Comm_rank(m_comm, &m_rank); + MPI_Comm_size(m_comm, &m_num_ranks); +} + +RestartCheckpoint::~RestartCheckpoint() = default; + +void RestartCheckpoint::RecordFailure(const std::string& message) { + m_local_failed = true; + m_local_failure_log += "\n - " + message; +} + +/** + * @details Implementation notes. Most checks in this file can fail on some ranks and not + * others, for example a malformed array in one rank's file. Aborting only on the failing rank + * would leave every other rank waiting forever in its next collective call, so each check + * records its failure (RecordFailure()) and every stage ends with this function, which every + * rank calls. The ranks agree on whether anyone failed. If so, each failing rank reports its + * own problems with MFEM_WARNING, since only it knows them, and every rank then aborts with a + * single shared MFEM_ABORT_0 message. + */ +void RestartCheckpoint::AbortIfAnyRankFailed(const char* stage) { + if (AllRanksTrue(!m_local_failed, m_comm)) { + m_local_failure_log.clear(); + return; + } + if (m_local_failed) { + MFEM_WARNING("Restart failed on rank " << m_rank << " while " << stage << ":" + << m_local_failure_log); + } + MFEM_ABORT_0("Restart failed while " << stage << " (see the per-rank warnings above)"); +} + +// ========================================================================= +// WRITING +// ========================================================================= + +/** + * @details Implementation notes. The decision uses only values every rank agrees on (the + * options, the time manager and m_last_written_cycles), so every rank makes the same decision + * and either all ranks or no ranks enter the collective WriteCheckpoint(). + * + * - IsCheckpointSafe(): the time manager is not in the middle of a retrial or sub-step + * sequence, whose counters are not checkpointed. With the current SystemDriver::Solve() this + * always holds after a converged step: retries loop inside Solve() until the step succeeds, + * AUTO stepping never sub-steps, and FIXED / CUSTOM stepping always report success. The + * guard keeps checkpoints correct if sub-stepping across driver iterations is ever added. + * - Counting cycles since the last checkpoint, rather than testing cycle % interval, keeps + * the interval correct after a restart from an arbitrary cycle and if a write is deferred. + * - `completed_cycles <= m_last_written_cycles` prevents writing the same cycle twice, for + * example a final checkpoint right after an interval checkpoint of the same cycle, and + * writing immediately after a restart. + */ +void RestartCheckpoint::WriteCheckpointIfDue(SimulationState& sim_state, + const std::size_t driver_step) { + if (!m_options.write) { + return; + } + const TimeManagement& time_manager = sim_state.m_time_manager; + if (!time_manager.IsCheckpointSafe()) { + return; + } + const TimeManagement::CheckpointState time_state = time_manager.GetCheckpointState(); + if (time_state.completed_cycles <= m_last_written_cycles) { + return; + } + const std::size_t cycles_since_last = time_state.completed_cycles - m_last_written_cycles; + const bool interval_due = m_options.interval > 0 && + cycles_since_last >= static_cast(m_options.interval); + const bool final_due = time_manager.IsFinished() && m_options.write_final; + if (!interval_due && !final_due) { + return; + } + WriteCheckpoint(sim_state, driver_step, time_state); +} + +/** + * @details Implementation notes. A checkpoint is written in three stages: + * + * 1. Rank 0 prepares the directory (PrepareCheckpointDirectory()). If that fails the + * checkpoint is skipped with a warning and the simulation continues; losing one checkpoint + * is better than losing the run. + * 2. Every rank fills a DataStore with its state and saves it to its rank file. The helpers + * each write one top-level group (see the background notes at the top of this file for + * the layout): + * WriteRestartInfo(), WriteMeshAndGrainIds(), WriteNodalFields(), + * WriteQuadratureSpaces(), WriteQuadratureFunctions() and WriteMaterials(). + * The large arrays are external views of SimulationState's host data, so nothing is copied + * in memory, and SimulationState must not change between building the store and saving it. + * The DataStore is destroyed right after saving, freeing the owned arrays (the mesh copy, + * mesh_ref and the small metadata). The external views never owned their memory. + * 3. Commit. The ranks agree that every save succeeded. Only then does rank 0 write + * commit.json, which is what makes the checkpoint usable, and apply the retention policy. + * The result is broadcast so every rank updates m_last_written_cycles consistently, and a + * checkpoint that was not committed is reported as a warning and does not count, so the + * next due cycle tries again. + * + * Collective; every rank, including ranks with no elements, takes part in every stage. + */ +void RestartCheckpoint::WriteCheckpoint(SimulationState& sim_state, + const std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::WriteCheckpoint"); + const double start_seconds = MPI_Wtime(); + const std::size_t completed_cycles = time_state.completed_cycles; + const double completed_time = time_state.completed_time; + const fs::path checkpoint_directory = m_options.directory / + CycleDirectoryName(completed_cycles); + + // --------------------------------------------------------------------- + // Stage 1: directory + // --------------------------------------------------------------------- + if (!PrepareCheckpointDirectory(checkpoint_directory)) { + MFEM_WARNING_0("Unable to prepare the checkpoint directory " + << checkpoint_directory << "; the checkpoint for cycle " << completed_cycles + << " was skipped and the simulation continues"); + return; + } + + // --------------------------------------------------------------------- + // Stage 2: every rank builds and saves its rank file + // --------------------------------------------------------------------- + bool saved = false; + { + DataStore store; + Group* root = store.getRoot(); + WriteRestartInfo(root, sim_state, driver_step, time_state); + WriteMeshAndGrainIds(root, sim_state); + WriteNodalFields(root, sim_state); + WriteQuadratureSpaces(root, sim_state); + WriteQuadratureFunctions(root, sim_state); + WriteMaterials(root, sim_state); + saved = root->save((checkpoint_directory / RankFileName(m_rank)).string(), + RANK_FILE_PROTOCOL); + } + + // --------------------------------------------------------------------- + // Stage 3: commit (only if every rank saved its file) + // --------------------------------------------------------------------- + const bool all_saved = AllRanksTrue(saved, m_comm); + int committed = 0; + if (all_saved && m_rank == 0) { + committed = WriteCommitFile( + checkpoint_directory, completed_cycles, driver_step, completed_time) + ? 1 + : 0; + if (committed != 0) { + ApplyRetentionPolicy(); + } + } + MPI_Bcast(&committed, 1, MPI_INT, 0, m_comm); + + if (committed == 0) { + MFEM_WARNING_0("The checkpoint " << checkpoint_directory + << " was not committed (a rank file could not be " + "saved or the commit file could not be written); " + "the simulation continues"); + return; + } + m_last_written_cycles = completed_cycles; + if (m_rank == 0) { + std::cout << "Wrote restart checkpoint " << checkpoint_directory.string() << " (cycle " + << completed_cycles << ", driver step " << driver_step + << ", t = " << completed_time << ") in " << (MPI_Wtime() - start_seconds) << " s" + << std::endl; + } +} + +/** + * @brief Create the directory for a checkpoint about to be written + * + * @param checkpoint_directory Directory of the checkpoint + * @return True on every rank if the directory is ready, false on every rank otherwise + * + * @details Collective. Only rank 0 touches the file system, and its result is broadcast so + * every rank makes the same decision. Rank 0: + * - supersedes checkpoints of another timeline on this run's first write + * (SupersedeCheckpointsAfter()); + * - creates the directory; + * - removes an existing commit file, so a directory being rewritten for the same cycle is + * marked incomplete until the new commit, rather than appearing complete with a mix of old + * and new rank files. + * The supersede step is attempted only once, even if this write fails, since it renames + * rather than deletes and retrying it gains nothing. + */ +bool RestartCheckpoint::PrepareCheckpointDirectory(const fs::path& checkpoint_directory) { + int prepared = 1; + if (m_rank == 0) { + try { + if (m_supersede_pending) { + SupersedeCheckpointsAfter(m_last_written_cycles); + } + fs::create_directories(checkpoint_directory); + fs::remove(checkpoint_directory / COMMIT_FILE_NAME); + } catch (const fs::filesystem_error& error) { + MFEM_WARNING_0("Unable to prepare " << checkpoint_directory << ": " << error.what()); + prepared = 0; + } + } + m_supersede_pending = false; + MPI_Bcast(&prepared, 1, MPI_INT, 0, m_comm); + return prepared != 0; +} + +/** + * @brief Write the `restart_info/` group + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * @param driver_step Driver loop counter of the step just completed + * @param time_state Resumable time state of the time manager + * + * @details The format version, rank count and rank let OpenForRestart() reject a checkpoint + * that this build or this run cannot read before anything else is touched. The time state is + * stored as unambiguous quantities (see TimeManagement::CheckpointState) rather than the time + * manager's raw members, whose meaning depends on its internal state. The previous and + * original time steps are only used in log messages; storing them lets the restarted run log + * exactly what the uninterrupted run would have. + * + * The driver step is stored separately from the cycle count even though the two are equal + * with the current SystemDriver::Solve(). The boundary-condition schedule and post-processing + * are keyed by the driver step, and storing it separately means a restart never relies on the + * two staying equal. + * + * The time-step type, mesh order and ExaConstit version are informational; the mesh order is + * checked separately through the mesh space (ValidateMeshSpace()). + */ +void RestartCheckpoint::WriteRestartInfo(Group* root, + const SimulationState& sim_state, + const std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) const { + Group* info = root->createGroup("restart_info"); + info->createViewScalar("format_version", RESTART_FORMAT_VERSION); + info->createViewScalar("num_ranks", m_num_ranks); + info->createViewScalar("rank", m_rank); + info->createViewScalar("completed_cycles", + static_cast(time_state.completed_cycles)); + info->createViewScalar("driver_step", static_cast(driver_step)); + info->createViewScalar("completed_time", time_state.completed_time); + info->createViewScalar("next_dt", time_state.next_dt); + info->createViewScalar("previous_dt", time_state.previous_dt); + info->createViewScalar("original_dt", time_state.original_dt); + info->createViewScalar("time_type", + static_cast(sim_state.m_time_manager.GetTimeStepType())); + info->createViewScalar("mesh_order", sim_state.m_options.mesh.order); + info->createViewString("exaconstit_version", sim_state.m_options.version); +} + +/** + * @brief Write the `mesh/` group: this rank's mesh piece and the element grain IDs + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details The mesh is written by WriteParMeshBlueprint(), which is collective (the ranks + * agree on the element geometries), so every rank must call this. + * + * The grain IDs are written as an extra element field, `fields/grain_id`. A restarted run + * needs them because the element attributes no longer hold grain IDs once SimulationState has + * been set up: its constructor replaces them with region IDs, and the grain-to-region map that + * links the two comes from the region-mapping file, which may no longer exist. SimulationState + * keeps the grain IDs as a piecewise-constant grid function with one value per local element, + * stored as doubles; they are whole numbers and are converted back to int here. On a rank + * with no elements the array is empty. + */ +void RestartCheckpoint::WriteMeshAndGrainIds(Group* root, const SimulationState& sim_state) const { + Group* mesh_group = root->createGroup("mesh"); + WriteParMeshBlueprint(*sim_state.m_mesh, mesh_group); + + const int num_elements = sim_state.m_mesh->GetNE(); + mesh_group->createViewString("fields/grain_id/association", "element"); + mesh_group->createViewString("fields/grain_id/topology", "mesh"); + if (num_elements == 0) { + mesh_group->createView("fields/grain_id/values", axom::sidre::INT_ID, 0); + return; + } + int* grain_ids = mesh_group + ->createViewAndAllocate( + "fields/grain_id/values", axom::sidre::INT_ID, num_elements) + ->getData(); + const double* grain_values = sim_state.m_grains->HostRead(); + for (int element_index = 0; element_index < num_elements; ++element_index) { + grain_ids[element_index] = static_cast(grain_values[element_index]); + } +} + +/** + * @brief Write the `fields/` group: the nodal state and a fingerprint of its DOF layout + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Only the state that cannot be recomputed is stored: + * - `mesh_current`: the current coordinates (all local DOFs, i.e. VSize values). + * - `primal_field`: the velocity true DOFs (TrueVSize values), which the solver works with. + * - `mesh_ref`: the reference coordinates. These are stored as an owned copy rather than an + * external view, because phase 1 (RestoreReferenceConfiguration()) needs them inside the + * SimulationState constructor, before the read side has buffers to bind external views to. + * Everything else is derived from these after the restart (ApplyPostRestartFixup()): + * `mesh_t_beg`, the previous velocity, the velocity grid function and the displacement. + * + * External views read HostRead() data. In MFEM's CUDA, HIP and Umpire device memory spaces + * HostRead() makes a synchronous device-to-host copy, so the host data is current when the + * file is saved. (A blocking device copy only orders against the default stream, which is the + * stream the solver uses.) + * + * `mesh_space/` records the sizes of the coordinate space (local, owned true and global true + * DOFs), the mesh order and ElementLocalFieldHash() of mesh_ref. The restarted run checks + * these before interpreting the stored coordinates (ValidateMeshSpace() and + * VerifyRestoredDofLayout()), since the vectors are only meaningful if the rebuilt space + * numbers its DOFs exactly as the writing run did. + * + * On a rank with no elements every vector here is empty; the views are only described and the + * hash is that of an empty sequence. + */ +void RestartCheckpoint::WriteNodalFields(Group* root, const SimulationState& sim_state) const { + Group* fields = root->createGroup("fields"); + const mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + const mfem::ParGridFunction& mesh_current = *sim_state.m_mesh_nodes.at("mesh_current"); + const mfem::Vector& primal_field = *sim_state.m_primal_field; + + CreateOwnedDoubleCopy(fields, + "mesh_ref/values", + mesh_ref.Size() > 0 ? mesh_ref.HostRead() : nullptr, + mesh_ref.Size()); + CreateExternalDoubleView(fields, + "mesh_current/values", + mesh_current.Size() > 0 ? mesh_current.HostRead() : nullptr, + mesh_current.Size()); + CreateExternalDoubleView(fields, + "primal_field/values", + primal_field.Size() > 0 ? primal_field.HostRead() : nullptr, + primal_field.Size()); + + const mfem::ParFiniteElementSpace& mesh_space = *sim_state.m_mesh_fes; + Group* space_info = fields->createGroup("mesh_space"); + space_info->createViewScalar("order", sim_state.m_options.mesh.order); + space_info->createViewScalar("vsize", static_cast(mesh_space.GetVSize())); + space_info->createViewScalar("true_vsize", static_cast(mesh_space.TrueVSize())); + space_info->createViewScalar("global_true_vsize", + static_cast(mesh_space.GlobalTrueVSize())); + space_info->createViewScalar("mesh_ref_element_hash", ElementLocalFieldHash(mesh_ref)); +} + +/** + * @brief Write the `qspaces/` group: validation data for every quadrature space + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Quadrature spaces are not restored; the restarted run rebuilds them in its + * constructor from the mesh and the regions. What is stored lets ValidateQuadratureSpaces() + * confirm that each rebuilt space matches the original one point for point, so a quadrature + * function's stored values can be copied straight into the rebuilt function: the integration + * order, the number of quadrature point values, the number of local elements, whether the + * space covers the whole local mesh, and a hash of its local-to-global element map (which + * elements, in which order). + * + * The global spaces exist on every rank, possibly empty. Region spaces only exist on ranks + * that own elements of the region, so they are only written there; the same rank of the + * restarted run will not have them either. + */ +void RestartCheckpoint::WriteQuadratureSpaces(Group* root, const SimulationState& sim_state) const { + Group* qspaces = root->createGroup("qspaces"); + for (const auto& [name, space] : sim_state.m_map_qs) { + if (!space) { + continue; + } + Group* entry = qspaces->createGroup(SidreSafeName(name)); + entry->createViewScalar("order", space->GetOrder()); + entry->createViewScalar("size", static_cast(space->GetSize())); + entry->createViewScalar("num_local_elements", + static_cast(space->GetNumLocalElements())); + entry->createViewScalar("is_full_space", space->IsFullSpace() ? 1 : 0); + entry->createViewScalar("local2global_hash", HashIntArray(space->GetLocal2Global())); + } +} + +/** + * @brief Write the `qfuncs/` group: every registered quadrature function + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Every non-null function in SimulationState's registry is written, except the ones + * that are recomputed before they are next read (RecomputedQuadratureFunctionNames()). Writing + * everything rather than a hand-picked list means functions added by future models or + * post-processing are checkpointed without any change here. Each entry records the name of + * the function's space, its vdim and its number of values, which the reader checks before + * restoring, and then either the values (an external view) or `copy_of`. + * + * **Begin/end pairs and `copy_of`.** Material state is kept in pairs: beginning-of-step + * (`_beg`) and end-of-step (`_end`) functions, registered in m_model_update_qf_pairs. + * SystemDriver::UpdateModel() ends with SimulationState::SetupModelVariables(), which copies + * every `_beg` into its `_end` partner, and the checkpoint is written after UpdateModel(), so + * the two normally hold identical values. Instead of assuming that (post-processing runs in + * between and its code is not part of this contract), each pair is compared bit for bit on + * this rank: + * - identical: the `_end` entry only records `copy_of = <_beg name>`, and the reader restores + * it by copying the restored `_beg` (BindFieldsAndQuadratureFunctions()). For the state + * variables and stresses this nearly halves the size of the checkpoint; + * - different: the `_end` values are written like any other function. + * Either way the restart is bitwise exact. Since the comparison is per rank, one rank's file + * may use `copy_of` for a pair that another rank's file writes in full; each rank only reads + * its own file. + * + * Empty functions (a global function on a rank with no elements) are never compared, since + * their data pointers may be null, and are written as described zero-length views. + */ +void RestartCheckpoint::WriteQuadratureFunctions(Group* root, + const SimulationState& sim_state) const { + // End-of-step name -> beginning-of-step name, for the pairs that are bitwise identical. + std::map end_copy_of; + for (const auto& [beg_name, end_name] : sim_state.m_model_update_qf_pairs) { + const auto beg_it = sim_state.m_map_qfs.find(beg_name); + const auto end_it = sim_state.m_map_qfs.find(end_name); + if (beg_it == sim_state.m_map_qfs.end() || end_it == sim_state.m_map_qfs.end() || + !beg_it->second || !end_it->second) { + continue; + } + const mfem::expt::PartialQuadratureFunction& beg_function = *beg_it->second; + const mfem::expt::PartialQuadratureFunction& end_function = *end_it->second; + if (beg_function.Size() == 0 || beg_function.Size() != end_function.Size() || + beg_function.GetVDim() != end_function.GetVDim() || + beg_function.GetPartialSpaceShared() != end_function.GetPartialSpaceShared()) { + continue; + } + const std::size_t byte_count = static_cast(beg_function.Size()) * + sizeof(double); + if (std::memcmp(beg_function.HostRead(), end_function.HostRead(), byte_count) == 0) { + end_copy_of[end_name] = beg_name; + } + } + + const std::set recomputed = RecomputedQuadratureFunctionNames(sim_state); + Group* qfuncs = root->createGroup("qfuncs"); + for (const auto& [name, function] : sim_state.m_map_qfs) { + if (!function || recomputed.count(name) != 0) { + continue; + } + Group* entry = qfuncs->createGroup(SidreSafeName(name)); + entry->createViewString( + "qspace_name", + FindQuadratureSpaceName(sim_state.m_map_qs, function->GetPartialSpaceShared().get())); + entry->createViewScalar("vdim", function->GetVDim()); + entry->createViewScalar("value_count", static_cast(function->Size())); + const auto copy_it = end_copy_of.find(name); + if (copy_it != end_copy_of.end()) { + entry->createViewString("copy_of", copy_it->second); + } else { + CreateExternalDoubleView(entry, + "values", + function->Size() > 0 ? function->HostRead() : nullptr, + function->Size()); + } + } +} + +/** + * @brief Write the `materials/` group: the identity and properties of every region + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details For each region: the material name, the model type and the effective number of + * state variables (including the orientation components SimulationState adds), and the + * material properties. ValidateMaterials() aborts the restart if the identity changed, since + * the stored state variables would then be meaningless, but only warns if the properties + * changed, since editing properties for a continuation is a legitimate use. + * + * Every rank holds the material list and properties for every region, whether or not it owns + * elements of that region, so every rank writes the same group. + */ +void RestartCheckpoint::WriteMaterials(Group* root, const SimulationState& sim_state) const { + Group* materials = root->createGroup("materials"); + const int num_regions = static_cast(sim_state.m_material_name_region.size()); + materials->createViewScalar("num_regions", num_regions); + for (int region = 0; region < num_regions; ++region) { + const auto region_index = static_cast(region); + Group* entry = materials->createGroup("region_" + std::to_string(region)); + entry->createViewString("name", sim_state.m_material_name_region[region_index].first); + entry->createViewScalar("mech_type", + static_cast(sim_state.m_region_material_type[region_index])); + entry->createViewScalar("num_state_vars", + sim_state.m_options.materials[region_index].state_vars.num_vars); + const std::vector& properties = sim_state.GetMaterialProperties(region); + CreateOwnedDoubleCopy( + entry, "properties", properties.data(), static_cast(properties.size())); + } +} + +/** + * @brief Write the commit file that marks a checkpoint as complete + * + * @param checkpoint_directory Directory of the checkpoint + * @param completed_cycles Number of completed cycles the checkpoint holds + * @param driver_step Driver loop counter of the checkpoint + * @param completed_time End time of the last completed step + * @return True if the commit file is in place + * + * @details Called on rank 0 only, after every rank saved its file. The file is written under + * a temporary name and then renamed. A rename within one file system is atomic on POSIX, so a + * reader either sees no commit file or a complete one, never a partial one. Besides marking + * the checkpoint complete, the commit file carries the format version and rank count that + * OpenForRestart() checks before any rank opens its own file. It uses the JSON protocol so it + * can be read by eye. + */ +bool RestartCheckpoint::WriteCommitFile(const fs::path& checkpoint_directory, + const std::size_t completed_cycles, + const std::size_t driver_step, + const double completed_time) const { + DataStore commit_store; + Group* commit = commit_store.getRoot(); + commit->createViewScalar("format_version", RESTART_FORMAT_VERSION); + commit->createViewScalar("num_ranks", m_num_ranks); + commit->createViewScalar("completed_cycles", static_cast(completed_cycles)); + commit->createViewScalar("driver_step", static_cast(driver_step)); + commit->createViewScalar("completed_time", completed_time); + commit->createViewString("rank_file_protocol", RANK_FILE_PROTOCOL); + + const fs::path temporary = checkpoint_directory / (std::string(COMMIT_FILE_NAME) + ".tmp"); + if (!commit->save(temporary.string(), COMMIT_PROTOCOL)) { + return false; + } + std::error_code error; + fs::rename(temporary, checkpoint_directory / COMMIT_FILE_NAME, error); + return !error; +} + +/** + * @brief Rename checkpoints newer than this run's starting point out of the way + * + * @param completed_cycles This run's starting point: the restart cycle, or 0 for a fresh run + * + * @details Called on rank 0 only, on this run's first write. Checkpoints in the write + * directory that are newer than the starting point belong to a different timeline: a fresh + * run reusing an old directory, or a restart from an older checkpoint. If they stayed, a + * later restart that picks "the newest committed checkpoint" could pick one of them instead + * of this run's. They are renamed to `.superseded` (or `.superseded.N`), never deleted, + * so nothing is lost if this was a mistake; the renamed directories no longer match + * ParseCycleDirectoryName() and are ignored. A rename failure is reported and otherwise + * ignored, since it only affects which checkpoint a later automatic selection would pick. + */ +void RestartCheckpoint::SupersedeCheckpointsAfter(const std::size_t completed_cycles) const { + for (const CheckpointDirectoryEntry& entry : ListCheckpointDirectories(m_options.directory)) { + if (entry.completed_cycles <= completed_cycles) { + continue; + } + fs::path target = entry.path; + target += ".superseded"; + for (int suffix = 1; fs::exists(target); ++suffix) { + target = entry.path; + target += ".superseded." + std::to_string(suffix); + } + std::error_code error; + fs::rename(entry.path, target, error); + if (error) { + MFEM_WARNING_0("Unable to rename the superseded checkpoint " << entry.path << ": " + << error.message()); + } else { + std::cout << "Renamed superseded restart checkpoint " << entry.path.string() << " to " + << target.string() << std::endl; + } + } +} + +/** + * @brief Remove checkpoints beyond the configured retention + * + * @details Called on rank 0 only, right after a successful commit, so the newest checkpoint + * is always complete before anything older is removed. Two kinds of directories are removed + * from the write directory: + * - the oldest committed checkpoints beyond the newest `keep_last` (keep_last = 0 keeps all); + * - incomplete (uncommitted) checkpoints older than the newest committed one, which are left + * over from failed or interrupted writes and can never be restarted from. + * An incomplete checkpoint newer than the newest committed one is left alone, since it may be + * a write still in progress by another run. A removal failure is reported and otherwise + * ignored; it only costs disk space. + */ +void RestartCheckpoint::ApplyRetentionPolicy() const { + const std::vector entries = ListCheckpointDirectories( + m_options.directory); + std::size_t newest_committed = 0; + bool any_committed = false; + std::vector committed; + for (const CheckpointDirectoryEntry& entry : entries) { + if (entry.committed) { + committed.push_back(&entry); + newest_committed = entry.completed_cycles; + any_committed = true; + } + } + + std::vector to_remove; + if (m_options.keep_last > 0 && + committed.size() > static_cast(m_options.keep_last)) { + // entries is sorted oldest first, so the first remove_count committed ones go. + const std::size_t remove_count = committed.size() - + static_cast(m_options.keep_last); + for (std::size_t i = 0; i < remove_count; ++i) { + to_remove.push_back(committed[i]->path); + } + } + for (const CheckpointDirectoryEntry& entry : entries) { + if (!entry.committed && any_committed && entry.completed_cycles < newest_committed) { + to_remove.push_back(entry.path); + } + } + + for (const fs::path& path : to_remove) { + std::error_code error; + fs::remove_all(path, error); + if (error) { + MFEM_WARNING_0("Unable to remove the old checkpoint " << path << ": " + << error.message()); + } + } +} + +// ========================================================================= +// RESTARTING, PHASE 1 (inside the SimulationState constructor) +// ========================================================================= + +/** + * @brief Pick the checkpoint directory to restart from and check its commit file + * + * @return Path of the selected checkpoint directory, the same on every rank + * + * @details Collective. Only rank 0 looks at the file system, so every rank is guaranteed to + * open the same checkpoint even if directories appear or disappear while the run starts. + * Rank 0 selects `[Restart] read_cycle` in the read directory if it is set, otherwise the + * newest committed checkpoint there, and reads its commit file. Any problem (requested + * checkpoint missing or incomplete, nothing committed, unreadable commit file) is broadcast as + * a message so every rank aborts with it. + * + * The format version and rank count from the commit file are then checked on every rank. + * Checking them from the commit file, before any rank opens its own file, gives a clear + * message for the common mistake of restarting with a different number of ranks, instead of + * some ranks finding no file at all. The checkpoint layout is partition dependent (each rank + * holds its own piece of the mesh and state), so the rank count has to match exactly. + */ +std::string RestartCheckpoint::SelectCheckpointDirectory() { + std::string selected_directory; + std::string selection_error; + int commit_format_version = -1; + int commit_num_ranks = -1; + if (m_rank == 0) { + const fs::path& read_directory = m_options.GetReadDirectory(); + fs::path selected; + if (m_options.read_cycle.has_value()) { + selected = read_directory / CycleDirectoryName(*m_options.read_cycle); + if (!fs::exists(selected / COMMIT_FILE_NAME)) { + selection_error = "the requested checkpoint " + selected.string() + + " does not exist or is incomplete (no " + COMMIT_FILE_NAME + ")"; + } + } else { + const auto entries = ListCheckpointDirectories(read_directory); + for (auto it = entries.rbegin(); it != entries.rend(); ++it) { + if (it->committed) { + selected = it->path; + break; + } + } + if (selected.empty()) { + selection_error = "no complete checkpoint was found in " + read_directory.string(); + } + } + if (selection_error.empty()) { + DataStore commit_store; + Group* commit = commit_store.getRoot(); + if (commit->load((selected / COMMIT_FILE_NAME).string(), COMMIT_PROTOCOL) && + commit->hasView("format_version") && commit->hasView("num_ranks")) { + commit_format_version = ReadSidreScalar(commit, "format_version"); + commit_num_ranks = ReadSidreScalar(commit, "num_ranks"); + selected_directory = selected.string(); + } else { + selection_error = "unable to read " + (selected / COMMIT_FILE_NAME).string(); + } + } + } + + BroadcastString(selection_error, m_comm); + MFEM_VERIFY_0(selection_error.empty(), "Unable to restart: " << selection_error); + BroadcastString(selected_directory, m_comm); + MPI_Bcast(&commit_format_version, 1, MPI_INT, 0, m_comm); + MPI_Bcast(&commit_num_ranks, 1, MPI_INT, 0, m_comm); + MFEM_VERIFY_0(commit_format_version == RESTART_FORMAT_VERSION, + "Unable to restart: the checkpoint " + << selected_directory << " has format version " << commit_format_version + << ", but this build reads version " << RESTART_FORMAT_VERSION); + MFEM_VERIFY_0(commit_num_ranks == m_num_ranks, + "Unable to restart: the checkpoint " + << selected_directory << " was written with " << commit_num_ranks + << " MPI ranks, but this run has " << m_num_ranks + << "; a restart must use the same number of ranks"); + return selected_directory; +} + +/** + * @details Implementation notes. + * 1. SelectCheckpointDirectory() picks the checkpoint on rank 0 and checks its commit file on + * every rank. + * 2. Every rank loads its own rank file into m_read_store, which stays alive until the end of + * RestoreState(). Group::load() materializes the owned arrays (the mesh, mesh_ref, grain + * IDs and metadata) and restores the external views (the bulk state) as descriptions only; + * their data is read later, straight into SimulationState's buffers. + * 3. Each rank checks that its file's restart_info matches: format version, rank count and, + * importantly, its own rank, which catches files renamed or copied between ranks. The + * stored time state and driver step are kept for phase 2. + * 4. m_last_written_cycles is set to the restart point, so the interval for this run's first + * checkpoint is counted from the restart, and its first write supersedes any checkpoints + * newer than the restart point (SupersedeCheckpointsAfter()). + * + * A problem with one rank's file is recorded and every rank aborts together at the end of the + * stage (AbortIfAnyRankFailed()). + */ +void RestartCheckpoint::OpenForRestart() { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::OpenForRestart"); + + const std::string selected_directory = SelectCheckpointDirectory(); + + m_read_rank_file = fs::path(selected_directory) / RankFileName(m_rank); + m_read_store = std::make_unique(); + Group* root = m_read_store->getRoot(); + if (!fs::exists(m_read_rank_file) || + !root->load(m_read_rank_file.string(), RANK_FILE_PROTOCOL)) { + RecordFailure("unable to load " + m_read_rank_file.string()); + } else if (!root->hasGroup("restart_info") || !root->hasGroup("mesh")) { + RecordFailure(m_read_rank_file.string() + " has no restart_info/ or mesh/ group"); + } else { + Group* info = root->getGroup("restart_info"); + const int stored_version = ReadSidreScalar(info, "format_version"); + const int stored_rank = ReadSidreScalar(info, "rank"); + const int stored_num_ranks = ReadSidreScalar(info, "num_ranks"); + if (stored_version != RESTART_FORMAT_VERSION || stored_rank != m_rank || + stored_num_ranks != m_num_ranks) { + RecordFailure(m_read_rank_file.string() + " was written by rank " + + std::to_string(stored_rank) + " of " + std::to_string(stored_num_ranks) + + " with format version " + std::to_string(stored_version) + + ", which does not match this rank"); + } else { + m_restart_driver_step = static_cast( + ReadSidreScalar(info, "driver_step")); + m_restart_time_state.completed_cycles = static_cast( + ReadSidreScalar(info, "completed_cycles")); + m_restart_time_state.completed_time = ReadSidreScalar(info, "completed_time"); + m_restart_time_state.next_dt = ReadSidreScalar(info, "next_dt"); + m_restart_time_state.previous_dt = ReadSidreScalar(info, "previous_dt"); + m_restart_time_state.original_dt = ReadSidreScalar(info, "original_dt"); + } + } + AbortIfAnyRankFailed("opening the checkpoint"); + + m_last_written_cycles = m_restart_time_state.completed_cycles; + + if (m_rank == 0) { + std::cout << "Restarting from checkpoint " << selected_directory << " (cycle " + << m_restart_time_state.completed_cycles << ", driver step " + << m_restart_driver_step << ", t = " << m_restart_time_state.completed_time << ")" + << std::endl; + } +} + +/** + * @details Implementation notes. ExaSidreParMesh rebuilds the mesh and checks the stored + * arrays itself (collectively). This function adds a final check that the edges, faces and + * communication groups MFEM regenerated from the connectivity match the counts recorded at + * write time. The mesh is returned as a plain mfem::ParMesh, so SimulationState owns it like + * any other mesh. + */ +std::shared_ptr RestartCheckpoint::ReconstructMesh() { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::ReconstructMesh"); + MFEM_VERIFY_0(m_read_store, "ReconstructMesh() requires OpenForRestart() first"); + Group* mesh_group = m_read_store->getRoot()->getGroup("mesh"); + + std::shared_ptr mesh = ExaSidreParMesh::Create(mesh_group, m_comm); + + const MeshTopologyCounts stored = ReadStoredMeshTopologyCounts(mesh_group); + const MeshTopologyCounts rebuilt = CountMeshTopology(*mesh); + if (!stored.Matches(rebuilt)) { + std::ostringstream message; + message << "the rebuilt mesh topology differs from the checkpoint (vertices, elements, " + "boundary elements, edges, faces, communication groups: stored " + << stored.num_vertices << ", " << stored.num_elements << ", " + << stored.num_boundary_elements << ", " << stored.num_edges << ", " + << stored.num_faces << ", " << stored.num_communication_groups << "; rebuilt " + << rebuilt.num_vertices << ", " << rebuilt.num_elements << ", " + << rebuilt.num_boundary_elements << ", " << rebuilt.num_edges << ", " + << rebuilt.num_faces << ", " << rebuilt.num_communication_groups << ")"; + RecordFailure(message.str()); + } + AbortIfAnyRankFailed("rebuilding the mesh"); + return mesh; +} + +/** + * @details Implementation notes. The grain IDs were written by WriteMeshAndGrainIds() as one + * int per local element, in local element order. The rebuilt mesh has the same local element + * order, so they are copied straight across. The count is checked against the local element + * count, which also covers ranks with no elements (both zero, nothing copied). + */ +void RestartCheckpoint::ReadElementGrainIds(mfem::Array& grain_ids) { + MFEM_VERIFY_0(m_read_store, "ReadElementGrainIds() requires OpenForRestart() first"); + Group* mesh_group = m_read_store->getRoot()->getGroup("mesh"); + const std::string path = "fields/grain_id/values"; + if (!mesh_group->hasView(path)) { + RecordFailure("the checkpoint has no mesh/" + path); + } else { + View* view = mesh_group->getView(path); + if (view->getNumElements() != grain_ids.Size()) { + RecordFailure("the checkpoint has " + std::to_string(view->getNumElements()) + + " grain IDs for " + std::to_string(grain_ids.Size()) + " local elements"); + } else if (grain_ids.Size() > 0) { + const int* stored = view->getData(); + std::copy(stored, stored + grain_ids.Size(), grain_ids.HostWrite()); + } + } + AbortIfAnyRankFailed("reading the grain IDs"); +} + +/** + * @details Implementation notes. + * + * **Why the coordinates are restored during construction.** "Build first, overlay last" + * would normally leave every field for phase 2. The coordinates are the exception, because + * some constructors compute and keep values derived from the starting geometry: + * - SystemDriver's constructor, with mono_def_bcs, selects essential DOFs from the mesh nodes + * using tight min/max coordinate comparisons (1e-12). + * - AbaqusUmatModel::InitLocSFGrads() stores reference shape-function gradients that every + * later step reuses. + * In a fresh run these see the reference coordinates projected from the original mesh file's + * Nodes. The rebuilt mesh has no Nodes, so the SimulationState constructor's + * Mesh::GetNodes(GridFunction&) projects from the vertex positions instead, which agrees with + * the original only to rounding (a few entries differ by about 1e-16). Small as that is, it + * changes those stored values and so the restarted run would no longer be bitwise identical. + * Overwriting the coordinate fields with the stored mesh_ref bits right after they are created + * gives every constructor exactly the geometry it saw in the original run. This is safe: in + * the original run mesh_ref, mesh_t_beg and mesh_current all held exactly these values until + * the first step (the constructor sets them equal and SwapNodes() makes mesh_current the + * mesh's Nodes; the driver only zeroes the velocity and displacement). + * + * **Steps.** + * 1. ValidateMeshSpace(): the coordinate space must have the same order and sizes as in the + * writing run, or the stored vector cannot be interpreted at all. + * 2. Check that mesh_ref was stored as an owned double array of the space's local size. It + * has to be owned: an external view would have no data until loadExternalData(), which + * phase 2 does. + * 3. Copy it into mesh_ref, mesh_t_beg and mesh_current, then tell the mesh its Nodes changed + * (mesh_current is the mesh's Nodes) so cached geometric factors are dropped. + * 4. VerifyRestoredDofLayout(): confirm, on the overlaid coordinates, that the rebuilt space + * numbers its DOFs exactly as the writing run did, before anything is built on it. + * 5. Drop the stored mesh_ref; it has been copied. + * + * On a rank with no elements the space and every vector are empty, nothing is copied, and the + * rank still takes part in the collective checks. + */ +void RestartCheckpoint::RestoreReferenceConfiguration(SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::RestoreReferenceConfiguration"); + MFEM_VERIFY_0(m_read_store, "RestoreReferenceConfiguration() requires OpenForRestart() first"); + Group* root = m_read_store->getRoot(); + + // Step 1: the coordinate space must match before the stored vector means anything. + ValidateMeshSpace(root, sim_state); + + // Step 2: locate the owned copy of the reference coordinates. + mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + const std::string path = "fields/mesh_ref/values"; + const double* stored_reference = nullptr; + if (!root->hasView(path)) { + RecordFailure("the checkpoint has no " + path); + } else { + View* view = root->getView(path); + if (view->isExternal() || view->getNumElements() != mesh_ref.Size() || + (mesh_ref.Size() > 0 && view->getTypeID() != axom::sidre::FLOAT64_ID)) { + RecordFailure(path + " is not an owned double array with one entry per local DOF " + "of the coordinate space"); + } else if (mesh_ref.Size() > 0) { + stored_reference = view->getData(); + } + } + AbortIfAnyRankFailed("restoring the reference configuration"); + + // Step 3: overwrite all three coordinate fields with the original reference bits. + for (const char* name : {"mesh_ref", "mesh_t_beg", "mesh_current"}) { + mfem::ParGridFunction& field = *sim_state.m_mesh_nodes.at(name); + if (field.Size() > 0) { + std::copy(stored_reference, stored_reference + field.Size(), field.HostWrite()); + } + } + sim_state.m_mesh->NodesUpdated(); + + // Step 4: verify the DOF layout before anything is built on these coordinates. + VerifyRestoredDofLayout(sim_state); + + // Step 5: the stored copy is no longer needed. + root->destroyGroupAndData("fields/mesh_ref"); +} + +/** + * @details Implementation notes. Called at the end of the SimulationState constructor, once + * the mesh has been rebuilt and the grain IDs copied out. The mesh arrays are often the + * largest owned data in the file, so freeing them early keeps the peak memory of the restart + * down; the rest of the store (the bulk state descriptions) is needed until phase 2. + */ +void RestartCheckpoint::ReleaseMeshArrays() { + if (m_read_store && m_read_store->getRoot()->hasGroup("mesh")) { + m_read_store->getRoot()->destroyGroupAndData("mesh"); + } +} + +/** + * @brief Check that the coordinate space matches the one the checkpoint was written with + * + * @param root Root group of the loaded rank file + * @param sim_state Simulation state under construction + * + * @details Compares the mesh order and the coordinate space's local, true and global true DOF + * counts with the stored values, and keeps the stored DOF-layout hash for + * VerifyRestoredDofLayout(). A different order (or, equivalently, different sizes) means the + * stored coordinate and velocity vectors cannot be mapped onto this space at all, so this is + * recorded as a failure; the caller aborts at the end of its stage. + */ +void RestartCheckpoint::ValidateMeshSpace(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("fields/mesh_space")) { + RecordFailure("the checkpoint has no fields/mesh_space group"); + return; + } + Group* space_info = root->getGroup("fields/mesh_space"); + const mfem::ParFiniteElementSpace& mesh_space = *sim_state.m_mesh_fes; + const int stored_order = ReadSidreScalar(space_info, "order"); + const auto stored_vsize = ReadSidreScalar(space_info, "vsize"); + const auto stored_true_vsize = ReadSidreScalar(space_info, "true_vsize"); + const auto stored_global_true_vsize = ReadSidreScalar(space_info, + "global_true_vsize"); + m_stored_mesh_ref_element_hash = ReadSidreScalar(space_info, + "mesh_ref_element_hash"); + if (stored_order != sim_state.m_options.mesh.order) { + RecordFailure("the mesh order changed (checkpoint " + std::to_string(stored_order) + + ", this run " + std::to_string(sim_state.m_options.mesh.order) + + "); the mesh order cannot change on a restart"); + } + if (stored_vsize != mesh_space.GetVSize() || stored_true_vsize != mesh_space.TrueVSize() || + stored_global_true_vsize != static_cast(mesh_space.GlobalTrueVSize())) { + RecordFailure( + "the coordinate space sizes differ (local, true, global true DOFs: checkpoint " + + std::to_string(stored_vsize) + ", " + std::to_string(stored_true_vsize) + ", " + + std::to_string(stored_global_true_vsize) + "; this run " + + std::to_string(mesh_space.GetVSize()) + ", " + std::to_string(mesh_space.TrueVSize()) + + ", " + std::to_string(static_cast(mesh_space.GlobalTrueVSize())) + ")"); + } +} + +/** + * @brief Verify that the rebuilt coordinate space numbers its DOFs as in the writing run + * + * @param sim_state Simulation state whose mesh_ref has just been overwritten + * + * @details Collective. Matching sizes are not enough: the stored vectors are only correct if + * every value lands on the same DOF it came from. Two independent checks, each run on the + * overlaid reference coordinates (a continuous field with distinct values at distinct nodes): + * + * (a) Local DOF numbering. ElementLocalFieldHash() of mesh_ref must equal the hash recorded + * at write time. It visits the values element by element in each element's DOF order, so + * any difference in how this rank numbers its local DOFs changes it. + * + * (b) Shared DOF matching across ranks. SharedDofInconsistency() restricts mesh_ref to true + * DOFs and prolongates it back. If every shared DOF is matched with the same true DOF on + * every rank, the field comes back unchanged up to rounding; a mismatch, such as a wrong + * shared-face orientation at H1 order >= 3, moves another node's coordinate onto the DOF, + * a jump of the order of the element size. The tolerance is 1e-10 times the largest + * coordinate magnitude in the mesh (agreed on with an MPI_Allreduce so every rank uses the + * same value), far above rounding and far below any real mismatch. A mesh whose + * coordinates are all zero (only possible on a degenerate mesh) skips the check. + * + * Every rank computes both checks, including ranks with no DOFs, since (b) and the scale + * reduction are collective. Failures abort every rank. + */ +void RestartCheckpoint::VerifyRestoredDofLayout(const SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::VerifyRestoredDofLayout"); + const mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + + // (a) Local DOF numbering. + if (ElementLocalFieldHash(mesh_ref) != m_stored_mesh_ref_element_hash) { + RecordFailure("the local DOF numbering of the rebuilt mesh differs from the checkpoint " + "(the element-local hash of mesh_ref does not match)"); + } + + // (b) Shared DOF matching across ranks. + double local_scale = 0.0; + if (mesh_ref.Size() > 0) { + const double* reference = mesh_ref.HostRead(); + for (int i = 0; i < mesh_ref.Size(); ++i) { + local_scale = std::max(local_scale, std::abs(reference[i])); + } + } + double global_scale = 0.0; + MPI_Allreduce(&local_scale, &global_scale, 1, MPI_DOUBLE, MPI_MAX, m_comm); + const double inconsistency = SharedDofInconsistency(mesh_ref); + const double tolerance = 1.0e-10 * global_scale; + if (global_scale > 0.0 && inconsistency > tolerance) { + std::ostringstream message; + message << "shared DOFs are matched inconsistently across ranks (max |x - P R x| = " + << inconsistency << ", tolerance " << tolerance << ")"; + RecordFailure(message.str()); + } + AbortIfAnyRankFailed("verifying the restored DOF layout"); +} + +// ========================================================================= +// RESTARTING, PHASE 2 (from the driver) +// ========================================================================= + +/** + * @details Implementation notes. By the time this runs, SimulationState, SystemDriver and + * PostProcessingDriver have all been constructed exactly as in a fresh run, so every buffer + * the checkpoint can fill exists. The steps are ordered so that nothing is overwritten until + * everything has been checked: + * + * 1. Validate (ValidateMaterials(), ValidateQuadratureSpaces()). A failure here aborts before + * any buffer is touched, so SimulationState is never left half restored. The coordinate + * space and DOF layout were already validated in phase 1. + * 2. Bind (BindFieldsAndQuadratureFunctions()). Every external view in the checkpoint that + * has a matching SimulationState buffer is pointed at that buffer's host memory. Functions + * stored as `copy_of` are queued in m_pending_copies instead. + * 3. Drop unbound external views. Group::loadExternalData() reads every external view + * described in the group whether or not it was bound, and an unbound view has nowhere to + * put its data. The unbound ones are zero-length arrays and functions that are only in the + * checkpoint (with a warning from step 2); they are destroyed first. + * 4. Load. A single loadExternalData() call per rank reads all bound arrays straight into + * SimulationState's buffers. HostReadWrite() during binding made the host copy the valid + * one, so on a GPU run the next device access copies the restored values over. The queued + * `copy_of` functions are then filled from their restored partners. + * 5. Resume time (TimeManagement::ResumeFromCheckpoint()) and recompute the fields derived + * from the restored state (ApplyPostRestartFixup()). + * 6. Release the checkpoint. From here on nothing restored is owned by Sidre. + * + * Collective, including on ranks with no elements: they bind nothing (every array of theirs + * is empty) but still take part in every agreement step. + */ +void RestartCheckpoint::RestoreState(SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::RestoreState"); + MFEM_VERIFY_0(m_read_store, "RestoreState() requires OpenForRestart() first"); + const double start_seconds = MPI_Wtime(); + Group* root = m_read_store->getRoot(); + + // Step 1: validate everything before anything is overwritten. + ValidateMaterials(root, sim_state); + ValidateQuadratureSpaces(root, sim_state); + AbortIfAnyRankFailed("checking the checkpoint against this run"); + + // Step 2: point each stored array at the SimulationState buffer it restores. + BindFieldsAndQuadratureFunctions(root, sim_state); + AbortIfAnyRankFailed("matching the checkpoint arrays to this run"); + + // Step 3: loadExternalData() would try to read unbound views too, so drop them. + std::vector> unbound; + CollectUnboundExternalViews(root, unbound); + for (auto& [group, view_name] : unbound) { + group->destroyView(view_name); + } + + // Step 4: one bulk read into the bound buffers, then fill the copy_of functions. + if (!root->loadExternalData(m_read_rank_file.string())) { + RecordFailure("unable to read the state arrays from " + m_read_rank_file.string()); + } + AbortIfAnyRankFailed("reading the checkpoint arrays"); + for (auto& [destination, source] : m_pending_copies) { + *destination = static_cast(*source); + } + m_pending_copies.clear(); + + // Step 5: time, then the fields derived from the restored state. + sim_state.m_time_manager.ResumeFromCheckpoint(m_restart_time_state); + ApplyPostRestartFixup(sim_state); + + // Step 6: nothing restored is owned by Sidre from here on. + m_read_store.reset(); + + const TimeManagement& time_manager = sim_state.m_time_manager; + if (m_rank == 0) { + std::cout << "Restored checkpoint state in " << (MPI_Wtime() - start_seconds) + << " s; next step: cycle " << (m_restart_time_state.completed_cycles + 1) + << ", driver step " << (m_restart_driver_step + 1) + << ", t_end = " << time_manager.GetTime() + << ", dt = " << time_manager.GetDeltaTime() + << ", t_final = " << time_manager.GetFinalTime() << std::endl; + } + if (time_manager.IsFinished()) { + MFEM_WARNING_0("The checkpoint is already at the configured final time, so there is " + "nothing left to run; increase t_final (or extend the custom time steps) " + "to continue the simulation"); + } +} + +/** + * @brief Check that the material regions match the ones the checkpoint was written with + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details The stored state variables are only meaningful for the same material model with + * the same state-variable layout, so a different number of regions, or a different name, + * model type or effective state-variable count for any region, is recorded as a failure. + * Changed material properties only produce a warning: editing properties for a continuation + * is legitimate, and this run's values are the ones used (the models read them from + * SimulationState when they are constructed). The warning is issued on every rank through + * MFEM_WARNING_0, which prints it once; every rank holds the same material data, so every + * rank reaches the same conclusion. + */ +void RestartCheckpoint::ValidateMaterials(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("materials")) { + RecordFailure("the checkpoint has no materials/ group"); + return; + } + Group* materials = root->getGroup("materials"); + const int stored_regions = ReadSidreScalar(materials, "num_regions"); + const int runtime_regions = static_cast(sim_state.m_material_name_region.size()); + if (stored_regions != runtime_regions) { + RecordFailure("the number of material regions changed (checkpoint " + + std::to_string(stored_regions) + ", this run " + + std::to_string(runtime_regions) + ")"); + return; + } + for (int region = 0; region < runtime_regions; ++region) { + const auto region_index = static_cast(region); + Group* entry = materials->getGroup("region_" + std::to_string(region)); + const std::string stored_name = entry->getView("name")->getString(); + const int stored_type = ReadSidreScalar(entry, "mech_type"); + const int stored_state_vars = ReadSidreScalar(entry, "num_state_vars"); + const std::string& runtime_name = sim_state.m_material_name_region[region_index].first; + const int runtime_type = static_cast(sim_state.m_region_material_type[region_index]); + const int runtime_state_vars = + sim_state.m_options.materials[region_index].state_vars.num_vars; + if (stored_name != runtime_name || stored_type != runtime_type || + stored_state_vars != runtime_state_vars) { + RecordFailure("region " + std::to_string(region + 1) + + " changed its material name, model type or number of state variables " + "(checkpoint '" + + stored_name + "', type " + std::to_string(stored_type) + ", " + + std::to_string(stored_state_vars) + " state variables; this run '" + + runtime_name + "', type " + std::to_string(runtime_type) + ", " + + std::to_string(runtime_state_vars) + " state variables)"); + continue; + } + + const std::vector& runtime_properties = sim_state.GetMaterialProperties(region); + View* stored_view = entry->getView("properties"); + const IndexType stored_count = stored_view->getNumElements(); + bool same = stored_count == static_cast(runtime_properties.size()); + if (same && stored_count > 0) { + const double* stored = stored_view->getData(); + same = std::equal(runtime_properties.begin(), runtime_properties.end(), stored); + } + if (!same) { + MFEM_WARNING_0("The material properties of region " + << (region + 1) << " ('" << runtime_name + << "') differ from the checkpoint; the restart continues with this " + "run's properties"); + } + } +} + +/** + * @brief Check that the rebuilt quadrature spaces match the ones the checkpoint was written + * with + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details Every quadrature space present both in this run and in the checkpoint must have the + * same integration order, number of values, number of local elements, full/partial status and + * local-to-global element map (by hash). Only then can a quadrature function's stored values + * be copied straight into the rebuilt function, point for point. A space present on only one + * side is not an error here: the functions on it are handled (and reported) during binding. + * + * On a rank with no elements only the (empty) global spaces exist and they match trivially; + * region spaces exist on neither side. + */ +void RestartCheckpoint::ValidateQuadratureSpaces(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("qspaces")) { + RecordFailure("the checkpoint has no qspaces/ group"); + return; + } + Group* qspaces = root->getGroup("qspaces"); + for (const auto& [name, space] : sim_state.m_map_qs) { + if (!space || !qspaces->hasGroup(SidreSafeName(name))) { + continue; + } + Group* entry = qspaces->getGroup(SidreSafeName(name)); + const bool same = ReadSidreScalar(entry, "order") == space->GetOrder() && + ReadSidreScalar(entry, "size") == space->GetSize() && + ReadSidreScalar(entry, "num_local_elements") == + space->GetNumLocalElements() && + ReadSidreScalar(entry, "is_full_space") == + (space->IsFullSpace() ? 1 : 0) && + ReadSidreScalar(entry, "local2global_hash") == + HashIntArray(space->GetLocal2Global()); + if (!same) { + RecordFailure("the quadrature space '" + name + + "' differs from the checkpoint (integration order, size, element " + "subset or element order)"); + } + } +} + +/** + * @brief Point every stored state array at the SimulationState buffer it restores + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details Nothing is read yet; this only sets data pointers (View::setExternalDataPtr()) so + * that RestoreState()'s single loadExternalData() call reads every array straight into place. + * + * **Nodal fields.** `mesh_current` and `primal_field` must be external double arrays with + * exactly the size of the target vector. mesh_ref was already restored in phase 1, and the + * other nodal fields are recomputed (ApplyPostRestartFixup()). + * + * **Quadrature functions.** Every function registered in this run, except the recomputed ones + * (RecomputedQuadratureFunctionNames()), is matched with the checkpoint entry of the same + * name: + * - A different space name, vdim or value count is a failure; the values would not line up. + * - A `copy_of` entry is queued in m_pending_copies and filled after the load from its + * partner, which must exist in this run with the same size and have stored values in the + * checkpoint. + * - Otherwise the stored values view must be an external double array of the right size, and + * is pointed at the function's host data. + * - Empty functions (on ranks with no elements of their space) are skipped; their stored view + * is zero length and is dropped before the load. + * + * **Mismatched sets are warnings, not failures.** A function registered in this run but not in + * the checkpoint keeps the value its constructor gave it, which is expected when a + * continuation enables extra post-processing. A function only in the checkpoint is skipped, + * which is expected when a continuation disables some. Both are reported on rank 0 through + * MFEM_WARNING_0. Because region functions only exist where the region has elements, the + * lists reflect rank 0's view; in practice the sets are decided by the options and are the + * same everywhere a region exists. + * + * HostReadWrite() is used for the target pointers so that the host copy becomes the valid one + * and, on a GPU run, the next device access copies the restored values to the device. + */ +void RestartCheckpoint::BindFieldsAndQuadratureFunctions(Group* root, SimulationState& sim_state) { + // --------------------------------------------------------------------- + // Part 1: nodal fields + // + // Only two nodal vectors are restored here. mesh_ref was restored in phase 1 (and its + // stored copy already dropped), and every other nodal field is recomputed from these two + // afterwards (ApplyPostRestartFixup()): + // - mesh_current: all local DOFs of the coordinate space (VSize entries). This is also the + // mesh's Nodes, so restoring it moves the mesh to its checkpointed configuration. + // - primal_field: the velocity true DOFs (TrueVSize entries), the solver's unknown. + // + // Each stored view must be an external double array with exactly the target's size. It + // was written as an external view, so a stored owned view or a different type means the + // file does not have the expected layout. The sizes were already implied by the mesh-space + // check in phase 1, so a size mismatch here means the file is damaged. + // + // Binding only records the target pointer; the data is read later by RestoreState()'s + // loadExternalData() call. HostReadWrite() marks the host copy as the valid one, so on a + // GPU run the next device access copies the restored values to the device instead of + // using stale device data. On a rank with no elements both vectors are empty: the size + // check still runs (0 == 0), nothing is bound, and the zero-length stored view is dropped + // before the load. + // --------------------------------------------------------------------- + const auto bind_vector = [&](const std::string& path, mfem::Vector& target) { + if (!root->hasView(path)) { + RecordFailure("the checkpoint has no " + path); + return; + } + View* view = root->getView(path); + if (!view->isExternal() || view->getTypeID() != axom::sidre::FLOAT64_ID || + view->getNumElements() != target.Size()) { + RecordFailure(path + " has " + std::to_string(view->getNumElements()) + + " entries or the wrong type, but this run needs " + + std::to_string(target.Size()) + " doubles"); + return; + } + if (target.Size() > 0) { + view->setExternalDataPtr(target.HostReadWrite()); + } + }; + bind_vector("fields/mesh_current/values", *sim_state.m_mesh_nodes.at("mesh_current")); + bind_vector("fields/primal_field/values", *sim_state.m_primal_field); + + // --------------------------------------------------------------------- + // Part 2: quadrature functions + // + // The loop runs over this run's registry rather than over the checkpoint, because the + // question being answered is "where does each of this run's functions get its value + // from?". Functions that are only in the checkpoint are reported in part 3. + // + // For each registered function, in order: + // a. Skip it if it is null (an unused registry slot) or recomputed before it is next + // read (RecomputedQuadratureFunctionNames()); it keeps its constructor value until + // then, exactly as it would after a normal step. + // b. Look up the checkpoint entry of the same (encoded) name. If there is none, remember + // the name for the warning in part 3 and leave the function as constructed. + // c. Check that the stored function has the same space name, vdim and number of values. + // ValidateQuadratureSpaces() has already checked that a space of that name has the + // same points in the same order, so matching names plus matching shapes means value i + // of the stored array belongs at value i of this function. Anything else would put + // values on the wrong quadrature points, so it is a failure, not a warning. + // d. If the entry is a `copy_of`, its values were not stored because they were bitwise + // equal to its beginning-of-step partner (see WriteQuadratureFunctions()). Queue the + // copy for after the load. The partner must exist in this run with the same size, and + // must have stored values in the checkpoint; otherwise nothing could be copied. + // e. Otherwise bind the stored values to the function's host data, after checking that + // the view is an external double array of the right size. An empty function (a + // global function on a rank with no elements) is skipped before that check: there is + // nothing to read, and its zero-length stored view is dropped before the load. + // + // Every problem is recorded rather than aborting immediately, so all of them are reported + // together and every rank reaches the agreement in RestoreState(). + // --------------------------------------------------------------------- + if (!root->hasGroup("qfuncs")) { + RecordFailure("the checkpoint has no qfuncs/ group"); + return; + } + Group* qfuncs = root->getGroup("qfuncs"); + const std::set recomputed = RecomputedQuadratureFunctionNames(sim_state); + std::vector runtime_only; + for (const auto& [name, function] : sim_state.m_map_qfs) { + // (a) Unused or recomputed. + if (!function || recomputed.count(name) != 0) { + continue; + } + + // (b) Not in the checkpoint. + const std::string encoded = SidreSafeName(name); + if (!qfuncs->hasGroup(encoded)) { + runtime_only.push_back(name); + continue; + } + + // (c) Same space, vdim and number of values. + Group* entry = qfuncs->getGroup(encoded); + const std::string stored_space = entry->getView("qspace_name")->getString(); + const std::string runtime_space = FindQuadratureSpaceName( + sim_state.m_map_qs, function->GetPartialSpaceShared().get()); + const int stored_vdim = ReadSidreScalar(entry, "vdim"); + const auto stored_count = ReadSidreScalar(entry, "value_count"); + if (stored_space != runtime_space || stored_vdim != function->GetVDim() || + stored_count != function->Size()) { + RecordFailure("the quadrature function '" + name + + "' changed shape (checkpoint: space '" + stored_space + "', vdim " + + std::to_string(stored_vdim) + ", " + std::to_string(stored_count) + + " values; this run: space '" + runtime_space + "', vdim " + + std::to_string(function->GetVDim()) + ", " + + std::to_string(function->Size()) + " values)"); + continue; + } + + // (d) Stored as a copy of its partner: copy after the load. The source pointer is the + // partner's SimulationState function, which the load fills from its own stored values + // before m_pending_copies is processed. + if (entry->hasView("copy_of")) { + const std::string source_name = entry->getView("copy_of")->getString(); + const auto source_it = sim_state.m_map_qfs.find(source_name); + const bool source_restorable = + source_it != sim_state.m_map_qfs.end() && source_it->second && + source_it->second->Size() == function->Size() && + qfuncs->hasGroup(SidreSafeName(source_name)) && + qfuncs->getGroup(SidreSafeName(source_name))->hasView("values"); + if (!source_restorable) { + RecordFailure("the quadrature function '" + name + "' is stored as a copy of '" + + source_name + "', which cannot be restored in this run"); + continue; + } + m_pending_copies.emplace_back(function, source_it->second); + continue; + } + + // (e) Stored values: point the stored view at the function's host data. + if (function->Size() == 0) { + continue; + } + if (!entry->hasView("values")) { + RecordFailure("the quadrature function '" + name + "' has no stored values"); + continue; + } + View* values = entry->getView("values"); + if (!values->isExternal() || values->getTypeID() != axom::sidre::FLOAT64_ID || + values->getNumElements() != function->Size()) { + RecordFailure("the stored values of the quadrature function '" + name + + "' have the wrong type or size"); + continue; + } + values->setExternalDataPtr(function->HostReadWrite()); + } + + // --------------------------------------------------------------------- + // Part 3: report the functions present on only one side + // + // Neither case stops the restart (see the function documentation for why both are + // expected when a continuation changes its post-processing options). The checkpoint-only + // functions are left unbound here, so RestoreState() drops their views before the load. + // + // MFEM_WARNING_0 is used so each message appears once, from rank 0. It is not a + // collective call: each rank evaluates its own lists, and only rank 0's output is shown. + // Region functions only exist on ranks that own elements of the region, so if rank 0 owns + // none of a region, a mismatch in that region's functions is not reported. In practice + // which functions exist is decided by the options and is the same on every rank where a + // region exists. + // --------------------------------------------------------------------- + for (const std::string& name : runtime_only) { + MFEM_WARNING_0("The quadrature function '" + << name + << "' is not in the checkpoint; it keeps the value it was initialized " + "with"); + } + for (Group& entry : qfuncs->groups()) { + bool registered = false; + for (const auto& [name, function] : sim_state.m_map_qfs) { + if (function && SidreSafeName(name) == entry.getName()) { + registered = true; + break; + } + } + if (!registered) { + MFEM_WARNING_0("The checkpoint quadrature function '" + << entry.getName() << "' is not used by this run and is skipped"); + } + } +} + +/** + * @brief Recompute the nodal fields derived from the restored state + * + * @param sim_state Simulation state whose stored fields were just restored + * + * @details This repeats exactly what SimulationState::FinishCycle() did at the end of the step + * the checkpoint was written after, using the restored coordinates and velocity: + * - mesh_t_beg = mesh_current (beginning-of-step coordinates); + * - previous velocity = velocity (true DOFs); + * - velocity grid function = the velocity true DOFs distributed to local DOFs (collective); + * - displacement = mesh_current - mesh_ref. + * mesh_current is the mesh's Nodes (SimulationState swaps it in during construction), and its + * values just changed underneath the mesh, so the mesh is told to drop its cached geometric + * factors. If FinishCycle() ever derives another field, it must be added here as well. + */ +void RestartCheckpoint::ApplyPostRestartFixup(SimulationState& sim_state) const { + mfem::ParGridFunction& mesh_current = *sim_state.m_mesh_nodes.at("mesh_current"); + mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + (*sim_state.m_mesh_nodes.at("mesh_t_beg")) = mesh_current; + (*sim_state.m_primal_field_prev) = *sim_state.m_primal_field; + sim_state.m_mesh_qoi_nodes.at("velocity")->Distribute(*sim_state.m_primal_field); + mfem::ParGridFunction& displacement = *sim_state.m_mesh_qoi_nodes.at("displacement"); + displacement = mesh_current; + displacement -= mesh_ref; + sim_state.m_mesh->NodesUpdated(); +} + +} // namespace exaconstit + +#endif // EXACONSTIT_USE_AXOM diff --git a/src/sim_state/restart_checkpoint.hpp b/src/sim_state/restart_checkpoint.hpp new file mode 100644 index 00000000..c476cfed --- /dev/null +++ b/src/sim_state/restart_checkpoint.hpp @@ -0,0 +1,306 @@ +#pragma once + +#include "mfem_expt/partial_qfunc.hpp" +#include "options/option_parser_v2.hpp" +#include "sim_state/simulation_state.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace axom { +namespace sidre { +class DataStore; +class Group; +} // namespace sidre +} // namespace axom + +namespace exaconstit { + +/** + * @brief Version of the checkpoint rank-file layout written by RestartCheckpoint + * + * Stored in `restart_info/format_version` of every rank file and in `commit.json`. Bump it + * whenever the layout changes, so older checkpoints are rejected with a clear message instead + * of being misread. The mesh subtree has its own version (MESH_CHECKPOINT_FORMAT_VERSION). + */ +inline constexpr int RESTART_FORMAT_VERSION = 1; + +/** + * @brief Binary checkpoint writing and restarting for a SimulationState + * + * @details Writes the evolving simulation state to disk with Axom Sidre (one HDF5 file per + * rank) and restores it in a later run. A restart works from the checkpoint alone: the mesh, + * grain, region-mapping, orientation and state-variable files of the original run are never + * opened again. See doc/restart.md for the user-facing description and the design record. + * + * **Checkpoint layout on disk:** + * @code + * /cycle_/ + * rank_.hdf5 written by every rank + * commit.json written last, by rank 0 + * @endcode + * A checkpoint without `commit.json` is incomplete ("torn") and is never read. + * + * **Restarting is split into two phases**, because the state a restart has to restore is + * created in two places: + * - Phase 1 runs inside the SimulationState constructor, in place of reading the original + * input files: OpenForRestart(), ReconstructMesh(), RestoreReferenceConfiguration(), + * ReadElementGrainIds() and ReleaseMeshArrays(), in that order. + * - Phase 2, RestoreState(), runs from the driver after SystemDriver and + * PostProcessingDriver have been constructed, so every quadrature function any component + * registers already exists. It overwrites the evolving state with the checkpointed values, + * replacing whatever the constructors initialized. + * + * **Ownership.** SimulationState owns the RestartCheckpoint (std::unique_ptr) and makes it a + * friend so it can reach the field and quadrature-function registries. RestartCheckpoint + * never stores a pointer back to its owner; the SimulationState& arguments are borrowed for + * the duration of each call. Everything restored ends up owned by SimulationState, and + * nothing is owned by Sidre once RestoreState() returns. + * + * **Collective calls.** Every public member function except GetRestartDriverStep() is + * collective over the communicator given to the constructor and must be called on every + * rank, including ranks that own no elements. A problem found on any rank aborts every rank. + * + * **What is checkpointed.** + * - Quadrature data: every function registered in SimulationState's quadrature-function + * registry (SimulationState::AddQuadratureFunction()), whatever component registered it, + * except `tangent_stiffness`. Quadrature data kept outside the registry is not saved. + * - Nodal data: a fixed list, not a registry: the reference and current coordinates and the + * velocity true DOFs. The other SimulationState grid functions are derived from these. + * Any new grid function or solution vector that carries history, e.g. an additional + * primal variable, must be added to WriteNodalFields() and to the binding in + * BindFieldsAndQuadratureFunctions(), or it is silently not restored. + * - Mesh: the volume ParMesh only. Anything built on top of it (submeshes, surface meshes, + * other topologies) is not saved and must either be rebuilt deterministically from the + * restored mesh or have its own checkpoint data. + * - Time state, the element grain IDs and the material properties (the properties are only + * compared against the restarted run's; that run uses its own). + * + * **Restart requirements.** The restarted run must use the same number of MPI ranks, the + * same mesh order, and the same material list (names, model types and state-variable + * counts). These are checked and violations abort with a message saying what changed. + */ +class RestartCheckpoint { +public: + /** + * @brief Construct the checkpoint handler + * + * @param options [Restart] options (copied) + * @param comm Communicator of the simulation (SimulationState uses MPI_COMM_WORLD) + */ + RestartCheckpoint(const RestartOptions& options, MPI_Comm comm); + ~RestartCheckpoint(); + + RestartCheckpoint(const RestartCheckpoint&) = delete; + RestartCheckpoint& operator=(const RestartCheckpoint&) = delete; + + // ========================================================================= + // WRITING + // ========================================================================= + + /** + * @brief Write a checkpoint if one is due + * + * @param sim_state Simulation state to write + * @param driver_step The driver's loop counter `ti` for the step just completed + * + * @details Call once at the end of every driver-loop iteration, after FinishCycle(), + * UpdateModel() and post-processing. A checkpoint is due when `[Restart] write` is set, + * the time manager is not in the middle of a retrial or sub-step sequence, at least one + * cycle has completed since the last checkpoint (or the restart point), and either + * `interval` cycles have passed or the simulation just finished and `write_final` is set. + * + * A failed write is reported as a warning and the simulation continues; the incomplete + * checkpoint is never read. + */ + void WriteCheckpointIfDue(SimulationState& sim_state, std::size_t driver_step); + + // ========================================================================= + // RESTARTING, PHASE 1 (inside the SimulationState constructor) + // ========================================================================= + + /** + * @brief Select the checkpoint to restart from and load this rank's file + * + * @details Uses `[Restart] read_cycle` if set, otherwise the newest complete checkpoint + * in the read directory. Aborts if no usable checkpoint exists, or if it was written with + * a different format version or number of ranks. Must be called before any other phase 1 + * or phase 2 function. + */ + void OpenForRestart(); + + /** + * @brief Rebuild the parallel mesh from the loaded checkpoint + * + * @return The rebuilt mesh, identical to the one that was written (see ExaSidreParMesh) + * + * @details Replaces reading the mesh file. Aborts if the rebuilt topology does not match + * the counts recorded in the checkpoint. + */ + std::shared_ptr ReconstructMesh(); + + /** + * @brief Fill in the element grain IDs from the checkpoint + * + * @param grain_ids Array sized to the local element count; receives one grain ID per + * local element + * + * @details Replaces reading the grain IDs from the mesh file's element attributes, which + * hold region IDs in a checkpointed mesh. + */ + void ReadElementGrainIds(mfem::Array& grain_ids); + + /** + * @brief Overwrite the coordinate fields with the checkpointed reference configuration + * + * @param sim_state Simulation state under construction + * + * @details Call from the SimulationState constructor right after the coordinate fields + * exist and before anything else reads them. Sets `mesh_ref`, `mesh_t_beg` and + * `mesh_current` (the mesh's Nodes) to the checkpointed reference coordinates, which is + * exactly what they held in the original run until its first step, and checks that the + * mesh's finite element space numbers its DOFs as in the original run. Aborts on a + * mismatch. + */ + void RestoreReferenceConfiguration(SimulationState& sim_state); + + /** + * @brief Free the mesh arrays held by the loaded checkpoint + * + * @details Call at the end of the SimulationState constructor; the mesh and grain IDs + * have been copied out by then. The rest of the checkpoint stays loaded for phase 2. + */ + void ReleaseMeshArrays(); + + // ========================================================================= + // RESTARTING, PHASE 2 (from the driver) + // ========================================================================= + + /** + * @brief Overwrite the evolving state with the checkpointed values + * + * @param sim_state Fully constructed simulation state + * + * @details Call once, after SystemDriver and PostProcessingDriver are constructed and + * before the first time step. Checks the checkpoint against this run (materials and + * quadrature spaces), restores the nodal fields and every quadrature function present in + * both, resumes the time state, recomputes the fields derived from the restored state, + * and releases the checkpoint. Aborts on a mismatch before anything is overwritten. + */ + void RestoreState(SimulationState& sim_state); + + /** + * @brief Driver loop counter recorded in the checkpoint + * + * @return The `ti` of the last completed step; the first restarted step is this + 1 + */ + std::size_t GetRestartDriverStep() const { + return m_restart_driver_step; + } + +private: + // --------------------------------------------------------------------- + // Writing (each documented in restart_checkpoint.cpp) + // --------------------------------------------------------------------- + void WriteCheckpoint(SimulationState& sim_state, + std::size_t driver_step, + const TimeManagement::CheckpointState& time_state); + bool PrepareCheckpointDirectory(const std::filesystem::path& checkpoint_directory); + void WriteRestartInfo(axom::sidre::Group* root, + const SimulationState& sim_state, + std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) const; + void WriteMeshAndGrainIds(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteNodalFields(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteQuadratureSpaces(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteQuadratureFunctions(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteMaterials(axom::sidre::Group* root, const SimulationState& sim_state) const; + bool WriteCommitFile(const std::filesystem::path& checkpoint_directory, + std::size_t completed_cycles, + std::size_t driver_step, + double completed_time) const; + void SupersedeCheckpointsAfter(std::size_t completed_cycles) const; + void ApplyRetentionPolicy() const; + + // --------------------------------------------------------------------- + // Restarting (each documented in restart_checkpoint.cpp) + // --------------------------------------------------------------------- + std::string SelectCheckpointDirectory(); + void ValidateMeshSpace(axom::sidre::Group* root, const SimulationState& sim_state); + void VerifyRestoredDofLayout(const SimulationState& sim_state); + void ValidateMaterials(axom::sidre::Group* root, const SimulationState& sim_state); + void ValidateQuadratureSpaces(axom::sidre::Group* root, const SimulationState& sim_state); + void BindFieldsAndQuadratureFunctions(axom::sidre::Group* root, SimulationState& sim_state); + void ApplyPostRestartFixup(SimulationState& sim_state) const; + + // --------------------------------------------------------------------- + // Failure reporting + // --------------------------------------------------------------------- + /** + * @brief Record a problem found on this rank + * + * @param message Description of the problem + * + * @details Problems are collected rather than aborting immediately, so that every rank + * reaches AbortIfAnyRankFailed() and the abort happens on every rank together. + */ + void RecordFailure(const std::string& message); + + /** + * @brief Abort on every rank if any rank recorded a failure + * + * @param stage Short description of what was being done, used in the abort message + * + * @details Collective. Each failing rank reports its own recorded problems as warnings, + * then every rank aborts with one shared message. + */ + void AbortIfAnyRankFailed(const char* stage); + + /** @brief [Restart] options this handler was created with */ + RestartOptions m_options; + /** @brief Communicator of the simulation */ + MPI_Comm m_comm; + /** @brief This rank in m_comm */ + int m_rank = 0; + /** @brief Number of ranks in m_comm */ + int m_num_ranks = 1; + + /** @brief Completed cycles at the last committed checkpoint, or at the restart point */ + std::size_t m_last_written_cycles = 0; + /** @brief Whether the next write still has to supersede checkpoints of another timeline */ + bool m_supersede_pending = true; + + /** @brief Checkpoint loaded by OpenForRestart(); released at the end of RestoreState() */ + std::unique_ptr m_read_store; + /** @brief Path of this rank's file in the checkpoint being restarted from */ + std::filesystem::path m_read_rank_file; + /** @brief Driver loop counter recorded in the checkpoint being restarted from */ + std::size_t m_restart_driver_step = 0; + /** @brief Time state recorded in the checkpoint being restarted from */ + TimeManagement::CheckpointState m_restart_time_state; + /** @brief Element-local hash of mesh_ref recorded in the checkpoint */ + std::uint64_t m_stored_mesh_ref_element_hash = 0; + + /** + * @brief End-of-step quadrature functions to restore by copying their beginning-of-step + * partner once the bulk load is done (destination, source) + */ + std::vector, + std::shared_ptr>> + m_pending_copies; + + /** @brief Whether this rank recorded a failure since the last AbortIfAnyRankFailed() */ + bool m_local_failed = false; + /** @brief Problems recorded on this rank since the last AbortIfAnyRankFailed() */ + std::string m_local_failure_log; +}; + +} // namespace exaconstit diff --git a/src/sim_state/simulation_state.cpp b/src/sim_state/simulation_state.cpp index 41910065..199c9f19 100644 --- a/src/sim_state/simulation_state.cpp +++ b/src/sim_state/simulation_state.cpp @@ -1,6 +1,9 @@ #include "sim_state/simulation_state.hpp" #include "utilities/unified_logger.hpp" +#if defined(EXACONSTIT_USE_AXOM) +#include "sim_state/restart_checkpoint.hpp" +#endif namespace { @@ -403,11 +406,112 @@ bool TimeManagement::BCTime(const double desired_bc_time) { return false; } +/** + * @details Implementation notes. After a successful step, UpdateDeltaTime() leaves the time + * manager describing the *next* step: `old_time` is the end of the completed step, `dt` and + * `time` are the next step's size and end time, `simulation_cycle` counts completed steps and + * the tracker says whether the next step is a NORMAL or the FINAL one. This function rebuilds + * exactly that state from the checkpoint, using this run's time-step settings so a + * continuation can change them: + * + * 1. Completed time and cycle count come from the checkpoint. The retrial and sub-step + * counters are reset: checkpoints are only written when IsCheckpointSafe(), i.e. when no + * retrial or sub-step sequence is in flight, so they were zero (or irrelevant) then too. + * 2. prev_dt and dt_orig come from the checkpoint. Only log messages use them, and taking them + * from the checkpoint makes the restarted run print the same messages as the uninterrupted + * run. + * - prev_dt is the size of the last completed step. The driver prints it as "dt old was" at + * the start of the next cycle. If this run's settings change dt, the message shows the + * last step taken before the restart followed by the new dt. + * - dt_orig is reset at the first failure of a retrial sequence, before it is ever printed. + * It is restored only so both runs' time managers hold the same values. + * 3. The next dt is chosen the way the success branch of UpdateDeltaTime() would choose it: + * CUSTOM takes custom_dt[completed cycles] (or the stored dt if this run's list is shorter), + * FIXED takes dt_fixed, and AUTO takes the stored next dt, which the writing run had + * already computed from its last Newton iteration count, clamped to this run's + * [dt_min, dt_max]. With unchanged settings this reproduces the writing run's dt exactly. + * 4. The final-time logic of UpdateDeltaTime() is applied against this run's time_final (same + * 1e-3 * dt tolerance): the next step becomes FINAL if it reaches t_final, and its dt is + * shortened if it would overshoot. One case UpdateDeltaTime() never meets is added: a + * checkpoint written at t_final (by write_final) resumes as FINISHED, so the driver loop + * does not run at all unless t_final has been increased. + */ +void TimeManagement::ResumeFromCheckpoint(const CheckpointState& state) { + // Step 1: completed time and cycles; no retrial or sub-step sequence is in flight. + old_time = state.completed_time; + simulation_cycle = state.completed_cycles; + num_failures = 0; + num_sub_steps = 0; + required_num_sub_steps = 0; + + // Step 2: the time steps used only in log messages. + prev_dt = state.previous_dt; + dt_orig = state.original_dt; + + // Step 3: next dt from this run's scheme. + if (time_type == TimeStepType::CUSTOM) { + dt = (simulation_cycle < custom_dt.size()) ? custom_dt[simulation_cycle] : state.next_dt; + } else if (time_type == TimeStepType::FIXED) { + dt = dt_fixed; + } else { + dt = std::clamp(state.next_dt, dt_min, dt_max); + } + + // Step 4: final-time logic against this run's time_final. + const double tolerance = std::abs(1e-3 * dt); + if (time_final - old_time <= tolerance) { + internal_tracker = TimeStep::FINISHED; + time = old_time; + return; + } + const double tnew = old_time + dt; + if (std::abs(tnew - time_final) <= tolerance) { + internal_tracker = TimeStep::FINAL; + time = tnew; + } else if (tnew > time_final) { + internal_tracker = TimeStep::FINAL; + dt = time_final - old_time; + time = time_final; + } else { + internal_tracker = TimeStep::NORMAL; + time = tnew; + } +} + SimulationState::SimulationState(ExaOptions& options) : m_time_manager(options), m_options(options), class_device(options.solvers.rtmodel) { MPI_Comm_rank(MPI_COMM_WORLD, &my_id); m_time_manager = TimeManagement(options); + + // Restart runs ([Restart] read = true) build this object exactly as a fresh run does, + // except that the few things a fresh run reads from input files are taken from the + // checkpoint instead, since those files may no longer exist ("phase 1" of a restart; see + // the background notes in restart_checkpoint.cpp). Each restart-specific step below is + // marked. The evolving state itself is restored later, from the driver, by + // RestoreFromCheckpoint() ("phase 2"), once every component has registered its + // quadrature functions. + // + // Restart step: open the checkpoint and rebuild the mesh from it instead of reading the + // mesh file. Both are collective, so they run on every rank, including ranks that end up + // owning no elements. m_restart is also created for runs that only write checkpoints. + const bool restart_run = options.restart.read; +#if defined(EXACONSTIT_USE_AXOM) + if (options.restart.read || options.restart.write) { + m_restart = std::make_unique(options.restart, + MPI_COMM_WORLD); + } + if (restart_run) { + m_restart->OpenForRestart(); + m_mesh = m_restart->ReconstructMesh(); + } else { + m_mesh = ::makeMesh(options, my_id); + } +#else + // RestartOptions::validate() already rejects this; kept as a safeguard for callers that + // construct SimulationState without validating their options. + MFEM_VERIFY_0(!restart_run, "Restarting from a checkpoint requires ExaConstit built with Axom"); m_mesh = ::makeMesh(options, my_id); +#endif ::setupBoundaryConditions(options); // m_bc_manager = BCManager::GetInstance(); @@ -460,6 +564,20 @@ SimulationState::SimulationState(ExaOptions& options) m_primal_field_prev->UseDevice(true); (*m_primal_field) = 0.0; (*m_primal_field_prev) = 0.0; + +#if defined(EXACONSTIT_USE_AXOM) + // Restart step: overwrite the coordinate fields just created with the checkpointed + // reference coordinates, and verify the DOF layout, before anything reads them. The + // rebuilt mesh has no Nodes, so GetNodes() above projected from the vertex positions, + // which differs from the original run's coordinates in the last bits. Components built + // before RestoreFromCheckpoint() keep values derived from the starting geometry + // (SystemDriver's mono_def_bcs DOF search, the UMAT reference shape gradients), so they + // must see exactly the original coordinates for the restart to be bitwise exact. See + // RestartCheckpoint::RestoreReferenceConfiguration(). Collective. + if (restart_run) { + m_restart->RestoreReferenceConfiguration(*this); + } +#endif } { @@ -512,23 +630,67 @@ SimulationState::SimulationState(ExaOptions& options) region_map = false; mfem::Array grains(loc_nelems); + // Grain IDs. A fresh run reads them from the mesh's element attributes. In a + // checkpointed mesh the element attributes already hold region IDs (they are + // overwritten below on a fresh run), so the restart step reads the grain IDs the + // checkpoint stored separately. Collective on restart runs. + if (restart_run) { +#if defined(EXACONSTIT_USE_AXOM) + m_restart->ReadElementGrainIds(grains); +#endif + } else { + for (int i = 0; i < loc_nelems; i++) { + grains.operator[](i) = m_mesh->GetAttribute(i); + } + } + { auto pfes = GetParFiniteElementSpace(1); m_grains = std::make_shared(pfes.get()); m_grains->HostWrite(); for (int i = 0; i < loc_nelems; i++) { - grains.operator[](i) = m_mesh->GetAttribute(i); - m_grains->operator[](i) = m_mesh->GetAttribute(i); + m_grains->operator[](i) = grains.operator[](i); } } - const auto grains2region = ::create_grains_to_map(options, grains); + std::map grains2region; + if (!restart_run) { + grains2region = ::create_grains_to_map(options, grains); - for (int i = 0; i < loc_nelems; i++) { - const int grain_id = grains.operator[](i); - const int region_id = grains2region.at(grain_id); - m_mesh->SetAttribute(i, region_id); - region_map(region_id - 1, i) = true; + for (int i = 0; i < loc_nelems; i++) { + const int grain_id = grains.operator[](i); + const int region_id = grains2region.at(grain_id); + m_mesh->SetAttribute(i, region_id); + region_map(region_id - 1, i) = true; + } + } else { + // Restart step: a fresh run maps grains to regions with the region-mapping file and + // then stores the region IDs in the element attributes. The checkpointed mesh + // already carries those 1-based region IDs, so the region map is built from them + // directly and the region-mapping file is never read. + // + // A region ID outside [1, number of materials] means the material list shrank + // since the checkpoint was written. That check depends on each rank's own + // elements, so the ranks agree on it with an MPI_Allreduce and all abort together; + // a rank with no elements contributes "no problem". + const int num_regions = static_cast(options.materials.size()); + int local_out_of_range = 0; + for (int i = 0; i < loc_nelems; i++) { + const int region_id = m_mesh->GetAttribute(i); + if (region_id < 1 || region_id > num_regions) { + local_out_of_range = 1; + continue; + } + region_map(region_id - 1, i) = true; + } + int any_out_of_range = 0; + MPI_Allreduce( + &local_out_of_range, &any_out_of_range, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + MFEM_VERIFY_0(any_out_of_range == 0, + "The checkpoint has elements in regions beyond the " + << num_regions + << " material(s) configured for this run; the material list " + "cannot change on a restart"); } // update all of our attributes @@ -626,8 +788,20 @@ SimulationState::SimulationState(ExaOptions& options) } CreateRegionCommunicators(); - InitializeStateVariables(grains2region); + // Restart step: skip the state-variable and orientation initialization. Its input + // files may no longer exist, and RestoreFromCheckpoint() overwrites all state + // variables (orientations included) with the checkpointed values anyway. + if (!restart_run) { + InitializeStateVariables(grains2region); + } } +#if defined(EXACONSTIT_USE_AXOM) + // Restart step: the mesh and grain IDs have been copied out of the checkpoint, so free + // those arrays now; the rest of the checkpoint stays loaded for RestoreFromCheckpoint(). + if (restart_run) { + m_restart->ReleaseMeshArrays(); + } +#endif } SimulationState::~SimulationState() { @@ -638,6 +812,34 @@ SimulationState::~SimulationState() { } } +void SimulationState::RestoreFromCheckpoint() { +#if defined(EXACONSTIT_USE_AXOM) + MFEM_VERIFY_0(m_restart && m_options.restart.read, + "RestoreFromCheckpoint() was called on a run that is not restarting"); + m_restart->RestoreState(*this); +#else + MFEM_ABORT_0("Restarting from a checkpoint requires ExaConstit built with Axom"); +#endif +} + +void SimulationState::WriteCheckpointIfDue(const size_t driver_step) { +#if defined(EXACONSTIT_USE_AXOM) + if (m_restart) { + m_restart->WriteCheckpointIfDue(*this, driver_step); + } +#else + static_cast(driver_step); +#endif +} + +size_t SimulationState::GetRestartDriverStep() const { +#if defined(EXACONSTIT_USE_AXOM) + return m_restart ? m_restart->GetRestartDriverStep() : 0; +#else + return 0; +#endif +} + bool SimulationState::AddQuadratureFunction(const std::string_view& qf_name, const int vdim, const int region) { @@ -647,7 +849,7 @@ bool SimulationState::AddQuadratureFunction(const std::string_view& qf_name, } // find(), not operator[]: a region with no elements on this rank has no quadrature space // here, and operator[] would insert a null entry into m_map_qs, which every loop over the - // spaces would then dereference. + // spaces (including the checkpoint writer) would then dereference. const auto space = m_map_qs.find(GetRegionName(region)); if (space == m_map_qs.end() || !space->second) { return false; diff --git a/src/sim_state/simulation_state.hpp b/src/sim_state/simulation_state.hpp index 6d9eb905..dbbcf066 100644 --- a/src/sim_state/simulation_state.hpp +++ b/src/sim_state/simulation_state.hpp @@ -14,6 +14,10 @@ #include #include +namespace exaconstit { +class RestartCheckpoint; +} // namespace exaconstit + /** * @brief Enumeration for time step status and control * @@ -253,20 +257,76 @@ class TimeManagement { } /** - * @brief Restart simulation from a specific time and cycle - * - * @param time_restart Time to restart from - * @param dt_restart Time step size to use - * @param cycle Cycle number to restart from + * @brief Time state needed to resume a run exactly after a completed cycle. + * + * @details Stored as unambiguous quantities because the raw members change meaning + * with the tracker: after a successful UpdateDeltaTime() that returns NORMAL/FINAL, + * `time` is the END time of the NEXT step and `simulation_cycle` counts completed + * steps; after FINAL->FINISHED, `time` is the end of the step just completed and + * `simulation_cycle` was NOT incremented. `old_time` is the end time of the step just + * completed in both cases. + * + * `previous_dt` and `original_dt` are carried along so that a restarted run's time-step + * logging (PrintTimeStats(), PrintRetrialStats(), PrintSubStepStats()) behaves exactly as + * the uninterrupted run's would. + */ + struct CheckpointState { + /** @brief End time of the last completed step (old_time) */ + double completed_time = 0.0; + /** @brief Time step the writing run would have taken next */ + double next_dt = 0.0; + /** @brief Time step of the last completed step (prev_dt) */ + double previous_dt = 0.0; + /** @brief Time step saved at the start of the last retrial sequence (dt_orig) */ + double original_dt = 0.0; + /** @brief Number of completed cycles (the index into custom_dt of the next step) */ + size_t completed_cycles = 0; + }; + + /** + * @brief Snapshot the resumable time state (see CheckpointState). + * @pre IsCheckpointSafe() + */ + CheckpointState GetCheckpointState() const { + CheckpointState state; + state.completed_time = old_time; + state.next_dt = dt; + state.previous_dt = prev_dt; + state.original_dt = dt_orig; + state.completed_cycles = simulation_cycle + + ((internal_tracker == TimeStep::FINISHED) ? 1 : 0); + return state; + } + + /** + * @brief True when no retrial or sub-step sequence is in flight, i.e. the retrial / + * sub-step counters carry no information a checkpoint would lose. + */ + bool IsCheckpointSafe() const { + return internal_tracker == TimeStep::NORMAL || internal_tracker == TimeStep::FINAL || + internal_tracker == TimeStep::FINISHED; + } + + /** + * @brief Resume from a CheckpointState using THIS run's time-step configuration. * - * @details Used for simulation restarts from checkpoint data. - * Sets all time-related state to the specified restart values. - * Does not modify time step type or other configuration parameters. + * @details The next dt comes from the current scheme (CUSTOM: custom_dt[completed + * cycles] when available; FIXED: dt_fixed; AUTO: the stored dt clamped to + * [dt_min, dt_max]), then the same final-time logic as UpdateDeltaTime() is applied + * against the current time_final, so extending t_final (or the custom dt list) on a + * restart continues the run and a checkpoint already at t_final reports FINISHED. + * The logging-only previous and original time steps are restored as stored. */ - void RestartTimeState(const double time_restart, const double dt_restart, const size_t cycle) { - simulation_cycle = cycle; - time = time_restart; - dt = dt_restart; + void ResumeFromCheckpoint(const CheckpointState& state); + + /** @brief Active time stepping mode */ + TimeStepType GetTimeStepType() const { + return time_type; + } + + /** @brief Configured final simulation time */ + double GetFinalTime() const { + return time_final; } /** @@ -459,6 +519,17 @@ class SimulationState { /** @brief Reference to simulation options */ ExaOptions& m_options; + /** + * @brief Binary Sidre checkpoint/restart driver; null unless [Restart] read or write. + * + * @details Owned here. It borrows `*this` for the duration of each call and never + * stores a pointer back to its owner (a stored shared_ptr would form a cycle, and + * shared_from_this() is unavailable inside the constructor where the mesh is rebuilt). + * Friend access lets it enumerate the QSpace/QF registries and resume TimeManagement + * without widening the public API. See doc/restart.md. + */ + std::unique_ptr m_restart; + friend class exaconstit::RestartCheckpoint; /** @brief MPI rank identifier */ int my_id; @@ -603,6 +674,27 @@ class SimulationState { * - Stress/strain tensors: vdim = 6 (Voigt notation) * - Deformation gradients: vdim = 9 (3x3 matrix) * - State variables: vdim = model-dependent + * + * **Checkpoint/restart contract.** Every registered quadrature function is written to + * restart checkpoints and restored on restart automatically, with no restart code + * needed for it (the one exception is `tangent_stiffness`, which is recomputed before + * use). Quadrature point data held anywhere else, such as a model's own mfem::Vector or + * PartialQuadratureFunction, is NOT checkpointed, and nothing warns about it. So: + * - Any per-point quantity that carries history from one step to the next must be + * registered here. Scratch data that is fully recomputed every step does not need to + * be. + * - Register it while the simulation objects are being constructed (SimulationState, + * SystemDriver, the models, PostProcessingDriver), i.e. before the driver restores a + * checkpoint. A function registered later keeps its initial value on restart. + * - The writing and restarted runs must register the same functions with the same vector + * dimension. A function only one of them registers is reported as a warning and starts + * from its initial value. + * - For a beginning/end-of-step pair, also register the pair with + * AddUpdateVariablePairNames(). This is optional for restart (an unregistered pair is + * simply stored in full), but lets the checkpoint store an identical end-of-step + * function as a reference to its partner. + * - Call this on every rank. Ranks that own none of the region's elements get false back + * and have no function of that name, which the checkpoint handles. */ bool AddQuadratureFunction(const std::string_view& qf_name, const int vdim = 1, @@ -1149,6 +1241,55 @@ class SimulationState { m_time_manager.PrintRetrialStats(); } + // ========================================================================= + // CHECKPOINT / RESTART + // ========================================================================= + + /** + * @brief Check whether this run restarts from a checkpoint + * + * @return True if [Restart] read is set + * + * @details On a restart run the constructor rebuilt the mesh and took the grain and + * region IDs from the checkpoint, and skipped the orientation and state-variable + * initialization. The driver must call RestoreFromCheckpoint() before the first time step. + */ + bool IsRestartRun() const { + return m_options.restart.read; + } + + /** + * @brief Overwrite the evolving state with the checkpointed values (restart phase 2) + * + * @details Call exactly once on a restart run, after SystemDriver and PostProcessingDriver + * have been constructed (so every quadrature function they register exists) and before the + * first time step. Restores the nodal fields and every quadrature function present both in + * this run and in the checkpoint, resumes the time state and recomputes the derived fields. + * Aborts if the checkpoint does not match this run. Collective over MPI_COMM_WORLD. + * See exaconstit::RestartCheckpoint::RestoreState(). + */ + void RestoreFromCheckpoint(); + + /** + * @brief Write a checkpoint if one is due + * + * @param driver_step The driver's loop counter `ti` for the step just completed + * + * @details Call at the end of every driver-loop iteration, after FinishCycle(), the model + * update and post-processing. Does nothing unless [Restart] write is set; when it is, the + * [Restart] interval and write_final settings decide whether this step is written. + * Collective over MPI_COMM_WORLD, so every rank must call it every iteration. + */ + void WriteCheckpointIfDue(size_t driver_step); + + /** + * @brief Driver loop counter recorded in the checkpoint + * + * @return The `ti` of the last step completed before the checkpoint, or 0 if this run is + * not a restart; the first restarted iteration is this value + 1 + */ + size_t GetRestartDriverStep() const; + private: /** @brief Create MPI communicators for each region containing only ranks with that region * @details This prevents deadlocks in collective operations when some ranks have no diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 63a29eb4..5f59f57a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,6 +57,12 @@ if(ENABLE_AXOM) OUTPUT_DIR ${TEST_OUTPUT_DIR} DEPENDS_ON ${EXACONSTIT_TEST_DEPENDS} gtest) + # SimulationState checkpoint, delete the input files, restart, compare bitwise + blt_add_executable(NAME test_restart_e2e + SOURCES test_restart_e2e.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${EXACONSTIT_TEST_DEPENDS} gtest) + # MPI tests never use more ranks than the machine has cores, so the suite does not # oversubscribe small machines such as the 4-core CI runners (Open MPI refuses to start more # ranks than slots by default). The default is the number of logical cores CMake detects; @@ -75,6 +81,26 @@ if(ENABLE_AXOM) NUM_MPI_TASKS ${_np}) endif() endforeach() + + foreach(_np 2 3) + if(NOT _np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + blt_add_test(NAME test_restart_e2e_np${_np} + COMMAND test_restart_e2e + NUM_MPI_TASKS ${_np}) + endif() + endforeach() + + # Diagnostic: lists how each quadrature function is stored in a checkpoint rank file + blt_add_executable(NAME inspect_restart_rank_file + SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/inspect_restart_rank_file.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON axom) + + # Compares two checkpoint directories view by view (bitwise by default) + blt_add_executable(NAME compare_restart_checkpoints + SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/compare_restart_checkpoints.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON axom) endif() if (NOT DEFINED PYTHON_EXECUTABLE) diff --git a/test/test_restart_e2e.cpp b/test/test_restart_e2e.cpp new file mode 100644 index 00000000..ece9a7ec --- /dev/null +++ b/test/test_restart_e2e.cpp @@ -0,0 +1,519 @@ +#include "sim_state/simulation_state.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// End-to-end restart test at the SimulationState level (no SystemDriver), for the checkpoint +// and restart code in src/sim_state/restart_checkpoint.* and the restart branches of the +// SimulationState constructor. +// +// Run A builds a real SimulationState from input files: a perturbed order-3 hex mesh whose +// element attributes are grain IDs, two UMAT regions assigned through a region-mapping file, +// and orientations for the first region. It is then given a synthetic evolved state (deformed +// coordinates, a velocity, distinct values in every quadrature function, three completed time +// steps) and writes a checkpoint. The mesh, region-mapping and orientation files are then +// deleted, and run B restarts from the checkpoint alone. The test checks that: +// - phase 1 reproduces the original reference coordinates bit for bit inside the constructor +// (the vertex-geometry projection a rebuilt mesh would otherwise use differs in the last +// bits; the summary line reports by how much); +// - after phase 2 the mesh attributes, grain IDs, region flags, every nodal field, every +// checkpointed quadrature function and the full time state are bitwise identical, and the +// time-step line the driver logs reads the same as in run A; +// - the tangent stiffness is not restored (it keeps its constructor value until the first +// model setup recomputes it); +// - the restarted run can write its next checkpoint. +// One begin/end state-variable pair is deliberately made unequal, so both the `copy_of` and the +// stored-values paths of the quadrature-function checkpoint are exercised. +// +// The first region also covers orientation placement: its grain table has no +// ori_state_var_loc, so the quaternions go at the end of each state vector. +// +// Run with 2 or 3 ranks, for example `mpirun -np 2 ./test_restart_e2e`. Scratch files go to +// restart_e2e_np/ in the working directory. + +namespace fs = std::filesystem; + +namespace { + +/** + * @brief Deterministic unit quaternion for a grain + * + * @param grain Grain ID + * @return Normalized quaternion (w, x, y, z), different for every grain + */ +std::array GrainQuaternion(const int grain) { + std::array quaternion = {1.0, 0.1 * grain, 0.2 * grain, 0.05 * grain}; + const double norm = std::sqrt(quaternion[0] * quaternion[0] + quaternion[1] * quaternion[1] + + quaternion[2] * quaternion[2] + quaternion[3] * quaternion[3]); + for (double& component : quaternion) { + component /= norm; + } + return quaternion; +} + +/** + * @brief Check two vectors for bitwise equality + * + * @param a First vector + * @param b Second vector + * @return True if both have the same size and every entry compares equal + */ +bool BitwiseEqual(const mfem::Vector& a, const mfem::Vector& b) { + if (a.Size() != b.Size()) { + return false; + } + if (a.Size() == 0) { + return true; + } + const double* x = a.HostRead(); + const double* y = b.HostRead(); + for (int i = 0; i < a.Size(); ++i) { + if (x[i] != y[i]) { + return false; + } + } + return true; +} + +/** + * @brief Capture the time-step line the driver logs at the start of a cycle + * + * @param sim_state Simulation state whose time manager is printed + * @return The output of SimulationState::PrintTimeStats(), printed with 17 significant digits + * so that any difference in the time steps shows up in the text + */ +std::string CaptureTimeStats(const SimulationState& sim_state) { + std::ostringstream captured; + std::streambuf* original = std::cout.rdbuf(captured.rdbuf()); + const std::streamsize original_precision = std::cout.precision(17); + sim_state.PrintTimeStats(); + std::cout.precision(original_precision); + std::cout.rdbuf(original); + return captured.str(); +} + +/** + * @brief Options for the test runs + * + * @param directory Scratch directory holding the input files and the checkpoints + * @param read Whether this run restarts from a checkpoint + * @return Options with two UMAT regions, AUTO time stepping and checkpoint writing enabled + * + * @details The first region ("soft") has 3 state variables plus quaternion orientations with + * no ori_state_var_loc, so the orientations go at the end of its state vector (7 values per + * point). The second region ("hard") has 5 state variables and no orientations. + */ +ExaOptions MakeOptions(const fs::path& directory, const bool read) { + ExaOptions options; + options.basename = "e2e"; + options.mesh.mesh_type = MeshType::FILE; + options.mesh.mesh_file = directory / "mesh.mesh"; + options.mesh.order = 3; + // AUTO stepping with dt_max well above the steps taken, so the Newton iteration counts + // FillEvolvedState() reports make every step a different size and the logged "dt old was" + // value differs from the next dt. + options.time.time_type = TimeStepType::AUTO; + options.time.auto_time = TimeOptions::AutoTimeOptions{}; + options.time.auto_time->dt_start = 0.1; + options.time.auto_time->dt_min = 1e-3; + options.time.auto_time->dt_max = 10.0; + options.time.auto_time->t_final = 100.0; + options.solvers.rtmodel = RTModel::CPU; + options.region_mapping_file = directory / "regions.txt"; + for (int region = 0; region < 2; ++region) { + MaterialOptions material; + material.material_name = (region == 0) ? "soft" : "hard"; + material.region_id = region + 1; + material.mech_type = MechType::UMAT; + material.properties.num_props = 2; + material.properties.properties = {10.0 + region, 20.0 + region}; + material.state_vars.num_vars = 3 + 2 * region; + for (int k = 0; k < material.state_vars.num_vars; ++k) { + material.state_vars.initial_values.push_back(1.0 + k + 10 * region); + } + if (region == 0) { + GrainInfo grains; + grains.orientation_file = directory / "orientations.txt"; + grains.ori_type = OriType::QUAT; + grains.ori_stride = 4; + grains.num_grains = 8; + grains.ori_state_var_loc = -1; + material.grain_info = grains; + } + options.materials.push_back(material); + } + options.restart.write = true; + options.restart.interval = 1; + options.restart.keep_last = 2; + options.restart.read = read; + options.restart.directory = directory / "restart"; + return options; +} + +/** + * @brief A quadrature function registry entry: base name and region (-1 for global) + */ +struct QuadratureFunctionName { + /** @brief Base name, as passed to SimulationState::GetQuadratureFunction() */ + std::string base; + /** @brief Region index, or -1 for a global function */ + int region; +}; + +/** + * @brief The quadrature functions the test fills and compares + * + * @return Global and per-region names; functions not created on a rank are skipped there + */ +std::vector QuadratureFunctionNames() { + std::vector names = {{"cauchy_stress_beg", -1}, + {"cauchy_stress_end", -1}, + {"cauchy_stress_avg", -1}, + {"kinetic_grads", -1}, + {"tangent_stiffness", -1}}; + for (int region = 0; region < 2; ++region) { + for (const char* base : {"state_var_beg", + "state_var_end", + "state_var_avg", + "cauchy_stress_beg", + "cauchy_stress_end", + "cauchy_stress_avg", + "tangent_stiffness", + "def_grad_beg"}) { + names.push_back({base, region}); + } + } + return names; +} + +/** + * @brief Give a freshly constructed SimulationState a synthetic evolved state + * + * @param sim_state Simulation state to modify + * + * @details Mirrors what the driver leaves behind after three converged steps, in the same + * order (solve, FinishCycle(), model update, post-processing, checkpoint): + * - a velocity with distinct values per true DOF, and current coordinates displaced by it + * (built from true DOFs, so the field is continuous across ranks); + * - distinct deterministic values in every quadrature function, different per name and rank; + * - three completed AUTO time steps with different Newton iteration counts, so each step has + * a different size; + * - SetupModelVariables(), which makes every end-of-step function equal to its + * beginning-of-step partner, as SystemDriver::UpdateModel() does. + * Finally one state-variable pair is made unequal on purpose, so its end-of-step values are + * written in full rather than as `copy_of`. + */ +void FillEvolvedState(SimulationState& sim_state) { + auto space = sim_state.GetMeshParFiniteElementSpace(); + mfem::Vector& primal = *sim_state.GetPrimalField(); + const HYPRE_BigInt offset = space->GetMyTDofOffset(); + for (int i = 0; i < primal.Size(); ++i) { + primal[i] = std::sin(0.37 * static_cast(offset + i)); + } + mfem::Vector reference_true; + sim_state.GetRefCoords()->GetTrueDofs(reference_true); + mfem::Vector current_true = reference_true; + current_true.Add(0.01, primal); + sim_state.GetCurrentCoords()->SetFromTrueDofs(current_true); + + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + int salt = 1; + for (const auto& name : QuadratureFunctionNames()) { + auto function = sim_state.GetQuadratureFunction(name.base, name.region); + ++salt; + if (!function || function->Size() == 0) { + continue; + } + double* data = function->HostReadWrite(); + for (int i = 0; i < function->Size(); ++i) { + data[i] = std::cos(0.013 * i + 0.7 * salt + 0.11 * rank); + } + } + + for (const int newton_iterations : {3, 7, 12}) { + sim_state.UpdateDeltaTime(newton_iterations, true); + } + sim_state.FinishCycle(); + sim_state.SetupModelVariables(); + + auto end_state = sim_state.GetQuadratureFunction("state_var_end", 0); + if (end_state && end_state->Size() > 0) { + end_state->HostReadWrite()[0] += 1.0; + } +} + +/** + * @brief Write the input files of run A (rank 0 only) + * + * @param directory Scratch directory + * + * @details A perturbed 4x4x4 hex mesh whose element attributes are grain IDs 1..8, an + * orientation file with one quaternion per grain, and a region-mapping file sending grains + * 1-4 to region 1 and grains 5-8 to region 2. The mesh is written with 17 significant digits so + * the vertex positions survive the text round trip exactly. + */ +void WriteInputFiles(const fs::path& directory) { + fs::remove_all(directory); + fs::create_directories(directory); + + mfem::Mesh mesh = mfem::Mesh::MakeCartesian3D( + 4, 4, 4, mfem::Element::HEXAHEDRON, 1.0, 1.0, 1.0, false); + for (int vertex = 0; vertex < mesh.GetNV(); ++vertex) { + double* x = mesh.GetVertex(vertex); + const double x0 = x[0]; + const double x1 = x[1]; + const double x2 = x[2]; + x[0] += 0.01 * std::sin(3.1 * x1 + 1.7 * x2 + 0.3); + x[1] += 0.01 * std::sin(2.3 * x2 + 0.9 * x0 + 1.1); + x[2] += 0.01 * std::sin(1.9 * x0 + 2.9 * x1 + 0.7); + } + for (int element = 0; element < mesh.GetNE(); ++element) { + mesh.SetAttribute(element, 1 + (element * 5) % 8); + } + mesh.SetAttributes(); + std::ofstream mesh_file(directory / "mesh.mesh"); + mesh_file.precision(17); + mesh.Print(mesh_file); + + std::ofstream orientations(directory / "orientations.txt"); + orientations.precision(17); + for (int grain = 1; grain <= 8; ++grain) { + const auto quaternion = GrainQuaternion(grain); + orientations << quaternion[0] << " " << quaternion[1] << " " << quaternion[2] << " " + << quaternion[3] << "\n"; + } + + std::ofstream regions(directory / "regions.txt"); + for (int grain = 1; grain <= 8; ++grain) { + regions << grain << " " << (grain <= 4 ? 1 : 2) << "\n"; + } +} + +/** + * @brief Count the quadrature points of region 0 whose state does not match the inputs + * + * @param sim_state Freshly constructed simulation state of run A + * @return Number of points whose state vector is not {1, 2, 3, quaternion of the grain} + */ +int CountMisplacedOrientations(SimulationState& sim_state) { + auto state = sim_state.GetQuadratureFunction("state_var_beg", 0); + if (!state || state->GetVDim() != 7) { + return state ? 1 : 0; + } + auto space = state->GetPartialSpaceShared(); + const auto& local_to_global = space->GetLocal2Global(); + const int num_elements = space->GetNumLocalElements(); + if (num_elements == 0) { + return 0; + } + const int points_per_element = state->Size() / 7 / num_elements; + const double* data = state->HostRead(); + const double* grains = sim_state.GetGrains()->HostRead(); + int wrong = 0; + for (int element = 0; element < num_elements; ++element) { + const auto quaternion = GrainQuaternion(static_cast(grains[local_to_global[element]])); + for (int point = 0; point < points_per_element; ++point) { + const double* s = data + + static_cast(element * points_per_element + point) * 7; + const bool ok = s[0] == 1.0 && s[1] == 2.0 && s[2] == 3.0 && s[3] == quaternion[0] && + s[4] == quaternion[1] && s[5] == quaternion[2] && s[6] == quaternion[3]; + wrong += ok ? 0 : 1; + } + } + return wrong; +} + +/** + * @brief Compare the original reference coordinates with a vertex-geometry projection + * + * @param directory Scratch directory holding the mesh file + * @param reference Reference coordinates of run A + * @return Number of differing entries on this rank and the largest difference + * + * @details A mesh rebuilt from a checkpoint has no Nodes, so GetNodes() projects from the + * vertex positions. This measures how far that is from the original reference coordinates, + * which is the gap the phase-1 coordinate overlay closes. + */ +std::pair VertexProjectionDifference(const fs::path& directory, + const mfem::Vector& reference) { + mfem::Mesh serial((directory / "mesh.mesh").string().c_str(), 1, 1, true); + mfem::ParMesh vertex_mesh(MPI_COMM_WORLD, serial); + mfem::H1_FECollection fec(3, 3); + mfem::ParFiniteElementSpace space(&vertex_mesh, &fec, 3); + mfem::ParGridFunction projected(&space); + vertex_mesh.GetNodes(projected); + int mismatches = 0; + double max_difference = 0.0; + if (projected.Size() == reference.Size()) { + for (int i = 0; i < projected.Size(); ++i) { + const double difference = std::abs(projected(i) - reference(i)); + mismatches += difference != 0.0 ? 1 : 0; + max_difference = std::max(max_difference, difference); + } + } + return {mismatches, max_difference}; +} + +} // namespace + +TEST(restart_simulation_state, checkpoint_and_restart) { + int rank = 0; + int num_ranks = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + const fs::path directory = fs::current_path() / ("restart_e2e_np" + std::to_string(num_ranks)); + if (rank == 0) { + WriteInputFiles(directory); + } + MPI_Barrier(MPI_COMM_WORLD); + + // --------------------------------------------------------------------- + // Run A: build from the input files, evolve, checkpoint + // --------------------------------------------------------------------- + ExaOptions options_a = MakeOptions(directory, false); + SimulationState run_a(options_a); + EXPECT_EQ(CountMisplacedOrientations(run_a), 0) + << "region 0 state vectors should be {1, 2, 3} followed by the grain's quaternion"; + const auto [projection_mismatches, projection_max_difference] = VertexProjectionDifference( + directory, *run_a.GetRefCoords()); + + FillEvolvedState(run_a); + run_a.WriteCheckpointIfDue(3); + EXPECT_TRUE(fs::exists(directory / "restart" / "cycle_0000000003" / "commit.json")) + << "run A should have committed a checkpoint for cycle 3"; + + // The restart must not need the original input files. + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0) { + fs::remove(directory / "mesh.mesh"); + fs::remove(directory / "regions.txt"); + fs::remove(directory / "orientations.txt"); + } + MPI_Barrier(MPI_COMM_WORLD); + + // --------------------------------------------------------------------- + // Run B: phase 1 (constructor) must reproduce the original construction-time geometry + // --------------------------------------------------------------------- + ExaOptions options_b = MakeOptions(directory, true); + SimulationState run_b(options_b); + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetRefCoords())) << "phase 1: mesh_ref"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetCurrentCoords())) + << "phase 1: mesh_current should equal the reference coordinates"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetTimeStartCoords())) + << "phase 1: mesh_t_beg should equal the reference coordinates"; + + // --------------------------------------------------------------------- + // Run B: phase 2 must restore everything bitwise + // --------------------------------------------------------------------- + run_b.RestoreFromCheckpoint(); + + EXPECT_EQ(run_a.GetMesh()->GetNE(), run_b.GetMesh()->GetNE()) << "local element count"; + for (int element = 0; element < std::min(run_a.GetMesh()->GetNE(), run_b.GetMesh()->GetNE()); + ++element) { + if (run_a.GetMesh()->GetAttribute(element) != run_b.GetMesh()->GetAttribute(element)) { + ADD_FAILURE() << "region attribute of element " << element; + break; + } + } + EXPECT_TRUE(BitwiseEqual(*run_a.GetGrains(), *run_b.GetGrains())) << "grain IDs"; + for (int region = 0; region < 2; ++region) { + EXPECT_EQ(run_a.IsRegionActive(region), run_b.IsRegionActive(region)) + << "region " << region << " active flag"; + } + + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetRefCoords())) << "mesh_ref"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetCurrentCoords(), *run_b.GetCurrentCoords())) + << "mesh_current"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetTimeStartCoords(), *run_b.GetTimeStartCoords())) + << "mesh_t_beg"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetPrimalField(), *run_b.GetPrimalField())) << "velocity"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetPrimalFieldPrev(), *run_b.GetPrimalFieldPrev())) + << "previous velocity"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetVelocity(), *run_b.GetVelocity())) + << "velocity grid function"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetDisplacement(), *run_b.GetDisplacement())) << "displacement"; + + int compared = 0; + for (const auto& name : QuadratureFunctionNames()) { + auto function_a = run_a.GetQuadratureFunction(name.base, name.region); + auto function_b = run_b.GetQuadratureFunction(name.base, name.region); + const std::string label = name.base + " (region " + std::to_string(name.region) + ")"; + if (!function_a || !function_b) { + EXPECT_TRUE(!function_a && !function_b) << "present in only one run: " << label; + continue; + } + if (name.base == "tangent_stiffness") { + // Not checkpointed: it keeps the constructor's zeros until the first model setup. + bool zero = true; + if (function_b->Size() > 0) { + const double* data = function_b->HostRead(); + for (int i = 0; i < function_b->Size(); ++i) { + zero = zero && data[i] == 0.0; + } + } + EXPECT_TRUE(zero) << "the tangent should not be restored: " << label; + } else { + EXPECT_TRUE(BitwiseEqual(*function_a, *function_b)) << "restored values: " << label; + ++compared; + } + } + + EXPECT_EQ(run_a.GetTime(), run_b.GetTime()) << "time"; + EXPECT_EQ(run_a.GetDeltaTime(), run_b.GetDeltaTime()) << "dt"; + EXPECT_EQ(run_a.GetSimulationCycle(), run_b.GetSimulationCycle()) << "cycle"; + EXPECT_EQ(run_a.GetTrueCycleTime(), run_b.GetTrueCycleTime()) << "completed time"; + EXPECT_EQ(run_a.IsLastStep(), run_b.IsLastStep()) << "time step tracker (last step)"; + EXPECT_EQ(run_a.IsFinished(), run_b.IsFinished()) << "time step tracker (finished)"; + EXPECT_EQ(run_b.GetRestartDriverStep(), 3u) << "driver step"; + // The line the driver logs at the start of the next cycle, including the logging-only + // previous time step, must read exactly as in the uninterrupted run. + EXPECT_EQ(CaptureTimeStats(run_a), CaptureTimeStats(run_b)) << "logged time-step change"; + + // --------------------------------------------------------------------- + // Both runs take one more step; run B writes its next checkpoint + // --------------------------------------------------------------------- + run_a.UpdateDeltaTime(5, true); + run_a.FinishCycle(); + run_b.UpdateDeltaTime(5, true); + run_b.FinishCycle(); + EXPECT_EQ(run_a.GetTime(), run_b.GetTime()) << "time after one more step"; + EXPECT_EQ(run_a.GetDeltaTime(), run_b.GetDeltaTime()) << "dt after one more step"; + EXPECT_EQ(CaptureTimeStats(run_a), CaptureTimeStats(run_b)) + << "logged time-step change after one more step"; + run_b.WriteCheckpointIfDue(4); + EXPECT_TRUE(fs::exists(directory / "restart" / "cycle_0000000004" / "commit.json")) + << "the restarted run should have committed a checkpoint for cycle 4"; + + int global_mismatches = 0; + double global_max_difference = 0.0; + MPI_Allreduce(&projection_mismatches, &global_mismatches, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce( + &projection_max_difference, &global_max_difference, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (rank == 0) { + std::cout << "Restart end-to-end: " << num_ranks << " ranks, " << compared + << " quadrature functions compared on rank 0; a vertex-geometry projection " + "would have differed from the original reference coordinates in " + << global_mismatches << " entries (max " << global_max_difference << ")" + << std::endl; + } +} + +int main(int argc, char* argv[]) { + MPI_Init(&argc, &argv); + mfem::Hypre::Init(); + ::testing::InitGoogleTest(&argc, argv); + const int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} From 201d6216f3a245c65b0fae8650c92c4eb2b178c4 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Mon, 19 Oct 2026 11:43:27 -0700 Subject: [PATCH 06/12] [Claude] Add long run-level restart tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/test_restart_runs.py runs real simulations (Voce FCC crystal plasticity through ExaCMech on a 4096-element mesh) and checks that restarted runs match an uninterrupted reference bitwise: - cyclic: four ±1% strain cycles, chained every cycle and every two cycles, and restarted from each past checkpoint of the reference - monotonic: 10% tension in z, chained halfway and restarted from the reference's halfway checkpoint - empty_ranks: a 2-element mesh, so all but two ranks own no elements Every checkpoint a restarted run writes is compared with the reference checkpoint of the same step, and its volume-average output row by row. Each restarted run's log must show that it restarted from the expected checkpoint and ran only the remaining steps. A negative control restarts from a checkpoint with the state variables perturbed by 1e-12 and requires the results to differ. The suite takes 10 to 15 minutes on 8 ranks, so it is not part of the default (CI) test suite. It is registered only when configured with -DEXACONSTIT_ENABLE_LONG_TESTS=ON, runs with `ctest -L long`, and uses EXACONSTIT_RESTART_RUN_TEST_NP ranks (default 8, capped at EXACONSTIT_TEST_MAX_MPI_TASKS). Developers changing the restart code must run it locally; developers_guide.md and doc/restart.md say so. --- developers_guide.md | 13 +- doc/restart.md | 23 ++ test/CMakeLists.txt | 39 ++ test/test_restart_runs.py | 850 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 924 insertions(+), 1 deletion(-) create mode 100644 test/test_restart_runs.py diff --git a/developers_guide.md b/developers_guide.md index 0bf32985..33430f7c 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -597,6 +597,16 @@ cd test/data mpirun -np 4 ../../build/mechanics -opt example.toml ``` +### Long Tests +Some tests take too long for CI and are only registered when configured with +`-DEXACONSTIT_ENABLE_LONG_TESTS=ON`. They currently cover the checkpoint/restart code on real +simulations; anyone changing that code must run them locally and make sure they pass: +```bash +ctest -L long --output-on-failure +``` +No test uses more MPI ranks than `EXACONSTIT_TEST_MAX_MPI_TASKS`, which defaults to the +number of logical cores detected at configure time. + ### Example Workflows The `test/data/` directory contains various example cases: - **Crystal plasticity simulations** @@ -660,7 +670,8 @@ original, local numbering included. Anything that cannot be rebuilt that way nee checkpoint data. **Test it.** `test/test_restart_e2e.cpp` compares a restarted SimulationState with the -original bitwise; add any new state to it. +original bitwise; add any new state to it, and run the [long tests](#long-tests) for anything +that changes what a simulation evolves. ### Performance Considerations - **GPU kernels**: Use RAJA for performance portability diff --git a/doc/restart.md b/doc/restart.md index 2caec526..9f2ffa21 100644 --- a/doc/restart.md +++ b/doc/restart.md @@ -177,6 +177,29 @@ No test uses more MPI ranks than the machine has cores: rank counts above `EXACONSTIT_TEST_MAX_MPI_TASKS` (default: the logical core count detected at configure time) are skipped. +**Long tests** (not on CI; developers changing the restart code must run them locally): +configure with `-DEXACONSTIT_ENABLE_LONG_TESTS=ON`, then run + +```bash +ctest -L long --output-on-failure +``` + +This runs `test/test_restart_runs.py` on real simulations (Voce FCC crystal plasticity on a +4096-element mesh, 8 ranks by default via `EXACONSTIT_RESTART_RUN_TEST_NP`, capped at +`EXACONSTIT_TEST_MAX_MPI_TASKS`), which takes 10 to 15 minutes: +- **cyclic**: four ±1% strain cycles, chained every cycle and every two cycles, and restarted + from each past checkpoint of the uninterrupted reference; +- **monotonic**: 10% tension, chained halfway and restarted from the reference's halfway + checkpoint; +- **empty_ranks**: a 2-element mesh, so all but two ranks own no elements. + +Every checkpoint a restarted run writes is compared bitwise with the reference checkpoint of +the same step, and its volume-average output row by row. Each restarted run's log is checked to +confirm it really restarted from the expected checkpoint and ran only the remaining steps, and +a negative control restarts from a slightly perturbed checkpoint and requires the result to +differ. `python3 test_restart_runs.py --help` lists the options, including a smaller mesh, fewer +steps and fewer ranks for quick local checks. + ## Limitations - **Same rank count.** Restarting on a different number of ranks would need a diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5f59f57a..0726553b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -101,6 +101,45 @@ if(ENABLE_AXOM) SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/compare_restart_checkpoints.cpp OUTPUT_DIR ${TEST_OUTPUT_DIR} DEPENDS_ON axom) + + # Restart tests on real runs (test_restart_runs.py): cyclic loading chained every 1 and 2 + # cycles, long monotonic tension chained halfway, a run with empty ranks, restarts from past + # reference checkpoints, and a negative control. They take 10-15 minutes on 8 ranks, so they + # are not part of the default (CI) test suite. Developers working on the restart code must + # enable them locally with -DEXACONSTIT_ENABLE_LONG_TESTS=ON and make sure they pass: + # ctest -L long --output-on-failure + option(EXACONSTIT_ENABLE_LONG_TESTS + "Register the long-running tests (run-level restart tests); not for CI" OFF) + set(EXACONSTIT_RESTART_RUN_TEST_NP 8 CACHE STRING + "MPI ranks for test_restart_runs (capped at EXACONSTIT_TEST_MAX_MPI_TASKS)") + if(EXACONSTIT_ENABLE_LONG_TESTS) + set(_restart_runs_np ${EXACONSTIT_RESTART_RUN_TEST_NP}) + if(_restart_runs_np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + message(STATUS "test_restart_runs: using ${EXACONSTIT_TEST_MAX_MPI_TASKS} MPI ranks " + "instead of ${EXACONSTIT_RESTART_RUN_TEST_NP} " + "(EXACONSTIT_TEST_MAX_MPI_TASKS)") + set(_restart_runs_np ${EXACONSTIT_TEST_MAX_MPI_TASKS}) + endif() + set(_restart_runs_mpiexec "${MPIEXEC_EXECUTABLE}") + if(NOT _restart_runs_mpiexec) + set(_restart_runs_mpiexec "mpirun") + endif() + add_custom_command(TARGET compare_restart_checkpoints POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/test/test_restart_runs.py + $/../test/test_restart_runs.py) + add_test(NAME test_restart_runs + COMMAND ${PYTHON_EXECUTABLE} test_restart_runs.py + --np ${_restart_runs_np} + --mpiexec ${_restart_runs_mpiexec} + --exe $ + --compare-tool $ + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_runs PROPERTIES + LABELS "restart;long" + PROCESSORS ${_restart_runs_np} + TIMEOUT 3600) + endif() endif() if (NOT DEFINED PYTHON_EXECUTABLE) diff --git a/test/test_restart_runs.py b/test/test_restart_runs.py new file mode 100644 index 00000000..527d52ac --- /dev/null +++ b/test/test_restart_runs.py @@ -0,0 +1,850 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +ExaConstit Restart Testing Framework +Runs real ExaConstit jobs and checks that restarted runs match uninterrupted ones exactly + +Each test case runs a reference job once, uninterrupted, writing checkpoints along the way. +It then checks two kinds of restart against that reference: + +* Chained runs: the job is stopped and restarted from its own checkpoints, one segment at a + time (each segment extends t_final). +* Restarts from the reference's checkpoints: a continuation reads a past checkpoint of the + reference with `[Restart] read_directory` and runs to the end, writing its own checkpoints + elsewhere. Any number of past checkpoints can be checked this way without rerunning the + reference. + +Every checkpoint a restarted job writes is compared with the reference checkpoint of the same +cycle (scripts/restart/compare_restart_checkpoints.cpp), and the volume-average outputs are +compared row by row. Both comparisons are bitwise unless --rtol is given. + +A bitwise match alone cannot tell a real restart from a run that ignored the checkpoint and +recomputed from step 0 (the runs are deterministic), so two more checks guard against that: + +* Each restarted job's log must report restarting from the expected checkpoint, and the job + must run exactly the remaining steps, starting at the step after the checkpoint. +* Negative control (needs h5py; skipped with a warning otherwise): the state variables in a + copy of a reference checkpoint are perturbed, a continuation restarts from it, and its final + checkpoint and outputs must DIFFER from the reference. This proves the restored state is what + the continuation uses, and that the comparisons detect a difference. + +Cases: + cyclic Fully reversed strain cycles along z (0 -> +amp -> -amp -> 0) at a fixed dt. + Chains stop every cycle and every 2 cycles. Cycle ends fall inside a boundary + condition segment, so the boundary-condition replay on restart is exercised. + monotonic Long tension along z at a fixed dt. A chain stops halfway; the reference is also + restarted from its halfway checkpoint; the negative control runs here. + empty_ranks Short tension run on a 2-element mesh, so with more than 2 ranks every rank but + two owns no elements. Covers checkpointing and restarting with empty ranks. + +Defaults: 16^3 = 4096 hex elements, 8 MPI ranks, 4 cycles of +/-1% strain (160 steps), 10% +monotonic strain (100 steps), and a 20-step empty-rank run. Run with --help for the options +(mesh size, rank count, checkpoint interval and retention, which reference checkpoints to +restart from). +""" + +import argparse +import random +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, List, Optional, Tuple + +import numpy as np + +SCRIPT_DIR = Path(__file__).resolve().parent + +# The top face is pulled in z at this velocity and every run uses this fixed time step, giving +# 1e-3 strain per step on the unit cube. +TOP_VELOCITY = 0.01 +DT = 0.1 + + +class Failure(Exception): + """Raised when a run fails outright, which stops the whole test""" + + +@dataclass +class MeshSpec: + """Auto-generated mesh used by a test case""" + nxyz: Tuple[int, int, int] + grain_file: str + + @property + def num_elements(self) -> int: + return self.nxyz[0] * self.nxyz[1] * self.nxyz[2] + + +# ============================================================================ +# INPUT FILES +# ============================================================================ + +def find_data_dir(explicit: Optional[str]) -> Path: + """ + Find the directory holding the Voce material input files + + Args: + explicit: Directory given on the command line, or None to search next to this script + + Returns: + Directory containing props_cp_voce.txt + """ + candidates = [Path(explicit)] if explicit else [SCRIPT_DIR, SCRIPT_DIR / "data"] + for candidate in candidates: + if (candidate / "props_cp_voce.txt").exists(): + return candidate + raise Failure(f"cannot find props_cp_voce.txt in {[str(c) for c in candidates]}") + + +def prepare_inputs(workdir: Path, data_dir: Path, meshes: List[MeshSpec], num_grains: int, + seed: int) -> None: + """ + Copy the material inputs into the work directory and write one grain file per mesh + + Args: + workdir: Work directory (created if needed) + data_dir: Directory with the Voce material input files + meshes: Meshes whose grain files to write + num_grains: Number of grains (orientations) to draw from + seed: Seed for the element-to-grain assignment, so every run gets the same grains + """ + workdir.mkdir(parents=True, exist_ok=True) + for name in ("props_cp_voce.txt", "state_cp_voce.txt", "voce_quats.ori"): + shutil.copy(data_dir / name, workdir / name) + for mesh in meshes: + rng = random.Random(seed) + with open(workdir / mesh.grain_file, "w") as f: + for _ in range(mesh.num_elements): + f.write(f"{rng.randint(1, num_grains)}\n") + + +def bc_block(update_steps: List[int], top_velocities: List[float]) -> str: + """ + Build the [BCs] table for a run whose top-face velocity changes at given steps + + The bottom face is fixed in z, the x = 0 and y = 0 faces are symmetry planes, and the top + face moves in z with the given velocity. The legacy changing-BC format is used because it + expresses a per-step velocity schedule directly. + + Args: + update_steps: Steps at which a new velocity takes effect (must start with 1) + top_velocities: Top-face z velocity for each update step + + Returns: + TOML text of the [BCs] table + """ + ids = ", ".join(["[1, 2, 3, 4]"] * len(update_steps)) + comps = ", ".join(["[3, 1, 2, 3]"] * len(update_steps)) + vals = ",\n ".join( + "[" + ", ".join(["0.0"] * 11 + [repr(v)]) + "]" for v in top_velocities) + return f"""[BCs] + changing_ess_bcs = true + update_steps = {list(update_steps)} + essential_ids = [{ids}] + essential_comps = [{comps}] + essential_vals = [{vals}] +""" + + +def restart_block(restart: Dict[str, object]) -> str: + """ + Build the [Restart] table + + Args: + restart: [Restart] keys and values (bool, int, float or string) + + Returns: + TOML text of the [Restart] table + """ + lines = ["[Restart]"] + for key, value in restart.items(): + if isinstance(value, bool): + lines.append(f" {key} = {'true' if value else 'false'}") + elif isinstance(value, (int, float)): + lines.append(f" {key} = {value}") + else: + lines.append(f' {key} = "{value}"') + return "\n".join(lines) + "\n" + + +def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: str, + t_final: float, restart: Dict[str, object]) -> None: + """ + Write the options file for one run + + Every run uses the same material (Voce FCC crystal plasticity through ExaCMech), solver + settings and volume-average outputs; only the mesh, boundary conditions, final time and + restart settings differ between runs. + + Args: + path: Options file to write + basename: Output basename; runs sharing it append to the same output files + mesh: Mesh of the test case + num_grains: Number of grains in the orientation file + bcs: [BCs] table from bc_block() + t_final: Final simulation time of this run + restart: [Restart] keys and values + """ + path.write_text(f"""Version = "0.10.0" +basename = "{basename}" +grain_file = "{mesh.grain_file}" + +[[Materials]] + material_name = "voce_fcc" + mech_type = "exacmech" + temperature = 298.0 + [Materials.Properties] + floc = "props_cp_voce.txt" + num_props = 17 + [Materials.State_Vars] + floc = "state_cp_voce.txt" + num_vars = 24 + [Materials.Grain] + ori_state_var_loc = 9 + ori_stride = 4 + ori_type = "quat" + num_grains = {num_grains} + orientation_file = "voce_quats.ori" + [Materials.Model] + crystal_plasticity = true + [Materials.Model.ExaCMech] + shortcut = "evptn_FCC_A" + +{bcs} +[Time] + [Time.Fixed] + dt = {DT} + t_final = {t_final!r} + +[Visualizations] + visit = false + conduit = false + paraview = false + +[PostProcessing] + [PostProcessing.volume_averages] + enabled = true + stress = true + def_grad = true + euler_strain = true + plastic_work = true + output_frequency = 1 + output_directory = "./results" + +[Solvers] + assembly = "FULL" + rtmodel = "CPU" + [Solvers.NR] + iter = 25 + rel_tol = 5e-5 + abs_tol = 5e-10 + [Solvers.Krylov] + iter = 1000 + rel_tol = 1e-7 + abs_tol = 1e-27 + solver = "CG" + +[Mesh] + type = "auto" + ref_ser = 0 + ref_par = 0 + p_refinement = 1 + [Mesh.Auto] + mxyz = [1.0, 1.0, 1.0] + nxyz = [{mesh.nxyz[0]}, {mesh.nxyz[1]}, {mesh.nxyz[2]}] + +{restart_block(restart)}""") + + +def time_of(steps: int) -> float: + """ + Simulation time after a number of fixed steps + + Args: + steps: Number of completed steps + + Returns: + steps * DT, rounded so it prints and compares cleanly + """ + return round(steps * DT, 10) + + +# ============================================================================ +# RUNNING AND COMPARING +# ============================================================================ + +class Harness: + """Runs ExaConstit jobs and records the outcome of every check""" + + def __init__(self, args: argparse.Namespace, workdir: Path): + """ + Args: + args: Parsed command-line options + workdir: Work directory holding the inputs, checkpoints, outputs and logs + """ + self.args = args + self.workdir = workdir + self.logdir = workdir / "logs" + self.logdir.mkdir(exist_ok=True) + self.failures: List[str] = [] + self.checks = 0 + + def check(self, ok: bool, message: str) -> None: + """ + Record and print the outcome of one check + + Args: + ok: Whether the check passed + message: Description of what was checked and what was found + """ + self.checks += 1 + print(f" [{'PASS' if ok else 'FAIL'}] {message}", flush=True) + if not ok: + self.failures.append(message) + + def run(self, name: str, toml_path: Path) -> None: + """ + Run ExaConstit on an options file, logging its output + + With --np 1 the binary is run directly (no MPI launcher) unless --always-mpiexec is + given, which allows single-rank runs where the launcher cannot be used. + + Args: + name: Run name, used for the log file + toml_path: Options file in the work directory + + Raises: + Failure: If the run exits with a non-zero status + """ + if self.args.np == 1 and not self.args.always_mpiexec: + cmd = [str(self.args.exe), "-opt", toml_path.name] + else: + cmd = self.args.mpiexec.split() + ["-np", str(self.args.np), str(self.args.exe), + "-opt", toml_path.name] + log = self.logdir / f"{name}.log" + start = time.time() + with open(log, "w") as f: + result = subprocess.run(cmd, cwd=self.workdir, stdout=f, stderr=subprocess.STDOUT) + elapsed = time.time() - start + print(f" ran {name} in {elapsed:.1f} s (log: {log.relative_to(self.workdir.parent)})", + flush=True) + if result.returncode != 0: + tail = log.read_text().splitlines()[-30:] + raise Failure(f"run {name} failed with exit code {result.returncode}:\n " + + "\n ".join(tail)) + + def verify_restart_log(self, name: str, read_dir: str, cycle: int, end_step: int) -> None: + """ + Check from a run's log that it really restarted, from the expected checkpoint + + The run must report restarting from read_dir/cycle_ and restoring the state, and + it must run exactly steps cycle+1 .. end_step. A run that ignored the checkpoint and + started from step 0 would otherwise go unnoticed, since it produces the same results. + + Args: + name: Run name (its log is checked) + read_dir: Checkpoint directory the run should have read, relative to the work + directory + cycle: Cycle of the checkpoint it should have restarted from + end_step: Last step it should have run + """ + text = (self.logdir / f"{name}.log").read_text() + expected = f"Restarting from checkpoint {read_dir}/cycle_{cycle:010d} (cycle {cycle}," + restarted = expected in text and "Restored checkpoint state" in text + steps = [int(line.split(":")[1]) for line in text.splitlines() + if line.startswith("Simulation cycle:")] + ran_remaining = steps == list(range(cycle + 1, end_step + 1)) + detail = (f"steps {steps[0]}..{steps[-1]}" if steps else "no steps") + \ + f" (expected {cycle + 1}..{end_step})" + self.check(restarted and ran_remaining, + f"{name}: restarted from {read_dir}/cycle_{cycle:010d} and ran only the " + f"remaining steps: {'yes' if restarted else 'NO restart message'}, {detail}") + + def checkpoint(self, ckpt_root: str, cycle: int) -> Path: + """ + Path of a checkpoint directory + + Args: + ckpt_root: Checkpoint root directory, relative to the work directory + cycle: Completed cycles of the checkpoint + + Returns: + Path of the cycle_ directory + """ + return self.workdir / ckpt_root / f"cycle_{cycle:010d}" + + def compare_checkpoints(self, label: str, ckpt_a: Path, ckpt_b: Path, + expect_differ: bool = False) -> None: + """ + Compare two checkpoints with compare_restart_checkpoints + + Args: + label: Description used in the check message + ckpt_a: First checkpoint directory (normally the reference) + ckpt_b: Second checkpoint directory + expect_differ: If True, the check passes only if the checkpoints differ + """ + for ckpt in (ckpt_a, ckpt_b): + if not (ckpt / "commit.json").exists(): + self.check(False, f"{label}: committed checkpoint " + f"{ckpt.relative_to(self.workdir)} exists") + return + cmd = [str(self.args.compare_tool), str(ckpt_a), str(ckpt_b)] + if self.args.rtol > 0: + cmd += ["--rtol", repr(self.args.rtol)] + result = subprocess.run(cmd, capture_output=True, text=True) + summary = (result.stdout.strip().splitlines() or ["(no output)"])[-1] + if expect_differ: + self.check(result.returncode == 1, f"{label}: must differ: {summary}") + return + self.check(result.returncode == 0, f"{label}: {summary}") + if result.returncode != 0: + for line in result.stdout.strip().splitlines()[:15]: + print(f" {line}") + + def compare_outputs(self, label: str, ref_basename: str, other_basename: str, + after_time: Optional[float] = None, expect_differ: bool = False) -> None: + """ + Compare every volume-average output file of two runs + + The time columns must match exactly, and the values bitwise (or within --rtol). + + Args: + label: Description used in the check message + ref_basename: Basename of the reference run + other_basename: Basename of the run to compare + after_time: If given, reference rows with time <= after_time are skipped (for runs + that restarted at that time and so only wrote the later rows) + expect_differ: If True, the check passes only if the outputs differ + """ + ref_dir = self.workdir / "results" / ref_basename + other_dir = self.workdir / "results" / other_basename + ref_files = sorted(p.name for p in ref_dir.glob("avg_*.txt")) + if not ref_files: + self.check(False, f"{label}: reference outputs exist in {ref_dir}") + return + worst = 0.0 + problems = [] + for name in ref_files: + other = other_dir / name + if not other.exists(): + problems.append(f"{name} missing") + continue + a = np.atleast_2d(np.loadtxt(ref_dir / name, comments="#")) + b = np.atleast_2d(np.loadtxt(other, comments="#")) + if after_time is not None: + a = a[a[:, 0] > after_time + 1e-9] + if a.shape != b.shape: + problems.append(f"{name}: {b.shape[0]} rows vs {a.shape[0]} expected") + continue + if not np.array_equal(a[:, 0], b[:, 0]): + problems.append(f"{name}: time columns differ") + continue + scale = np.maximum(np.maximum(np.abs(a), np.abs(b)), 1e-300) + rel = np.max(np.abs(a - b) / scale) + worst = max(worst, rel) + if (self.args.rtol == 0 and not np.array_equal(a, b)) or rel > self.args.rtol: + problems.append(f"{name}: max rel diff {rel:.3e}") + detail = "; ".join(problems) if problems else \ + f"{len(ref_files)} files, max rel diff {worst:.3e}" + if expect_differ: + self.check(bool(problems), f"{label}: outputs must differ from reference ({detail})") + return + self.check(not problems, f"{label}: outputs vs reference ({detail})") + + +# ============================================================================ +# TEST CASE BUILDING BLOCKS +# ============================================================================ + +def parse_cycles(spec: Optional[str], available: List[int], default: List[int]) -> List[int]: + """ + Interpret a --*-restart-from option + + Args: + spec: "all", "none", a comma-separated list of cycles, or None for the default + available: Reference checkpoint cycles that can be restarted from + default: Cycles to use when spec is None + + Returns: + Cycles to restart from + """ + if spec == "all": + return list(available) + if spec == "none": + return [] + if spec is None: + return default + return [int(s) for s in spec.split(",") if s] + + +def run_reference(h: Harness, name: str, mesh: MeshSpec, bcs: str, total_steps: int, + interval: int) -> List[int]: + """ + Run a case's uninterrupted reference job, writing checkpoints along the way + + Args: + h: Test harness + name: Reference run name (also its basename and checkpoint directory suffix) + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Number of steps to run + interval: Checkpoint interval in steps + + Returns: + Cycles of the complete checkpoints the reference kept + """ + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": interval, "write_final": True, + "keep_last": h.args.keep_last, "directory": f"ckpt_{name}"}) + h.run(name, h.workdir / f"{name}.toml") + ref_cycles = sorted(int(p.name.split("_")[1]) + for p in (h.workdir / f"ckpt_{name}").glob("cycle_*") + if (p / "commit.json").exists()) + print(f" reference checkpoints kept: {ref_cycles}") + return ref_cycles + + +def run_chain(h: Harness, ref: str, ref_cycles: List[int], name: str, mesh: MeshSpec, bcs: str, + stops: List[int]) -> None: + """ + Run a job in segments, each restarting from the previous one, and compare with the reference + + Segment 1 starts fresh; every later segment restarts from the newest checkpoint in the + chain's own checkpoint directory and extends t_final to its stop. Every segment writes a + final checkpoint. All segments share one basename, so their volume-average outputs append + into the same files, which are then compared with the reference's in full. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + name: Chain name (its basename and checkpoint directory suffix) + mesh: Mesh of the case + bcs: [BCs] table + stops: Step at which each segment stops, increasing; the last is the end of the run + """ + for i, stop in enumerate(stops): + segment = f"{name}_seg{i + 1}" + restart = {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{name}"} + if i > 0: + restart["read"] = True + write_toml(h.workdir / f"{segment}.toml", name, mesh, h.args.num_grains, bcs, + time_of(stop), restart) + h.run(segment, h.workdir / f"{segment}.toml") + if i > 0: + h.verify_restart_log(segment, f"ckpt_{name}", stops[i - 1], stop) + for stop in stops: + if stop in ref_cycles: + h.compare_checkpoints(f"{name} cycle {stop}", h.checkpoint(f"ckpt_{ref}", stop), + h.checkpoint(f"ckpt_{name}", stop)) + h.compare_outputs(name, ref, name) + + +def run_from_reference(h: Harness, ref: str, ref_cycles: List[int], from_cycles: List[int], + mesh: MeshSpec, bcs: str, total_steps: int) -> None: + """ + Restart from past reference checkpoints and compare each continuation with the reference + + Each continuation reads the reference's checkpoint through [Restart] read_directory and + writes its own checkpoints to a separate directory, so the reference's checkpoints are never + touched. Its final checkpoint is compared with the reference's final one, and its outputs + with the reference rows after the restart time. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + from_cycles: Cycles to restart from + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Last step of the run + """ + for cycle in from_cycles: + if cycle not in ref_cycles: + h.check(False, f"{ref}: requested restart cycle {cycle} is among the kept " + f"checkpoints {ref_cycles}") + continue + name = f"{ref}_from{cycle}" + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "read": True, "read_cycle": cycle, "read_directory": f"ckpt_{ref}", + "directory": f"ckpt_{name}"}) + h.run(name, h.workdir / f"{name}.toml") + h.verify_restart_log(name, f"ckpt_{ref}", cycle, total_steps) + h.compare_checkpoints(f"{name} final cycle {total_steps}", + h.checkpoint(f"ckpt_{ref}", total_steps), + h.checkpoint(f"ckpt_{name}", total_steps)) + h.compare_outputs(name, ref, name, after_time=time_of(cycle)) + + +def run_negative_control(h: Harness, ref: str, ref_cycles: List[int], mesh: MeshSpec, bcs: str, + total_steps: int) -> None: + """ + Restart from a perturbed copy of a reference checkpoint; the result must differ + + The latest reference checkpoint before the end is copied, and every beginning-of-step state + variable on rank 0 is scaled by (1 + 1e-12). The end-of-step partners are stored as copies + of those values, so they are perturbed consistently. A continuation then restarts from the + copy, and its final checkpoint and outputs must differ from the reference. This shows that + the restored state is what the continuation actually uses, and that the comparisons pick up + even a tiny difference. Skipped with a warning if h5py is not available. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Last step of the run + """ + try: + import h5py + except ImportError: + print(" WARNING: h5py is not available; skipping the negative control") + return + candidates = [c for c in ref_cycles if c < total_steps] + if not candidates: + return + cycle = candidates[-1] # the latest one keeps this extra run short + name = f"{ref}_perturbed{cycle}" + source = h.workdir / f"ckpt_{name}_src" + shutil.copytree(h.checkpoint(f"ckpt_{ref}", cycle), source / f"cycle_{cycle:010d}") + with h5py.File(source / f"cycle_{cycle:010d}" / "rank_000000.hdf5", "r+") as f: + paths = [f"sidre/external/qfuncs/{g}/values" for g in f["sidre/external/qfuncs"] + if g.startswith("state_var_beg")] + if not paths: + h.check(False, f"{name}: found state_var_beg arrays to perturb") + return + for path in paths: + f[path][...] = f[path][...] * (1.0 + 1e-12) + print(f" negative control: perturbed {', '.join(p.split('/')[3] for p in paths)} by 1e-12 " + f"(relative) in a copy of {ref} cycle {cycle}") + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "read": True, "read_cycle": cycle, "read_directory": source.name, + "directory": f"ckpt_{name}"}) + h.run(name, h.workdir / f"{name}.toml") + h.verify_restart_log(name, source.name, cycle, total_steps) + h.compare_checkpoints(f"{name} (negative control) final cycle {total_steps}", + h.checkpoint(f"ckpt_{ref}", total_steps), + h.checkpoint(f"ckpt_{name}", total_steps), expect_differ=True) + h.compare_outputs(f"{name} (negative control)", ref, name, after_time=time_of(cycle), + expect_differ=True) + + +# ============================================================================ +# TEST CASES +# ============================================================================ + +def run_cyclic(h: Harness, mesh: MeshSpec) -> None: + """ + Cyclic case: fully reversed strain cycles, chained every 1 and 2 cycles + + Each cycle goes 0 -> +amp -> -amp -> 0 in z, with the top-face velocity reversing at the + peaks, so a cycle takes 4 * steps_per_quarter steps. The boundary-condition update steps + fall at the peaks, while the checkpoints fall at cycle ends, in the middle of a segment, so + restarting exercises the boundary-condition replay. + + Args: + h: Test harness + mesh: Mesh of the case + """ + args = h.args + quarter = args.steps_per_quarter + steps_per_cycle = 4 * quarter + total_steps = args.cycles * steps_per_cycle + # +v for the first quarter, then alternating -v / +v for half a cycle each. + update_steps = [1] + velocities = [TOP_VELOCITY] + step = 1 + quarter + sign = -1.0 + while step <= total_steps: + update_steps.append(step) + velocities.append(sign * TOP_VELOCITY) + step += 2 * quarter + sign = -sign + bcs = bc_block(update_steps, velocities) + amplitude = quarter * TOP_VELOCITY * DT + interval = args.cyclic_checkpoint_interval or steps_per_cycle + print(f"\n== cyclic: {args.cycles} cycles of +/-{amplitude * 100:g}% strain, {total_steps} " + f"steps, BC updates at {update_steps}, reference checkpoints every {interval} steps") + + ref = "cyclic_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + + for stride in args.cyclic_chain_strides: + stops = list(range(stride * steps_per_cycle, total_steps + 1, stride * steps_per_cycle)) + if stops[-1] != total_steps: + stops.append(total_steps) + run_chain(h, ref, ref_cycles, f"cyclic_chain{stride}", mesh, bcs, stops) + + restartable = [c for c in ref_cycles if c < total_steps] + from_cycles = parse_cycles(args.cyclic_restart_from, restartable, restartable) + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + + +def run_monotonic(h: Harness, mesh: MeshSpec) -> None: + """ + Monotonic case: long tension in z, chained halfway, plus the negative control + + Args: + h: Test harness + mesh: Mesh of the case + """ + args = h.args + total_steps = args.monotonic_steps + half = total_steps // 2 + bcs = bc_block([1], [TOP_VELOCITY]) + interval = args.monotonic_checkpoint_interval or max(1, total_steps // 10) + print(f"\n== monotonic: {total_steps * TOP_VELOCITY * DT * 100:g}% tensile strain along z, " + f"{total_steps} steps, reference checkpoints every {interval} steps") + + ref = "mono_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + run_chain(h, ref, ref_cycles, "mono_chain_half", mesh, bcs, [half, total_steps]) + + restartable = [c for c in ref_cycles if c < total_steps] + default_from = [half] if half in ref_cycles else \ + [min(restartable, key=lambda c: abs(c - half))] if restartable else [] + from_cycles = parse_cycles(args.monotonic_restart_from, restartable, default_from) + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + if args.negative_control: + run_negative_control(h, ref, ref_cycles, mesh, bcs, total_steps) + + +def run_empty_ranks(h: Harness, mesh: MeshSpec) -> None: + """ + Empty-rank case: a short tension run on a 2-element mesh + + With fewer elements than ranks, MFEM puts element i on rank i, so ranks 0 and 1 own one + element each (and share a face) while every other rank owns nothing. Those ranks must still + write and read their (empty) checkpoint files and take part in every collective step. The + reference is chained halfway and restarted from its halfway checkpoint, like the monotonic + case. + + Args: + h: Test harness + mesh: Mesh of the case (2 x 1 x 1 elements) + """ + args = h.args + total_steps = args.empty_ranks_steps + half = total_steps // 2 + bcs = bc_block([1], [TOP_VELOCITY]) + interval = max(1, total_steps // 4) + empty = max(0, args.np - mesh.num_elements) + print(f"\n== empty_ranks: {mesh.num_elements} elements on {args.np} rank(s) ({empty} without " + f"elements), {total_steps} steps, reference checkpoints every {interval} steps") + + ref = "empty_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + run_chain(h, ref, ref_cycles, "empty_chain_half", mesh, bcs, [half, total_steps]) + from_cycles = [half] if half in ref_cycles else [] + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + + +# ============================================================================ +# MAIN +# ============================================================================ + +def main() -> int: + """ + Parse the options, run the requested cases and report the results + + Returns: + 0 if every check passed, 1 if a check or run failed, 2 for a setup error + """ + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--np", type=int, default=8, help="MPI ranks (default 8)") + p.add_argument("--mpiexec", default="mpirun", help="MPI launcher (default mpirun)") + p.add_argument("--always-mpiexec", action="store_true", + help="use the launcher even with --np 1 (default: run the binary directly)") + p.add_argument("--exe", default=str(SCRIPT_DIR / ".." / "bin" / "mechanics")) + p.add_argument("--compare-tool", + default=str(SCRIPT_DIR / ".." / "bin" / "compare_restart_checkpoints")) + p.add_argument("--data-dir", default=None, help="directory with props_cp_voce.txt etc.") + p.add_argument("--workdir", default=str(Path.cwd() / "restart_runs")) + p.add_argument("--cases", default="cyclic,monotonic,empty_ranks", + help="comma-separated cases to run (default cyclic,monotonic,empty_ranks)") + p.add_argument("--nxyz", type=int, default=16, + help="cyclic/monotonic: elements per cube edge (default 16)") + p.add_argument("--num-grains", type=int, default=500) + p.add_argument("--seed", type=int, default=12345, help="seed for the element-to-grain map") + p.add_argument("--rtol", type=float, default=0.0, help="0 = bitwise (default)") + p.add_argument("--keep-last", type=int, default=0, + help="[Restart] keep_last for the reference runs (0 = keep every checkpoint)") + p.add_argument("--cycles", type=int, default=4, help="cyclic: number of cycles (default 4)") + p.add_argument("--steps-per-quarter", type=int, default=10, + help="cyclic: steps from 0 to peak strain; 1e-3 strain per step " + "(default 10 = 1%%)") + p.add_argument("--cyclic-chain-strides", default="1,2", + help="cyclic: chained runs stopping every N cycles (default 1,2)") + p.add_argument("--cyclic-checkpoint-interval", type=int, default=0, + help="cyclic: reference checkpoint interval in steps (default: one cycle)") + p.add_argument("--cyclic-restart-from", default=None, + help="cyclic: reference cycles to restart from: list, 'all' or 'none' " + "(default all kept, except the last)") + p.add_argument("--monotonic-steps", type=int, default=100, + help="monotonic: steps; 1e-3 strain per step (default 100 = 10%%)") + p.add_argument("--monotonic-checkpoint-interval", type=int, default=0, + help="monotonic: reference checkpoint interval in steps (default steps/10)") + p.add_argument("--monotonic-restart-from", default=None, + help="monotonic: reference cycles to restart from: list, 'all' or 'none' " + "(default: the halfway checkpoint)") + p.add_argument("--empty-ranks-steps", type=int, default=20, + help="empty_ranks: steps (default 20)") + p.add_argument("--no-negative-control", dest="negative_control", action="store_false", + help="skip the perturbed-checkpoint negative control") + args = p.parse_args() + args.cyclic_chain_strides = [int(s) for s in args.cyclic_chain_strides.split(",") if s] + args.exe = Path(args.exe).resolve() + args.compare_tool = Path(args.compare_tool).resolve() + + for tool in (args.exe, args.compare_tool): + if not tool.exists(): + print(f"ERROR: {tool} not found") + return 2 + + cases: Dict[str, Tuple[Callable[[Harness, MeshSpec], None], MeshSpec]] = { + "cyclic": (run_cyclic, MeshSpec((args.nxyz,) * 3, "grains.txt")), + "monotonic": (run_monotonic, MeshSpec((args.nxyz,) * 3, "grains.txt")), + "empty_ranks": (run_empty_ranks, MeshSpec((2, 1, 1), "grains_empty_ranks.txt")), + } + requested = [c.strip() for c in args.cases.split(",") if c.strip()] + unknown = [c for c in requested if c not in cases] + if unknown: + print(f"ERROR: unknown case(s) {unknown}; choose from {list(cases)}") + return 2 + + workdir = Path(args.workdir).resolve() + if workdir.exists(): + shutil.rmtree(workdir) + try: + meshes = {cases[c][1].grain_file: cases[c][1] for c in requested} + prepare_inputs(workdir, find_data_dir(args.data_dir), list(meshes.values()), + args.num_grains, args.seed) + h = Harness(args, workdir) + print(f"Restart run tests: {args.np} rank(s), workdir {workdir}") + for case in requested: + run_case, mesh = cases[case] + run_case(h, mesh) + except Failure as e: + print(f"\nERROR: {e}") + return 1 + + print(f"\n{h.checks - len(h.failures)}/{h.checks} checks passed") + for failure in h.failures: + print(f" FAILED: {failure}") + return 0 if not h.failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 9e9f3b87bd462292a76bdf8a17c0704313a10f4d Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 21 Oct 2026 12:42:11 -0700 Subject: [PATCH 07/12] [Claude] Fix GPU builds with Axom, Conduit and HDF5 Three problems stopped CUDA builds of the restart I/O stack and of ExaConstit against it: - Axom's AXOM_ENABLE_CUDA and AXOM_ENABLE_HIP are cmake_dependent_options on BLT's ENABLE_CUDA and ENABLE_HIP, like AXOM_ENABLE_MPI on ENABLE_MPI. The install scripts only passed AXOM_ENABLE_, so Axom defined AXOM_USE_CUDA (found through RAJA and Umpire) but compiled its sources with the host compiler, and the build stopped in axom::Array's check that device-aware code is compiled with a CUDA/HIP compiler. GPU builds now also pass ENABLE_=ON. - With both the shared and the static HDF5 library installed, Conduit links hdf5-static and Axom hdf5-shared, so ExaConstit got two copies of HDF5 on one link line, and nvcc (the linker in CUDA builds) rejects the versioned libhdf5.so. that hdf5-shared names. HDF5 is now built as a static library only, so Conduit and Axom both link libhdf5.a. - inspect_restart_rank_file and compare_restart_checkpoints only depended on axom, so BLT compiled them as host C++, which a GPU-enabled Axom rejects for the same axom::Array reason. They now also depend on blt::cuda / blt::hip in GPU builds; CPU builds are unchanged. Tested with a CUDA build (GTX 1070): HDF5, Conduit, Axom and ExaConstit build from clean and the full test suite passes. The HIP paths use the same options for the other backend and have not been built. --- .../common/build_functions_exaconstit.sh | 10 +++++++++- test/CMakeLists.txt | 19 +++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/install/common/build_functions_exaconstit.sh b/scripts/install/common/build_functions_exaconstit.sh index 7b3bf77e..6f5d5b1a 100644 --- a/scripts/install/common/build_functions_exaconstit.sh +++ b/scripts/install/common/build_functions_exaconstit.sh @@ -163,7 +163,10 @@ build_hdf5() { -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DBUILD_SHARED_LIBS=ON + # Static only. With both, Conduit links hdf5-static and Axom hdf5-shared, + # so ExaConstit gets two copies of HDF5, and nvcc (the linker in CUDA + # builds) rejects the versioned libhdf5.so. that hdf5-shared names. + -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=OFF @@ -315,7 +318,12 @@ build_axom() { if [ "${BUILD_TYPE}" != "cpu" ]; then # Spin's GPU paths run through RAJA -> Umpire memory plumbing. + # Like AXOM_ENABLE_MPI, AXOM_ENABLE_CUDA / AXOM_ENABLE_HIP are gated on + # BLT's ENABLE_CUDA / ENABLE_HIP. Without those, Axom still defines + # AXOM_USE_CUDA/HIP but compiles its sources with the host compiler, + # which fails in axom::Array's device-aware code. CMAKE_ARGS+=( + -DENABLE_${GPU_BACKEND}=ON -DAXOM_ENABLE_${GPU_BACKEND}=ON -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0726553b..c5e53e2c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -90,17 +90,32 @@ if(ENABLE_AXOM) endif() endforeach() + # The two tools below only use Axom, but an Axom built with GPU support requires every + # file using axom::Array to be compiled as device code, so they need the GPU dependency + # that makes BLT compile them with the CUDA/HIP compiler. + set(_restart_tool_depends axom) + if(ENABLE_CUDA) + if (${BLT_VERSION} VERSION_GREATER_EQUAL 0.6.0) + list(APPEND _restart_tool_depends blt::cuda_runtime blt::cuda) + else() + list(APPEND _restart_tool_depends cuda cuda_runtime) + endif() + endif() + if(ENABLE_HIP) + list(APPEND _restart_tool_depends blt::hip blt::hip_runtime) + endif() + # Diagnostic: lists how each quadrature function is stored in a checkpoint rank file blt_add_executable(NAME inspect_restart_rank_file SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/inspect_restart_rank_file.cpp OUTPUT_DIR ${TEST_OUTPUT_DIR} - DEPENDS_ON axom) + DEPENDS_ON ${_restart_tool_depends}) # Compares two checkpoint directories view by view (bitwise by default) blt_add_executable(NAME compare_restart_checkpoints SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/compare_restart_checkpoints.cpp OUTPUT_DIR ${TEST_OUTPUT_DIR} - DEPENDS_ON axom) + DEPENDS_ON ${_restart_tool_depends}) # Restart tests on real runs (test_restart_runs.py): cyclic loading chained every 1 and 2 # cycles, long monotonic tension chained halfway, a run with empty ranks, restarts from past From 37b147e9a49f1fb3dbd126cdc2ccf874e3554512 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 21 Oct 2026 12:42:11 -0700 Subject: [PATCH 08/12] [Claude] Fix restarting on ranks that own no elements A rank without elements writes its mesh_current and primal_field views with no data pointer, and Sidre stores such a view as EMPTY rather than EXTERNAL. On restart, BindFieldsAndQuadratureFunctions() required both views to be external before looking at their size, so every restart with an empty rank aborted with "fields/mesh_current/values has 0 entries or the wrong type, but this run needs 0 doubles". Empty nodal fields are now handled first, as empty quadrature functions already were: only the stored size is checked (it must be 0), and nothing is bound or loaded. Tested with the empty_ranks case of test_restart_runs.py (2 elements on 4 and 6 ranks), which failed before, on the CPU and on the GPU with PA and EA: every restart matches the uninterrupted run bitwise. --- src/sim_state/restart_checkpoint.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/sim_state/restart_checkpoint.cpp b/src/sim_state/restart_checkpoint.cpp index f17a72d5..8058f736 100644 --- a/src/sim_state/restart_checkpoint.cpp +++ b/src/sim_state/restart_checkpoint.cpp @@ -1777,9 +1777,9 @@ void RestartCheckpoint::BindFieldsAndQuadratureFunctions(Group* root, Simulation // Binding only records the target pointer; the data is read later by RestoreState()'s // loadExternalData() call. HostReadWrite() marks the host copy as the valid one, so on a // GPU run the next device access copies the restored values to the device instead of - // using stale device data. On a rank with no elements both vectors are empty: the size - // check still runs (0 == 0), nothing is bound, and the zero-length stored view is dropped - // before the load. + // using stale device data. On a rank with no elements both vectors are empty. Their views + // were written without a data pointer, which Sidre stores as an EMPTY view rather than an + // external one, so only the size is checked (0 == 0) and nothing is bound or loaded. // --------------------------------------------------------------------- const auto bind_vector = [&](const std::string& path, mfem::Vector& target) { if (!root->hasView(path)) { @@ -1787,6 +1787,13 @@ void RestartCheckpoint::BindFieldsAndQuadratureFunctions(Group* root, Simulation return; } View* view = root->getView(path); + if (target.Size() == 0) { + if (view->getNumElements() != 0) { + RecordFailure(path + " has " + std::to_string(view->getNumElements()) + + " entries, but this rank owns no elements"); + } + return; + } if (!view->isExternal() || view->getTypeID() != axom::sidre::FLOAT64_ID || view->getNumElements() != target.Size()) { RecordFailure(path + " has " + std::to_string(view->getNumElements()) + @@ -1794,9 +1801,7 @@ void RestartCheckpoint::BindFieldsAndQuadratureFunctions(Group* root, Simulation std::to_string(target.Size()) + " doubles"); return; } - if (target.Size() > 0) { - view->setExternalDataPtr(target.HostReadWrite()); - } + view->setExternalDataPtr(target.HostReadWrite()); }; bind_vector("fields/mesh_current/values", *sim_state.m_mesh_nodes.at("mesh_current")); bind_vector("fields/primal_field/values", *sim_state.m_primal_field); From e439168e863b2d310277c2d6c7baad14d777b81c Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 21 Oct 2026 12:42:11 -0700 Subject: [PATCH 09/12] [Claude] Share Umpire pools between MFEM and SNLS MFEM and SNLS each created their own host and device Umpire pools, and in GPU builds SNLS starts out with a GPU execution strategy. ExaCMech allocates its model parameters through SNLS before it sets the strategy of the model, so runs with rtmodel = "CPU" also allocated GPU memory. - GPU builds of MFEM in the install scripts now enable MFEM_USE_UMPIRE with the Umpire the scripts build. - The driver creates one host and one device Umpire pool and hands them to MFEM (by allocator name, which MFEM reuses instead of creating its own) and to SNLS (setHostAllocator/setDeviceAllocator), which ExaCMech allocates through. - The driver sets SNLS's execution strategy from rtmodel before any material model is created. CPU builds (MFEM without Umpire) only get the execution strategy setting. Tested with a CUDA build: MFEM reports host-umpire,cuda-umpire memory and the full test suite passes. --- .../install/common/build_functions_mfem.sh | 6 +++ src/mechanics_driver.cpp | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/scripts/install/common/build_functions_mfem.sh b/scripts/install/common/build_functions_mfem.sh index 3190d682..820e2f67 100644 --- a/scripts/install/common/build_functions_mfem.sh +++ b/scripts/install/common/build_functions_mfem.sh @@ -206,6 +206,12 @@ build_mfem() { -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" -DMFEM_USE_${GPU_BACKEND}=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + # Umpire, so that MFEM allocates from the same Umpire pools as SNLS and + # ExaCMech (ExaConstit's driver creates them and hands them to both) + -DMFEM_USE_UMPIRE=ON + -DUMPIRE_DIR="${UMPIRE_ROOT}" + -Dumpire_DIR="${UMPIRE_CMAKE_DIR}" + -Dfmt_DIR="${FMT_DIR}" ) if [ "${GPU_BACKEND}" = "CUDA" ]; then diff --git a/src/mechanics_driver.cpp b/src/mechanics_driver.cpp index 40ea95a9..339bde8f 100644 --- a/src/mechanics_driver.cpp +++ b/src/mechanics_driver.cpp @@ -88,6 +88,11 @@ #include "SNLS_config.h" #if defined(SNLS_RAJA_PORT_SUITE) +#include "SNLS_device_forall.h" +#include "SNLS_memory_manager.h" + +#include +#include #include #endif @@ -203,6 +208,38 @@ int main(int argc, char* argv[]) { device_config = "raja-hip"; #endif } + /* + * Umpire Memory Pools: + * - MFEM and SNLS (which ExaCMech allocates through) share one host and one device + * Umpire pool instead of each creating their own. + * - SNLS's execution strategy follows rtmodel, so that runs on the CPU allocate the + * ExaCMech model data on the host. + */ +#if defined(SNLS_RAJA_PORT_SUITE) + if (toml_opt.solvers.rtmodel == RTModel::GPU) { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::GPU); + } else if (toml_opt.solvers.rtmodel == RTModel::OPENMP) { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::OPENMP); + } else { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::CPU); + } +#if defined(MFEM_USE_UMPIRE) + { + auto& rm = umpire::ResourceManager::getInstance(); + auto host_pool = rm.makeAllocator( + "ExaConstit_HOST_pool", rm.getAllocator("HOST")); + mfem::MemoryManager::SetUmpireHostAllocatorName("ExaConstit_HOST_pool"); + snls::memoryManager::getInstance().setHostAllocator(host_pool.getId()); +#if defined(__snls_gpu_active__) + auto device_pool = rm.makeAllocator( + "ExaConstit_DEVICE_pool", rm.getAllocator("DEVICE")); + mfem::MemoryManager::SetUmpireDeviceAllocatorName("ExaConstit_DEVICE_pool"); + snls::memoryManager::getInstance().setDeviceAllocator(device_pool.getId()); +#endif + } +#endif +#endif + /* * MFEM Device Configuration: * - Configure device memory management for host/device data movement From 5401851e1a1b8b5157c0e8287ae89dedf6c0a8c6 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 21 Oct 2026 12:42:11 -0700 Subject: [PATCH 10/12] [Claude] Run the long restart tests on the GPU and cap test ranks at physical cores test_restart_runs.py wrote rtmodel = "CPU" and assembly = "FULL" into every run. It now takes --rtmodel and --assembly (defaults unchanged). GPU builds register test_restart_runs_gpu (labels restart, long, gpu), which runs the same cases with rtmodel = "GPU" and EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY (default EA) in its own work directory. The two long tests share a RESOURCE_LOCK so they never run at the same time. EXACONSTIT_TEST_MAX_MPI_TASKS defaulted to the number of logical cores, but Open MPI counts physical cores as slots by default, so on a machine with hardware threads the cap allowed rank counts that mpirun refuses to start (test_restart_runs at 8 ranks on 6 cores with 12 threads). The default is now the number of physical cores CMake detects, falling back to logical cores when that is unknown. restart.md and the developers guide document both; restart.md replaces "not yet tested on GPUs" with what has been tested: CUDA with PA and EA, not yet HIP. Tested with a CUDA build (GTX 1070): ctest -L long runs both tests and they pass. The GPU cases also passed with PA (run before the Umpire pool change, with the same harness settings). --- developers_guide.md | 5 +++-- doc/restart.md | 17 ++++++++++++----- test/CMakeLists.txt | 38 ++++++++++++++++++++++++++++++++++---- test/test_restart_runs.py | 27 +++++++++++++++++++-------- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/developers_guide.md b/developers_guide.md index 7cb9f114..f639f00d 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -604,8 +604,9 @@ simulations; anyone changing that code must run them locally and make sure they ```bash ctest -L long --output-on-failure ``` -No test uses more MPI ranks than `EXACONSTIT_TEST_MAX_MPI_TASKS`, which defaults to the -number of logical cores detected at configure time. +GPU builds also register a GPU version of them (`test_restart_runs_gpu`; `ctest -L gpu` runs +only that one). No test uses more MPI ranks than `EXACONSTIT_TEST_MAX_MPI_TASKS`, which defaults +to the number of physical cores detected at configure time (Open MPI's default number of slots). ### Example Workflows The `test/data/` directory contains various example cases: diff --git a/doc/restart.md b/doc/restart.md index 9f2ffa21..81fdffee 100644 --- a/doc/restart.md +++ b/doc/restart.md @@ -174,8 +174,8 @@ The restart tests are built when ExaConstit is configured with Axom and tests ar synthetic evolved state, deletes the input files, restarts, and compares everything bitwise. No test uses more MPI ranks than the machine has cores: rank counts above -`EXACONSTIT_TEST_MAX_MPI_TASKS` (default: the logical core count detected at configure time) -are skipped. +`EXACONSTIT_TEST_MAX_MPI_TASKS` (default: the physical core count detected at configure time, +since Open MPI counts physical cores as slots) are skipped. **Long tests** (not on CI; developers changing the restart code must run them locally): configure with `-DEXACONSTIT_ENABLE_LONG_TESTS=ON`, then run @@ -198,7 +198,13 @@ the same step, and its volume-average output row by row. Each restarted run's lo confirm it really restarted from the expected checkpoint and ran only the remaining steps, and a negative control restarts from a slightly perturbed checkpoint and requires the result to differ. `python3 test_restart_runs.py --help` lists the options, including a smaller mesh, fewer -steps and fewer ranks for quick local checks. +steps and fewer ranks for quick local checks, and the device (`--rtmodel`) and assembly +(`--assembly`) of the runs. + +GPU builds also register `test_restart_runs_gpu` (labels `restart;long;gpu`), which runs the same +cases with `rtmodel = "GPU"` and `EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY` (default `EA`, the +fastest GPU assembly for these runs). It checks that the restored host data reaches the device and +that GPU runs restart bitwise identically. Run only the GPU tests with `ctest -L gpu`. ## Limitations @@ -209,5 +215,6 @@ steps and fewer ranks for quick local checks. - **Conforming meshes only**, with one element geometry and one boundary geometry (segment, triangle, quadrilateral, tetrahedron or hexahedron). Nonconforming and NURBS meshes are rejected. -- **Not yet tested on GPUs.** The restored data is written on the host and marked valid there, - so it should be copied to the device on first use, but no GPU run has been tested yet. +- **GPU testing so far is CUDA only.** The restored data is written on the host and marked + valid there, so it is copied to the device on first use. `test_restart_runs_gpu` passes + bitwise with PA and EA on an NVIDIA GPU (CUDA); it has not been run on an AMD GPU (HIP) yet. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c5e53e2c..d1469d8a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -65,14 +65,19 @@ if(ENABLE_AXOM) # MPI tests never use more ranks than the machine has cores, so the suite does not # oversubscribe small machines such as the 4-core CI runners (Open MPI refuses to start more - # ranks than slots by default). The default is the number of logical cores CMake detects; - # set EXACONSTIT_TEST_MAX_MPI_TASKS to override it. Rank counts above the cap are skipped. - cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_LOGICAL_CORES) + # ranks than slots by default). Open MPI counts physical cores as slots, not hardware + # threads, so the default is the number of physical cores CMake detects (falling back to + # logical cores if that is unknown); set EXACONSTIT_TEST_MAX_MPI_TASKS to override it. Rank + # counts above the cap are skipped. + cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_PHYSICAL_CORES) + if(NOT _detected_cores OR _detected_cores LESS 1) + cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_LOGICAL_CORES) + endif() if(NOT _detected_cores OR _detected_cores LESS 1) set(_detected_cores 1) endif() set(EXACONSTIT_TEST_MAX_MPI_TASKS ${_detected_cores} CACHE STRING - "Largest number of MPI ranks any test may use (default: detected logical cores)") + "Largest number of MPI ranks any test may use (default: detected physical cores)") foreach(_np 1 2 3 4) if(NOT _np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) @@ -153,7 +158,32 @@ if(ENABLE_AXOM) set_tests_properties(test_restart_runs PROPERTIES LABELS "restart;long" PROCESSORS ${_restart_runs_np} + RESOURCE_LOCK restart_runs TIMEOUT 3600) + + # GPU builds also run the same tests on the device. Restored data is written on the + # host, so this checks that it reaches the device, and that GPU runs are bitwise + # reproducible across restarts. EA is the default since it is the fastest GPU assembly + # for these runs. + if(ENABLE_CUDA OR ENABLE_HIP) + set(EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY "EA" CACHE STRING + "Assembly used by test_restart_runs_gpu (FULL, PA or EA)") + add_test(NAME test_restart_runs_gpu + COMMAND ${PYTHON_EXECUTABLE} test_restart_runs.py + --np ${_restart_runs_np} + --rtmodel GPU + --assembly ${EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY} + --mpiexec ${_restart_runs_mpiexec} + --exe $ + --compare-tool $ + --workdir ${CMAKE_BINARY_DIR}/test/restart_runs_gpu + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_runs_gpu PROPERTIES + LABELS "restart;long;gpu" + PROCESSORS ${_restart_runs_np} + RESOURCE_LOCK restart_runs + TIMEOUT 7200) + endif() endif() endif() diff --git a/test/test_restart_runs.py b/test/test_restart_runs.py index 527d52ac..f46bd36b 100644 --- a/test/test_restart_runs.py +++ b/test/test_restart_runs.py @@ -171,7 +171,8 @@ def restart_block(restart: Dict[str, object]) -> str: def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: str, - t_final: float, restart: Dict[str, object]) -> None: + t_final: float, restart: Dict[str, object], rtmodel: str = "CPU", + assembly: str = "FULL") -> None: """ Write the options file for one run @@ -187,6 +188,8 @@ def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: bcs: [BCs] table from bc_block() t_final: Final simulation time of this run restart: [Restart] keys and values + rtmodel: [Solvers] rtmodel (CPU, OPENMP or GPU) + assembly: [Solvers] assembly (FULL, PA or EA) """ path.write_text(f"""Version = "0.10.0" basename = "{basename}" @@ -235,8 +238,8 @@ def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: output_directory = "./results" [Solvers] - assembly = "FULL" - rtmodel = "CPU" + assembly = "{assembly}" + rtmodel = "{rtmodel}" [Solvers.NR] iter = 25 rel_tol = 5e-5 @@ -500,7 +503,8 @@ def run_reference(h: Harness, name: str, mesh: MeshSpec, bcs: str, total_steps: write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, time_of(total_steps), {"write": True, "interval": interval, "write_final": True, - "keep_last": h.args.keep_last, "directory": f"ckpt_{name}"}) + "keep_last": h.args.keep_last, "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(name, h.workdir / f"{name}.toml") ref_cycles = sorted(int(p.name.split("_")[1]) for p in (h.workdir / f"ckpt_{name}").glob("cycle_*") @@ -535,7 +539,7 @@ def run_chain(h: Harness, ref: str, ref_cycles: List[int], name: str, mesh: Mesh if i > 0: restart["read"] = True write_toml(h.workdir / f"{segment}.toml", name, mesh, h.args.num_grains, bcs, - time_of(stop), restart) + time_of(stop), restart, rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(segment, h.workdir / f"{segment}.toml") if i > 0: h.verify_restart_log(segment, f"ckpt_{name}", stops[i - 1], stop) @@ -575,7 +579,8 @@ def run_from_reference(h: Harness, ref: str, ref_cycles: List[int], from_cycles: time_of(total_steps), {"write": True, "interval": 0, "write_final": True, "keep_last": 0, "read": True, "read_cycle": cycle, "read_directory": f"ckpt_{ref}", - "directory": f"ckpt_{name}"}) + "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(name, h.workdir / f"{name}.toml") h.verify_restart_log(name, f"ckpt_{ref}", cycle, total_steps) h.compare_checkpoints(f"{name} final cycle {total_steps}", @@ -630,7 +635,8 @@ def run_negative_control(h: Harness, ref: str, ref_cycles: List[int], mesh: Mesh time_of(total_steps), {"write": True, "interval": 0, "write_final": True, "keep_last": 0, "read": True, "read_cycle": cycle, "read_directory": source.name, - "directory": f"ckpt_{name}"}) + "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(name, h.workdir / f"{name}.toml") h.verify_restart_log(name, source.name, cycle, total_steps) h.compare_checkpoints(f"{name} (negative control) final cycle {total_steps}", @@ -764,6 +770,10 @@ def main() -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--np", type=int, default=8, help="MPI ranks (default 8)") + p.add_argument("--rtmodel", default="CPU", choices=["CPU", "OPENMP", "GPU"], + help="[Solvers] rtmodel of every run (default CPU)") + p.add_argument("--assembly", default="FULL", choices=["FULL", "PA", "EA"], + help="[Solvers] assembly of every run (default FULL)") p.add_argument("--mpiexec", default="mpirun", help="MPI launcher (default mpirun)") p.add_argument("--always-mpiexec", action="store_true", help="use the launcher even with --np 1 (default: run the binary directly)") @@ -832,7 +842,8 @@ def main() -> int: prepare_inputs(workdir, find_data_dir(args.data_dir), list(meshes.values()), args.num_grains, args.seed) h = Harness(args, workdir) - print(f"Restart run tests: {args.np} rank(s), workdir {workdir}") + print(f"Restart run tests: {args.np} rank(s), rtmodel {args.rtmodel}, assembly " + f"{args.assembly}, workdir {workdir}") for case in requested: run_case, mesh = cases[case] run_case(h, mesh) From 9f8bc24b3ae84f3effac53f6738d11b02ccc2511 Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Sun, 1 Nov 2026 17:21:43 -0800 Subject: [PATCH 11/12] [Claude] Write checkpoints inside the run's output folder Checkpoints went to the [Restart] directory relative to the working directory, while every other output of a run goes to /. A relative directory is now taken inside that output folder (by default ./results//restart), so a run's checkpoints sit next to its logs, volume averages and visualizations, and a restart with the same output_directory and basename finds them there. read_directory names checkpoints written elsewhere, often by another run, so a relative read_directory stays relative to the working directory. The check that a restart's checkpoint directory exists ran in RestartOptions::validate(), before the directory could be resolved; it now runs in ExaOptions::parse_options() right after. The long restart tests look for each run's checkpoints in its output folder and accept the full path the restart log now prints. --- src/options.toml | 13 +++++++-- src/options/option_parser_v2.cpp | 14 +++++++++ src/options/option_parser_v2.hpp | 8 ++++-- src/options/option_restart.cpp | 17 ++--------- test/test_restart_runs.py | 49 +++++++++++++++++++++++--------- 5 files changed, 67 insertions(+), 34 deletions(-) diff --git a/src/options.toml b/src/options.toml index e1a92a80..2b47b330 100644 --- a/src/options.toml +++ b/src/options.toml @@ -635,6 +635,7 @@ grain_file = "grains.txt" # [output_directory]/[basename]/ <- Main simulation folder # [output_directory]/[basename]/avg_*.txt <- Volume average files # [output_directory]/[basename]/visualizations/ <- Visualization files + # [output_directory]/[basename]/restart/ <- Checkpoints ([Restart]) # # Example: If output_directory = "./results" and basename = "tensile_test": # ./results/tensile_test/ <- All outputs go here @@ -925,7 +926,11 @@ grain_file = "grains.txt" keep_last = 2 # Directory checkpoints are written to, one subdirectory per checkpoint - # (cycle_) + # (cycle_). A relative path is inside the run's + # output folder, [output_directory]/[basename] (see [PostProcessing.volume_averages] + # output_directory), next to the other outputs: by default + # ./results//restart. A restart with the same output_directory and + # basename finds its checkpoints there. directory = "restart" # Restart this run from a checkpoint @@ -937,5 +942,7 @@ grain_file = "grains.txt" # Read the checkpoint from this directory instead of `directory` # Useful for restarting from another run's checkpoints without touching - # them; this run still writes its own checkpoints to `directory` - # read_directory = "other_run/restart" + # them; this run still writes its own checkpoints to `directory`. A relative + # path is relative to the directory ExaConstit is started in (not the output + # folder), e.g. another run's checkpoints: + # read_directory = "results/other_run/restart" diff --git a/src/options/option_parser_v2.cpp b/src/options/option_parser_v2.cpp index d8dee223..e301e5a9 100644 --- a/src/options/option_parser_v2.cpp +++ b/src/options/option_parser_v2.cpp @@ -203,6 +203,20 @@ void ExaOptions::parse_options(const std::string& filename, int my_id) { resolved_str.resize(static_cast(length)); MPI_Bcast(resolved_str.data(), length, MPI_CHAR, 0, MPI_COMM_WORLD); output_dir = resolved_str; + + // Checkpoints are part of this run's output: a relative [Restart] directory is taken inside + // the run's output folder, /, next to its logs, volume averages + // and visualizations. read_directory names checkpoints written elsewhere (often by another + // run), so a relative read_directory stays relative to the working directory. + if (restart.directory.is_relative()) { + restart.directory = fs::path(output_dir) / basename / restart.directory; + } + // Checked here rather than in RestartOptions::validate(), which runs before the directory + // is resolved. Which checkpoint inside it is used, and whether it is complete, is decided + // when the simulation starts. + VERIFY_0_OPT(!restart.read || fs::is_directory(restart.GetReadDirectory()), + "Restart table has `read` = true, but the checkpoint directory '" + << restart.GetReadDirectory().string() << "' does not exist."); } void ExaOptions::parse_from_toml(const toml::value& toml_input) { diff --git a/src/options/option_parser_v2.hpp b/src/options/option_parser_v2.hpp index f35182e8..772efed6 100644 --- a/src/options/option_parser_v2.hpp +++ b/src/options/option_parser_v2.hpp @@ -1385,6 +1385,9 @@ struct RestartOptions { /** * @brief Directory checkpoints are written to (and read from unless read_directory is set) + * + * @details A relative path is taken inside the run's output folder, + * / (ExaOptions::parse_options() replaces it by the full path). */ std::filesystem::path directory = "restart"; @@ -1392,8 +1395,9 @@ struct RestartOptions { * @brief Directory to read the restart checkpoint from instead of directory * * @details Lets a continuation restart from another run's checkpoints while writing its own - * to directory. Writing, superseding and keep_last pruning only ever act on directory, so - * the other run's checkpoints are never renamed or removed. + * to directory. A relative path is relative to the working directory. Writing, superseding and + * keep_last pruning only ever act on directory, so the other run's checkpoints are never + * renamed or removed. */ std::optional read_directory; diff --git a/src/options/option_restart.cpp b/src/options/option_restart.cpp index 7196fd48..e91b8115 100644 --- a/src/options/option_restart.cpp +++ b/src/options/option_restart.cpp @@ -1,11 +1,6 @@ #include "options/option_parser_v2.hpp" #include "options/option_util.hpp" -#include -#include - -namespace fs = std::filesystem; - RestartOptions RestartOptions::from_toml(const toml::value& toml_input) { RestartOptions options; @@ -78,16 +73,8 @@ bool RestartOptions::validate() const { return false; } - // Checking the directory here gives a clear message before any setup work is done. Which - // checkpoint inside it is used, and whether it is complete, is decided when the - // simulation starts. - if (read && !fs::is_directory(GetReadDirectory())) { - std::ostringstream err; - err << "Error: Restart table has `read` = true, but the checkpoint directory '" - << GetReadDirectory().string() << "' does not exist."; - WARNING_0_OPT(err.str()); - return false; - } + // Whether the checkpoint directory exists is checked in ExaOptions::parse_options(), once a + // relative `directory` has been placed inside the output folder. // The remaining cases are allowed but probably not what was intended. if (read_directory.has_value() && !read) { diff --git a/test/test_restart_runs.py b/test/test_restart_runs.py index f46bd36b..3c9559a7 100644 --- a/test/test_restart_runs.py +++ b/test/test_restart_runs.py @@ -45,6 +45,7 @@ import argparse import random +import re import shutil import subprocess import sys @@ -275,6 +276,22 @@ def time_of(steps: int) -> float: return round(steps * DT, 10) +def checkpoint_root(name: str) -> str: + """ + Where a run writes its checkpoints, relative to the work directory + + Every run here has basename name and [Restart] directory = "ckpt_", which ExaConstit + places inside the run's output folder, results/. + + Args: + name: Run name (its basename) + + Returns: + results//ckpt_ + """ + return f"results/{name}/ckpt_{name}" + + # ============================================================================ # RUNNING AND COMPARING # ============================================================================ @@ -350,13 +367,16 @@ def verify_restart_log(self, name: str, read_dir: str, cycle: int, end_step: int Args: name: Run name (its log is checked) read_dir: Checkpoint directory the run should have read, relative to the work - directory + directory (a full path in the log ending in it also matches) cycle: Cycle of the checkpoint it should have restarted from end_step: Last step it should have run """ text = (self.logdir / f"{name}.log").read_text() - expected = f"Restarting from checkpoint {read_dir}/cycle_{cycle:010d} (cycle {cycle}," - restarted = expected in text and "Restored checkpoint state" in text + # The log prints the directory as ExaConstit resolved it: as given for read_directory, + # the full path for a directory inside the output folder + expected = re.compile(r"Restarting from checkpoint (\S*/)?" + re.escape(read_dir) + + rf"/cycle_{cycle:010d} \(cycle {cycle},") + restarted = bool(expected.search(text)) and "Restored checkpoint state" in text steps = [int(line.split(":")[1]) for line in text.splitlines() if line.startswith("Simulation cycle:")] ran_remaining = steps == list(range(cycle + 1, end_step + 1)) @@ -507,7 +527,7 @@ def run_reference(h: Harness, name: str, mesh: MeshSpec, bcs: str, total_steps: rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(name, h.workdir / f"{name}.toml") ref_cycles = sorted(int(p.name.split("_")[1]) - for p in (h.workdir / f"ckpt_{name}").glob("cycle_*") + for p in (h.workdir / checkpoint_root(name)).glob("cycle_*") if (p / "commit.json").exists()) print(f" reference checkpoints kept: {ref_cycles}") return ref_cycles @@ -542,11 +562,12 @@ def run_chain(h: Harness, ref: str, ref_cycles: List[int], name: str, mesh: Mesh time_of(stop), restart, rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(segment, h.workdir / f"{segment}.toml") if i > 0: - h.verify_restart_log(segment, f"ckpt_{name}", stops[i - 1], stop) + h.verify_restart_log(segment, checkpoint_root(name), stops[i - 1], stop) for stop in stops: if stop in ref_cycles: - h.compare_checkpoints(f"{name} cycle {stop}", h.checkpoint(f"ckpt_{ref}", stop), - h.checkpoint(f"ckpt_{name}", stop)) + h.compare_checkpoints(f"{name} cycle {stop}", + h.checkpoint(checkpoint_root(ref), stop), + h.checkpoint(checkpoint_root(name), stop)) h.compare_outputs(name, ref, name) @@ -578,14 +599,14 @@ def run_from_reference(h: Harness, ref: str, ref_cycles: List[int], from_cycles: write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, time_of(total_steps), {"write": True, "interval": 0, "write_final": True, "keep_last": 0, - "read": True, "read_cycle": cycle, "read_directory": f"ckpt_{ref}", + "read": True, "read_cycle": cycle, "read_directory": checkpoint_root(ref), "directory": f"ckpt_{name}"}, rtmodel=h.args.rtmodel, assembly=h.args.assembly) h.run(name, h.workdir / f"{name}.toml") - h.verify_restart_log(name, f"ckpt_{ref}", cycle, total_steps) + h.verify_restart_log(name, checkpoint_root(ref), cycle, total_steps) h.compare_checkpoints(f"{name} final cycle {total_steps}", - h.checkpoint(f"ckpt_{ref}", total_steps), - h.checkpoint(f"ckpt_{name}", total_steps)) + h.checkpoint(checkpoint_root(ref), total_steps), + h.checkpoint(checkpoint_root(name), total_steps)) h.compare_outputs(name, ref, name, after_time=time_of(cycle)) @@ -620,7 +641,7 @@ def run_negative_control(h: Harness, ref: str, ref_cycles: List[int], mesh: Mesh cycle = candidates[-1] # the latest one keeps this extra run short name = f"{ref}_perturbed{cycle}" source = h.workdir / f"ckpt_{name}_src" - shutil.copytree(h.checkpoint(f"ckpt_{ref}", cycle), source / f"cycle_{cycle:010d}") + shutil.copytree(h.checkpoint(checkpoint_root(ref), cycle), source / f"cycle_{cycle:010d}") with h5py.File(source / f"cycle_{cycle:010d}" / "rank_000000.hdf5", "r+") as f: paths = [f"sidre/external/qfuncs/{g}/values" for g in f["sidre/external/qfuncs"] if g.startswith("state_var_beg")] @@ -640,8 +661,8 @@ def run_negative_control(h: Harness, ref: str, ref_cycles: List[int], mesh: Mesh h.run(name, h.workdir / f"{name}.toml") h.verify_restart_log(name, source.name, cycle, total_steps) h.compare_checkpoints(f"{name} (negative control) final cycle {total_steps}", - h.checkpoint(f"ckpt_{ref}", total_steps), - h.checkpoint(f"ckpt_{name}", total_steps), expect_differ=True) + h.checkpoint(checkpoint_root(ref), total_steps), + h.checkpoint(checkpoint_root(name), total_steps), expect_differ=True) h.compare_outputs(f"{name} (negative control)", ref, name, after_time=time_of(cycle), expect_differ=True) From e564d1faa04b94790a79c10c62cb65b97c6e1bff Mon Sep 17 00:00:00 2001 From: Robert Carson Date: Sun, 1 Nov 2026 17:21:43 -0800 Subject: [PATCH 12/12] [Claude] Keep the visualization files written before a restart A restarted run saved cycle 0 again at t = 0 when its data collections were created, rewriting the first run's step-0 files, and the ParaView collection file (.pvd) was started anew, so ParaView no longer showed the steps before the restart. A restart now skips the cycle-0 save, and ParaView collections use MFEM's restart mode, which keeps the .pvd entries before the current time. VisIt writes one root file per cycle, so its earlier cycles stay listed. MFEM's restart mode compares the current time with the times printed in the .pvd file, so when a run restarts from an earlier checkpoint, the entry of the step it computes again can look earlier than the current time, and MFEM stops the run rather than overwrite that cycle. Before the first save of a restart, rank 0 therefore removes the .pvd entries from that cycle on, by cycle number. test_restart_outputs (a short "outputs" case of test_restart_runs.py, in the default suite) checks that checkpoints are written inside the output folder and that, after a restart and after a restart from an earlier checkpoint, the first run's step-0 files are unchanged, every .pvd lists each step once, and the visualization files are byte for byte those of an uninterrupted run. --- src/postprocessing/postprocessing_driver.cpp | 78 +++++++++- src/postprocessing/postprocessing_driver.hpp | 12 ++ test/CMakeLists.txt | 26 ++++ test/test_restart_runs.py | 141 ++++++++++++++++++- 4 files changed, 248 insertions(+), 9 deletions(-) diff --git a/src/postprocessing/postprocessing_driver.cpp b/src/postprocessing/postprocessing_driver.cpp index 10f9d838..d70cec3c 100644 --- a/src/postprocessing/postprocessing_driver.cpp +++ b/src/postprocessing/postprocessing_driver.cpp @@ -12,6 +12,9 @@ #include "SNLS_linalg.h" #include +#include +#include +#include namespace fs = std::filesystem; namespace { @@ -1512,6 +1515,12 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { paraview.SetLevelsOfDetail(options.mesh.order); paraview.SetDataFormat(mfem::VTKFormat::BINARY); paraview.SetHighOrderOutput(false); + // Restart: keep the .pvd entries written before the restart time + paraview.UseRestartMode(m_sim_state->IsRestartRun()); + if (m_sim_state->IsRestartRun()) { + m_restart_pvd_files.push_back(output_dir_vizs / + (output_dir_vizs.filename().string() + ".pvd")); + } dcs_keys.push_back(key); } #ifdef MFEM_USE_ADIOS2 @@ -1534,9 +1543,14 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { dcs->RegisterField(disp_key, value.get()); } } - dcs->SetCycle(0); - dcs->SetTime(0.0); - dcs->Save(); + // The initial state, written once by the run that starts at t = 0. A restart + // keeps the files written before it and adds to them from its first output + // step on, so it must not overwrite cycle 0. + if (!m_sim_state->IsRestartRun()) { + dcs->SetCycle(0); + dcs->SetTime(0.0); + dcs->Save(); + } } } } @@ -1572,6 +1586,12 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { paraview.SetLevelsOfDetail(options.mesh.order); paraview.SetDataFormat(mfem::VTKFormat::BINARY); paraview.SetHighOrderOutput(false); + // Restart: keep the .pvd entries written before the restart time + paraview.UseRestartMode(m_sim_state->IsRestartRun()); + if (m_sim_state->IsRestartRun()) { + m_restart_pvd_files.push_back(output_dir_vizs / + (output_dir_vizs.filename().string() + ".pvd")); + } dcs_keys.push_back(key); } #ifdef MFEM_USE_ADIOS2 @@ -1595,14 +1615,60 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { dcs->RegisterField(disp_key, value.get()); } } - dcs->SetCycle(0); - dcs->SetTime(0.0); - dcs->Save(); + // The initial state, written only by the run that starts at t = 0 (see above) + if (!m_sim_state->IsRestartRun()) { + dcs->SetCycle(0); + dcs->SetTime(0.0); + dcs->Save(); + } } } } +namespace { + +/** + * @brief Remove the entries of cycle first_cycle and later from a ParaView collection file + * + * @param pvd_file The .pvd file (nothing is done if it does not exist) + * @param first_cycle First cycle to remove + * + * Lines other than DataSet entries (the header and closing tags) are kept. MFEM's restart mode + * then appends the new entries after the kept ones. + */ +void TrimParaViewCollection(const fs::path& pvd_file, const int first_cycle) { + std::ifstream in(pvd_file); + if (!in) { + return; + } + static const std::regex cycle_regex("file=\"Cycle(\\d+)"); + std::ostringstream kept; + std::string line; + std::smatch match; + while (std::getline(in, line)) { + if (std::regex_search(line, match, cycle_regex) && std::stoi(match[1]) >= first_cycle) { + continue; + } + kept << line << "\n"; + } + in.close(); + std::ofstream out(pvd_file, std::ios::trunc); + out << kept.str(); +} + +} // namespace + void PostProcessingDriver::UpdateDataCollections(const int step, const double time) { + // First save of a restart run: drop the ParaView entries this run writes again (written by + // rank 0, which alone writes the .pvd files) + if (!m_restart_pvd_files.empty()) { + if (m_mpi_rank == 0) { + for (const auto& pvd_file : m_restart_pvd_files) { + TrimParaViewCollection(pvd_file, step); + } + } + m_restart_pvd_files.clear(); + } for (auto& [tmp, dcs] : m_map_dcs) { dcs->SetCycle(step); dcs->SetTime(time); diff --git a/src/postprocessing/postprocessing_driver.hpp b/src/postprocessing/postprocessing_driver.hpp index 7be26c8f..fcbe4a29 100644 --- a/src/postprocessing/postprocessing_driver.hpp +++ b/src/postprocessing/postprocessing_driver.hpp @@ -965,6 +965,18 @@ class PostProcessingDriver { */ std::map> m_map_dcs; + /** + * @brief ParaView collection files (.pvd) to trim before the first save of a restart run + * + * A restart keeps the visualization files written before it. ParaView's restart mode + * (ParaViewDataCollection::UseRestartMode) keeps the .pvd entries whose time is below the + * current time, but it compares with the times printed in the file, so an entry of the cycle + * being written again (a restart from an earlier checkpoint) can look earlier and stop the + * run. UpdateDataCollections() therefore first removes, by cycle, the entries from the cycle + * it writes on. Empty after that first save, and in runs that are not restarts. + */ + std::vector m_restart_pvd_files; + /** * @brief Registered projection operations * diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d1469d8a..a897a44f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -122,6 +122,32 @@ if(ENABLE_AXOM) OUTPUT_DIR ${TEST_OUTPUT_DIR} DEPENDS_ON ${_restart_tool_depends}) + # Restart outputs (a short case of test_restart_runs.py, part of the default suite): + # checkpoints go inside the run's output folder, and restarts keep the visualization files + # written before them (ParaView and VisIt, also restarting from an earlier checkpoint) + set(_restart_outputs_np 2) + if(_restart_outputs_np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + set(_restart_outputs_np ${EXACONSTIT_TEST_MAX_MPI_TASKS}) + endif() + set(_restart_outputs_mpiexec "${MPIEXEC_EXECUTABLE}") + if(NOT _restart_outputs_mpiexec) + set(_restart_outputs_mpiexec "mpirun") + endif() + add_test(NAME test_restart_outputs + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test_restart_runs.py + --cases outputs + --nxyz 4 + --np ${_restart_outputs_np} + --always-mpiexec + --mpiexec ${_restart_outputs_mpiexec} + --exe $ + --compare-tool $ + --workdir ${CMAKE_BINARY_DIR}/test/restart_outputs + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_outputs PROPERTIES + LABELS "restart" + PROCESSORS ${_restart_outputs_np}) + # Restart tests on real runs (test_restart_runs.py): cyclic loading chained every 1 and 2 # cycles, long monotonic tension chained halfway, a run with empty ranks, restarts from past # reference checkpoints, and a negative control. They take 10-15 minutes on 8 ranks, so they diff --git a/test/test_restart_runs.py b/test/test_restart_runs.py index 3c9559a7..523d8f8f 100644 --- a/test/test_restart_runs.py +++ b/test/test_restart_runs.py @@ -36,6 +36,10 @@ restarted from its halfway checkpoint; the negative control runs here. empty_ranks Short tension run on a 2-element mesh, so with more than 2 ranks every rank but two owns no elements. Covers checkpointing and restarting with empty ranks. + outputs Short tension run with ParaView and VisIt output (not run by default; a short + ctest of its own): checkpoints are written inside the run's output folder, and + a restart (also from an earlier checkpoint) keeps the visualization files written + before it, ending with exactly the files of an uninterrupted run. Defaults: 16^3 = 4096 hex elements, 8 MPI ranks, 4 cycles of +/-1% strain (160 steps), 10% monotonic strain (100 steps), and a 20-step empty-rank run. Run with --help for the options @@ -173,7 +177,7 @@ def restart_block(restart: Dict[str, object]) -> str: def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: str, t_final: float, restart: Dict[str, object], rtmodel: str = "CPU", - assembly: str = "FULL") -> None: + assembly: str = "FULL", visualize: bool = False) -> None: """ Write the options file for one run @@ -191,7 +195,9 @@ def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: restart: [Restart] keys and values rtmodel: [Solvers] rtmodel (CPU, OPENMP or GPU) assembly: [Solvers] assembly (FULL, PA or EA) + visualize: Write ParaView and VisIt files at every step """ + viz = "true" if visualize else "false" path.write_text(f"""Version = "0.10.0" basename = "{basename}" grain_file = "{mesh.grain_file}" @@ -224,9 +230,10 @@ def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: t_final = {t_final!r} [Visualizations] - visit = false + steps = 1 + visit = {viz} conduit = false - paraview = false + paraview = {viz} [PostProcessing] [PostProcessing.volume_averages] @@ -479,6 +486,61 @@ def compare_outputs(self, label: str, ref_basename: str, other_basename: str, self.check(not problems, f"{label}: outputs vs reference ({detail})") + def visualization_files(self, basename: str) -> Dict[str, bytes]: + """ + Every visualization file of a run, by path with the basename replaced by "BASENAME" + + Args: + basename: Basename of the run + + Returns: + {relative path: contents}, with the basename also replaced in the contents (VisIt + root files name their data files) + """ + root = self.workdir / "results" / basename / "visualizations" + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + rel = str(path.relative_to(root)).replace(basename, "BASENAME") + files[rel] = path.read_bytes().replace(basename.encode(), b"BASENAME") + return files + + def compare_visualizations(self, label: str, ref_basename: str, other_basename: str) -> None: + """ + Check that a run has exactly the visualization files of the reference, byte for byte + + Args: + label: Description used in the check message + ref_basename: Basename of the reference run + other_basename: Basename of the run to compare + """ + ref = self.visualization_files(ref_basename) + other = self.visualization_files(other_basename) + missing = sorted(set(ref) - set(other)) + extra = sorted(set(other) - set(ref)) + differ = sorted(k for k in set(ref) & set(other) if ref[k] != other[k]) + problems = [f"{len(v)} {what} (e.g. {v[0]})" for v, what in + ((missing, "missing"), (extra, "extra"), (differ, "differ")) if v] + self.check(bool(ref) and not problems, + f"{label}: visualization files vs reference (" + f"{'; '.join(problems) if problems else f'{len(ref)} files identical'})") + + def paraview_times(self, basename: str) -> Dict[str, List[float]]: + """ + Times listed in each ParaView collection (.pvd) of a run + + Args: + basename: Basename of the run + + Returns: + {collection file relative to the visualizations folder: times in file order} + """ + root = self.workdir / "results" / basename / "visualizations" + return {str(pvd.relative_to(root)): [float(t) for t in + re.findall(r'timestep="([^"]+)"', pvd.read_text())] + for pvd in sorted(root.rglob("*.pvd"))} + + # ============================================================================ # TEST CASE BUILDING BLOCKS # ============================================================================ @@ -777,6 +839,78 @@ def run_empty_ranks(h: Harness, mesh: MeshSpec) -> None: run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) +def run_outputs(h: Harness, mesh: MeshSpec) -> None: + """ + Output case: where checkpoints go, and visualization files across restarts + + A short tension run with ParaView and VisIt output at every step. The reference runs + uninterrupted, writing checkpoints. A chain stops after step 2 and restarts to the end, then + restarts once more from its checkpoint of step 1 (an earlier one), as when a run is redone + from an older checkpoint. Checks: + - checkpoints are written inside the run's output folder, results//, + and nowhere in the work directory; + - a restart keeps the visualization files written before it: the first segment's step 0 is + not rewritten, and after each restart the visualization files (ParaView .pvd collections, + VTU pieces, VisIt root and data files) are byte for byte those of the uninterrupted run, + with every step listed once in each .pvd file. + + Args: + h: Test harness + mesh: Mesh of the case + """ + total_steps = 4 + stop = 2 + bcs = bc_block([1], [TOP_VELOCITY]) + print(f"\n== outputs: {total_steps} steps with ParaView and VisIt output, chain stopped at " + f"step {stop}, then restarted again from step 1") + + ref = "out_ref" + write_toml(h.workdir / f"{ref}.toml", ref, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 1, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{ref}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly, visualize=True) + h.run(ref, h.workdir / f"{ref}.toml") + inside = h.workdir / checkpoint_root(ref) / f"cycle_{total_steps:010d}" / "commit.json" + h.check(inside.exists() and not (h.workdir / f"ckpt_{ref}").exists(), + f"{ref}: checkpoints are in {checkpoint_root(ref)} (inside the output folder), " + f"not in the work directory") + expected_times = [time_of(step) for step in range(total_steps + 1)] + ref_times = h.paraview_times(ref) + h.check(bool(ref_times) and all(t == expected_times for t in ref_times.values()), + f"{ref}: every .pvd lists steps 0..{total_steps} ({ref_times})") + + chain = "out_chain" + segments = [(f"{chain}_seg1", stop, {}), + (f"{chain}_seg2", total_steps, {"read": True}), + (f"{chain}_seg3", total_steps, {"read": True, "read_cycle": 1})] + step0 = None + for i, (segment, end, read) in enumerate(segments): + restart = {"write": True, "interval": 1, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{chain}", **read} + write_toml(h.workdir / f"{segment}.toml", chain, mesh, h.args.num_grains, bcs, + time_of(end), restart, rtmodel=h.args.rtmodel, assembly=h.args.assembly, + visualize=True) + h.run(segment, h.workdir / f"{segment}.toml") + files = h.visualization_files(chain) + if i == 0: + step0 = {k: v for k, v in files.items() if "Cycle000000" in k or "_000000" in k} + continue + h.verify_restart_log(segment, checkpoint_root(chain), read.get("read_cycle", stop), end) + kept = all(files.get(k) == v for k, v in step0.items()) + h.check(bool(step0) and kept, + f"{segment}: the {len(step0)} step-0 visualization files of the first segment " + f"are still there, unchanged") + times = h.paraview_times(chain) + h.check(bool(times) and all(t == expected_times for t in times.values()), + f"{segment}: every .pvd lists steps 0..{total_steps} once ({times})") + h.compare_visualizations(segment, ref, chain) + if i == 1: + # Volume-average files are only appended to, so they are compared before segment 3 + # computes steps 2..4 again + h.compare_outputs(segment, ref, chain) + + # ============================================================================ # MAIN # ============================================================================ @@ -848,6 +982,7 @@ def main() -> int: "cyclic": (run_cyclic, MeshSpec((args.nxyz,) * 3, "grains.txt")), "monotonic": (run_monotonic, MeshSpec((args.nxyz,) * 3, "grains.txt")), "empty_ranks": (run_empty_ranks, MeshSpec((2, 1, 1), "grains_empty_ranks.txt")), + "outputs": (run_outputs, MeshSpec((args.nxyz,) * 3, "grains.txt")), } requested = [c.strip() for c in args.cases.split(",") if c.strip()] unknown = [c for c in requested if c not in cases]