diff --git a/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake b/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake index bf0df071..275bdb11 100644 --- a/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake +++ b/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake @@ -9,6 +9,7 @@ set(_tpls exacmech mfem caliper + axom threads) foreach(_tpl ${_tpls}) @@ -122,6 +123,43 @@ if(SNLS_USE_RAJA_PORT_SUITE) endif() endif() # End SNLS_USE_RAJA_PORT_SUITE check +################################ +# Axom (optional) +################################ +# Axom installs a proper CMake package config (axom-config.cmake under +# ${AXOM_DIR}/lib/cmake/axom). find_package CONFIG mode picks it up +# automatically and imports the roll-up `axom` target plus per-component +# targets (axom::core, axom::sidre, axom::slic, ...). We consume the +# roll-up target so whatever components Axom was built with come along +# transitively -- including sidre and its Conduit / HDF5 dependencies, +# which axom-config.cmake locates from the paths Axom was built against. + +if (DEFINED AXOM_DIR) + find_package(axom REQUIRED CONFIG + NO_DEFAULT_PATH + PATHS ${AXOM_DIR}) + if (axom_FOUND) + # ---- Workaround for upstream Axom export bug ---- + # axom::slic's INTERFACE_LINK_LIBRARIES contains a bare 'lumberjack' + # entry inherited from BLT's internal target tracking when Axom is + # built with AXOM_ENABLE_LUMBERJACK=ON. Lumberjack is not in + # AXOM_COMPONENTS_ENABLED (it's a feature folded into slic, not a + # component built as its own library), so the reference is dangling. + # Without a stub here, every consumer of axom::slic gets -llumberjack + # on its link line and the linker fails to find it. + if (NOT TARGET lumberjack) + add_library(lumberjack INTERFACE IMPORTED) + endif() + option(ENABLE_AXOM "Enable Axom" ON) + message(STATUS "Found Axom: ${AXOM_DIR}") + else() + message(FATAL_ERROR "Unable to find Axom with given path ${AXOM_DIR}") + endif() +else() + message(STATUS "Axom support disabled") +endif() + + ################################ # Caliper ################################ diff --git a/developers_guide.md b/developers_guide.md index df334298..9e08ac26 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -67,6 +67,7 @@ For detailed installation instructions, refer to the build scripts in `scripts/i **Optional Dependencies:** - **ADIOS2**: (MFEM-based) High-performance I/O for visualization - **Caliper**: Performance profiling +- **Axom** (with Sidre, Conduit and HDF5): Checkpoint/restart support, enabled with `-DAXOM_DIR=/lib/cmake` (see `doc/restart.md`) ### Basic Build Process ```bash @@ -218,6 +219,7 @@ The `src/` directory contains the core ExaConstit implementation organized into - **`option_solvers.cpp`**: Linear and nonlinear solver settings - **`option_time.cpp`**: Time-stepping parameters - **`option_post_processing.cpp`**: Post-processing configuration +- **`option_restart.cpp`**: Checkpoint/restart configuration - **`option_enum.cpp`**: Enumeration type conversions - **`option_util.hpp`**: Utility functions for option parsing @@ -242,6 +244,8 @@ The `src/` directory contains the core ExaConstit implementation organized into #### `sim_state/` **Purpose**: Simulation state management and field storage - **`simulation_state.hpp/cpp`**: Central state container class +- **`restart_checkpoint.hpp/cpp`**: Writing checkpoints and restarting from them (requires Axom; see `doc/restart.md`) +- **`exa_sidre_parmesh.hpp/cpp`**: Writing and rebuilding the parallel mesh for checkpoints **Manages**: - Finite element spaces and mesh data @@ -594,6 +598,17 @@ cd test/data mpirun -np 4 ../../build/mechanics -opt example.toml ``` +### Long Tests +Some tests take too long for CI and are only registered when configured with +`-DEXACONSTIT_ENABLE_LONG_TESTS=ON`. They currently cover the checkpoint/restart code on real +simulations; anyone changing that code must run them locally and make sure they pass: +```bash +ctest -L long --output-on-failure +``` +GPU builds also register a GPU version of them (`test_restart_runs_gpu`; `ctest -L gpu` runs +only that one). No test uses more MPI ranks than `EXACONSTIT_TEST_MAX_MPI_TASKS`, which defaults +to the number of physical cores detected at configure time (Open MPI's default number of slots). + ### Example Workflows The `test/data/` directory contains various example cases: - **Crystal plasticity simulations** @@ -615,14 +630,51 @@ The `test/data/` directory contains various example cases: 1. Create header/source in `models/mechanics_newmodel.hpp/cpp` 2. Inherit from `ExaModel` base class 3. Implement required virtual methods -4. Add configuration parsing support -5. Update `CMakeLists.txt` +4. Register every per-point quantity that carries history between steps with `SimulationState::AddQuadratureFunction()` (see [Keeping New State Restartable](#keeping-new-state-restartable)) +5. Add configuration parsing support +6. Update `CMakeLists.txt` #### New Boundary Condition Type 1. Extend `BCManager` class 2. Add parsing support in `option_boundary_conditions.cpp` 3. Update documentation and examples +#### Keeping New State Restartable +Checkpoints (`src/sim_state/restart_checkpoint.*`, described in `doc/restart.md`) save some +state automatically and some only because it is listed explicitly. Any new feature that +carries state from one step to the next must fit one of the cases below, or a restarted run +will silently continue from the wrong state. + +**Quadrature point data is saved automatically if it is registered.** Every function in +SimulationState's quadrature-function registry is written and restored, whichever component +registered it; the only exception is `tangent_stiffness`, which is recomputed before use. +- Create history data with `SimulationState::AddQuadratureFunction()`. Data kept in a model's + own `mfem::Vector` or `PartialQuadratureFunction` is not saved, and nothing warns about it. + Scratch data recomputed every step can stay private. +- Register it while the objects are being constructed (SimulationState, SystemDriver, the + models, PostProcessingDriver). The driver restores the checkpoint after all of them exist; + anything registered later keeps its initial value. +- Register beginning/end-of-step pairs with `AddUpdateVariablePairNames()` as usual; the + checkpoint then stores an identical end-of-step function as a reference to its partner. +- Data computed once in a constructor is fine if it depends only on the reference + configuration, the mesh and the options, since a restarted run reproduces those exactly. + +**Nodal data is not automatic.** Only the reference and current coordinates and the velocity +are saved; the other SimulationState grid functions are derived from them after a restart. A +new grid function or solution vector that carries history (an additional primal variable, +Lagrange multipliers, and so on) must be added explicitly to `WriteNodalFields()` and to the +binding in `BindFieldsAndQuadratureFunctions()`, with a check that the rebuilt space has the +same size. Store true DOFs where possible. + +**Only the volume mesh is saved.** Submeshes, surface meshes and other topologies built on top +of it must be rebuilt in the restarted run from the restored mesh, which is identical to the +original, local numbering included. Anything that cannot be rebuilt that way needs its own +checkpoint data. + +**Test it.** `test/test_restart_e2e.cpp` compares a restarted SimulationState with the +original bitwise; add any new state to it, and run the [long tests](#long-tests) for anything +that changes what a simulation evolves. + ### Performance Considerations - **GPU kernels**: Use RAJA for performance portability - **Memory management**: Follow MFEM memory patterns diff --git a/doc/install.md b/doc/install.md index 387ea633..e77ab408 100644 --- a/doc/install.md +++ b/doc/install.md @@ -86,7 +86,11 @@ scripts/install/ ├── common/ │ ├── dependency_versions.sh # Centralized version control │ ├── preflight_checks.sh # Validation and utilities -│ └── build_functions.sh # Shared build logic +│ ├── build_functions.sh # Loader that sources the files below +│ ├── build_helpers.sh # Logging, cloning, build-dir helpers +│ ├── build_functions_common.sh # BLT, CAMP, RAJA, Umpire, CHAI +│ ├── build_functions_mfem.sh # Hypre, METIS, MFEM +│ └── build_functions_exaconstit.sh # SNLS, ExaCMech, HDF5, Conduit, Axom, ExaConstit ├── configs/ │ ├── cpu_intel_config.sh # Intel compiler configuration │ ├── cpu_mac_config.sh # macOS configuration @@ -326,12 +330,23 @@ All dependency versions are centralized in `common/dependency_versions.sh`: code ExaConstit/scripts/install/common/dependency_versions.sh ``` ```bash +# Build infrastructure (shared by every RADIUSS-stack package via BLT_SOURCE_DIR) +export BLT_VER="v0.7.2" + # Portability libraries export CAMP_VER="v2025.12.0" export RAJA_VER="v2025.12.2" export UMPIRE_VER="v2025.12.0" export CHAI_VER="v2025.12.0" +# Standalone SNLS (batch solvers enabled) +export SNLS_VER="v0.4.4" + +# Restart / checkpoint I/O (Axom Sidre and its dependencies) +export HDF5_VER="hdf5_1.14.6" # Stay on the 1.14 series +export CONDUIT_VER="v0.9.7" +export AXOM_VER="v0.14.0" + # Material models export EXACMECH_REPO="https://github.com/LLNL/ExaCMech.git" export EXACMECH_BRANCH="v0.4.3" # Tag or branch to check out @@ -460,18 +475,31 @@ your_build_directory/ ├── camp/ │ ├── build_${BUILD_SUFFIX}/ # Build artifacts │ └── install_${BUILD_SUFFIX}/ # Installed library +├── blt/ # Shared BLT checkout (no build step) ├── RAJA/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ -├── Umpire/ # GPU builds only +├── Umpire/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── CHAI/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ -├── CHAI/ # GPU builds only +├── SNLS/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ ├── ExaCMech/ │ ├── build_${BUILD_SUFFIX}/ │ └── install_${BUILD_SUFFIX}/ +├── hdf5/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── conduit/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ +├── axom/ +│ ├── build_${BUILD_SUFFIX}/ +│ └── install_${BUILD_SUFFIX}/ ├── hypre/ │ ├── build_${BUILD_SUFFIX}/ │ └── src/hypre_${BUILD_SUFFIX}/ # Installed library @@ -506,10 +534,8 @@ less my_raja_build # View the build log ### **Disk Space Requirements** Typical disk space usage: -- **CPU build**: ~1 GB total -- **GPU build (CUDA/HIP)**: ~2 GB total - - Includes additional Umpire and CHAI libraries - - GPU architectures add to binary sizes +- **CPU build**: a few GB total (HDF5, Conduit, and Axom checkouts make up a large share) +- **GPU build (CUDA/HIP)**: larger still, since GPU architectures add to binary sizes --- @@ -747,16 +773,21 @@ If you prefer to build manually or need more control over the build process: ### **Prerequisites** You'll need to manually build all dependencies first: -1. **CAMP** (v2025.12.0) -2. **RAJA** (v2025.12.2) -3. **Umpire** (v2025.12.0) - GPU builds only -4. **CHAI** (v2025.12.0) - GPU builds only -5. **ExaCMech** (v0.4.3) +1. **BLT** (v0.7.2) - shared by every package below via `-DBLT_SOURCE_DIR` +2. **CAMP** (v2025.12.0) +3. **RAJA** (v2025.12.2) +4. **Umpire** (v2025.12.0) +5. **CHAI** (v2025.12.0) 6. **Hypre** (v3.1.0) 7. **METIS** (5.1.0) 8. **MFEM** (`exaconstit-dev` branch of https://github.com/rcarson3/mfem; v0.10.0 and v0.10.1 were tested with commit `74ded90191cacabfa294b974e7d69d11ea84d364`) +9. **SNLS** (v0.4.4) - built with `USE_BATCH_SOLVERS=ON` +10. **ExaCMech** (v0.4.3) - built against the standalone SNLS via `SNLS_DIR` +11. **HDF5** (1.14.6) - serial, C library only +12. **Conduit** (v0.9.7) - with MPI and HDF5 +13. **Axom** (v0.14.0) - with Sidre (optional for ExaConstit; enabled when `AXOM_DIR` is given) -See `scripts/install/common/build_functions.sh` for the exact build commands and CMake options. +See the `scripts/install/common/build_functions_*.sh` files for the exact build commands and CMake options. ### **Building ExaConstit** @@ -779,11 +810,16 @@ cmake .. \ -DENABLE_OPENMP=OFF \ -DENABLE_FORTRAN=OFF \ -DPYTHON_EXECUTABLE=/usr/bin/python3 \ + -DBLT_SOURCE_DIR=${BLT_DIR} \ -DMFEM_DIR=${MFEM_INSTALL_DIR}/lib/cmake/mfem/ \ -DECMECH_DIR=${EXACMECH_INSTALL_DIR}/ \ - -DSNLS_DIR=${EXACMECH_INSTALL_DIR}/ \ + -DSNLS_DIR=${SNLS_INSTALL_DIR}/lib/cmake/snls/ \ + -DAXOM_DIR=${AXOM_INSTALL_DIR}/lib/cmake/ \ -DRAJA_DIR=${RAJA_INSTALL_DIR}/lib/cmake/raja/ \ -Dcamp_DIR=${CAMP_INSTALL_DIR}/lib/cmake/camp/ \ + -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ + -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ + -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai \ -DCMAKE_BUILD_TYPE=Release # 4. Build @@ -803,10 +839,7 @@ cmake .. \ -DCMAKE_CUDA_COMPILER=${CUDA_ROOT}/bin/nvcc \ -DCMAKE_CUDA_HOST_COMPILER=${HOST_CXX_COMPILER} \ -DCMAKE_CUDA_ARCHITECTURES=80 \ - -DENABLE_CUDA=ON \ - -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ - -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ - -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai + -DENABLE_CUDA=ON ``` For HIP builds, add: @@ -816,10 +849,7 @@ cmake .. \ -DCMAKE_CXX_COMPILER=${ROCM_ROOT}/bin/amdclang++ \ -DCMAKE_HIP_COMPILER=${ROCM_ROOT}/bin/amdclang++ \ -DCMAKE_HIP_ARCHITECTURES=gfx942 \ - -DENABLE_HIP=ON \ - -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \ - -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \ - -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai + -DENABLE_HIP=ON ``` --- diff --git a/doc/restart.md b/doc/restart.md new file mode 100644 index 00000000..81fdffee --- /dev/null +++ b/doc/restart.md @@ -0,0 +1,220 @@ +# Checkpoint and Restart + +ExaConstit can write binary checkpoints during a run and restart a later run from one of them. +A restarted run continues exactly where the checkpoint left off: with the same number of MPI +ranks it produces bitwise identical results to a run that never stopped. + +A restart needs only the checkpoint and the options file. The mesh, grain, region-mapping, +orientation and state-variable files of the original run are not read again and may have been +deleted. + +## Contents +1. [Requirements](#requirements) +2. [Using checkpoints](#using-checkpoints) +3. [What may change between runs](#what-may-change-between-runs) +4. [How it works](#how-it-works) +5. [Checkpoint layout](#checkpoint-layout) +6. [Testing](#testing) +7. [Limitations](#limitations) + +## Requirements + +Checkpoints are written with Axom's Sidre library, one HDF5 file per MPI rank. ExaConstit must +be configured with `-DAXOM_DIR=/lib/cmake`, where Axom was built with Sidre and +its Conduit and HDF5 dependencies. The install scripts in `scripts/install/` build all three +(Axom v0.14.0, Conduit v0.9.7, HDF5 1.14.6). Without Axom, a `[Restart]` table that asks to +write or read checkpoints is rejected when the options are validated. + +## Using checkpoints + +Everything is controlled by the `[Restart]` table of the options file (documented with every +key in `src/options.toml`): + +```toml +[Restart] + write = true # write checkpoints during this run (default false) + interval = 50 # every 50 completed steps; 0 = only the final one (default 0) + write_final = true # also write one when the run finishes (default true) + keep_last = 2 # keep the newest 2 checkpoints; 0 = keep all (default 2) + directory = "restart" # where checkpoints are written (default "restart") + read = false # restart this run from a checkpoint (default false) + read_cycle = 1200 # which checkpoint to read; newest complete one if omitted + read_directory = "other" # read from here instead of `directory` (optional) +``` + +Each checkpoint is a directory `/cycle_/`. To continue a run, set +`read = true`, keep `directory` pointing at the same place, and increase `t_final` (or extend +the custom time steps). The restarted run picks the newest complete checkpoint unless +`read_cycle` names one, and keeps writing checkpoints to `directory` if `write` is set. + +`read_directory` restarts from another run's checkpoints while writing this run's own +checkpoints to `directory`. The other run's checkpoints are never renamed or removed, which is +useful for trying several continuations of one run, and is how the restart tests reuse a +single reference run. + +A checkpoint written at the final time is complete but has nothing left to run; restarting +from it prints a warning unless `t_final` has been increased. + +**Output files.** Volume-average output files are opened in append mode, so a continuation +that keeps the same `basename` appends to the original run's files and they read as one +uninterrupted run. Use a different `basename` to keep the continuation's output separate. + +## What may change between runs + +**Must stay the same** (checked; a mismatch aborts with a message saying what changed): +- the number of MPI ranks, since each rank holds its own piece of the mesh and state; +- the mesh order; +- the material list: the number of materials, their order, names, model types and state + variable counts (including orientations). + +**May change:** +- `t_final` and the time-step settings; +- solver, preconditioner and assembly settings; +- the device (`rtmodel`); +- visualization and post-processing settings. A quadrature function enabled only in the + continuation starts from its initial value, and one dropped by the continuation is skipped; + both are reported as warnings; +- material properties, with a warning; the continuation uses its own values. + +**Boundary conditions** for the steps already completed must stay the same; later entries of +the schedule may change. **Ignored on restart:** `ref_ser`, `ref_par` and the mesh file and +auto-mesh settings, since the mesh comes from the checkpoint. + +## How it works + +The implementation is documented in detail in the code. This section is an overview. + +| File | Contents | +|---|---| +| `src/sim_state/restart_checkpoint.hpp/.cpp` | `exaconstit::RestartCheckpoint`: writing, selecting, validating and restoring checkpoints | +| `src/sim_state/exa_sidre_parmesh.hpp/.cpp` | Writing a rank's piece of an `mfem::ParMesh` and rebuilding the identical `ParMesh` from it | +| `src/sim_state/restart_sidre_utils.hpp` | Small helpers shared by the two | +| `src/options/option_restart.cpp` | The `[Restart]` table | +| `src/sim_state/simulation_state.cpp` | The restart branches of the `SimulationState` constructor and `TimeManagement::ResumeFromCheckpoint()` | +| `src/mechanics_driver.cpp` | Restoring the state and boundary conditions before the time loop, and writing checkpoints in it | + +The background notes at the top of `restart_checkpoint.cpp` and `exa_sidre_parmesh.cpp` are +the place to start before changing anything. + +**Build first, overlay last.** A restarted run constructs `SimulationState`, `SystemDriver` +and `PostProcessingDriver` exactly as a fresh run does, then overwrites the evolving state +(coordinates, velocity, state variables, stresses and time) with the checkpointed values. +Anything a constructor initializes is simply replaced, and any quadrature function a component +registers is restored, without the restart code needing to know about it. + +The one exception is that the `SimulationState` constructor would read input files that may +no longer exist. On restart runs it takes those few things from the checkpoint instead +("phase 1"): the mesh, the reference coordinates, and the grain and region IDs. It also skips +the state-variable and orientation initialization. Everything else is restored from the driver +once every object exists ("phase 2"). + +**The mesh** is rebuilt from what each rank wrote, with the same local numbering, element +vertex order, communication groups, shared entities and shared-entity orientation as the +original, so the finite element spaces number their DOFs identically. The standard +`axom::sidre::MFEMSidreDataCollection` is not used: it records shared edges and faces in each +rank's local orientation, which matches shared DOFs incorrectly as soon as a shared face has +more than one interior DOF (H1 order 3 and up on hexes). It also references the mesh's memory +instead of copying it, and does not hand ownership of what it rebuilds to the caller. + +**Checks before anything is overwritten.** The restarted run verifies the checkpoint's format +version and rank count, the rebuilt mesh topology, the coordinate space sizes, that every DOF +is numbered as in the original (an element-local hash and a shared-DOF round trip of the +reference coordinates), the material identities, and every quadrature space and function's +shape. Problems found on any rank abort every rank together, with each failing rank reporting +its own details. + +**Writing is safe to interrupt.** Each rank writes its own file, and rank 0 writes `commit.json` +only once every rank has succeeded. A checkpoint without it is never read, so a crash during +a write cannot leave a checkpoint that looks complete. A failed write is reported as a warning +and the run continues. + +**Ranks without elements** are fully supported: they write and read empty checkpoint files and +take part in every collective step. + +**Adding new state.** Quadrature point data registered with +`SimulationState::AddQuadratureFunction()` is checkpointed automatically, including data a new +material model registers. Nodal data is not: only the coordinates and velocity are saved, and +a new grid function or solution vector has to be added to the checkpoint explicitly. Only the +volume mesh is saved; submeshes and other topologies must be rebuilt from it. The developer's +guide section "Keeping New State Restartable" lists the rules. + +## Checkpoint layout + +``` +/cycle_/ + rank_.hdf5 one per rank + commit.json written last by rank 0; marks the checkpoint complete +``` + +Each rank file holds: + +| Group | Contents | +|---|---| +| `restart_info/` | format version, rank count, rank, completed steps, driver step, completed time, next, previous and original time steps, time-step type, mesh order, ExaConstit version | +| `mesh/` | the rank's piece of the mesh (Conduit Blueprint layout) and each element's grain ID | +| `fields/` | reference and current coordinates, velocity, and the coordinate space sizes and DOF-layout hash | +| `qspaces/` | validation data for every quadrature space | +| `qfuncs/` | every quadrature function except the tangent stiffness (recomputed before use); an end-of-step function identical to its beginning-of-step partner is stored as `copy_of` instead of repeating its values | +| `materials/` | each region's material name, model type, state variable count and properties | + +Two diagnostic tools are built alongside the tests: `inspect_restart_rank_file` lists how each +quadrature function is stored in a rank file, and `compare_restart_checkpoints` compares two +checkpoints view by view (bitwise by default). + +## Testing + +The restart tests are built when ExaConstit is configured with Axom and tests are enabled. + +**Default suite** (fast; runs with `ctest`, including on CI): +- `test_restart_mesh_roundtrip_np{1..4}`: writes and rebuilds hex and tet meshes, with and + without parallel refinement, and a case where half of the ranks own no elements. Checks the + rebuilt mesh's topology, communication groups, shared-entity order and orientation, and that + H1 spaces of order 1 to 4 have identical DOF layouts. +- `test_restart_e2e_np{2,3}`: builds a real `SimulationState` from input files, checkpoints a + synthetic evolved state, deletes the input files, restarts, and compares everything bitwise. + +No test uses more MPI ranks than the machine has cores: rank counts above +`EXACONSTIT_TEST_MAX_MPI_TASKS` (default: the physical core count detected at configure time, +since Open MPI counts physical cores as slots) are skipped. + +**Long tests** (not on CI; developers changing the restart code must run them locally): +configure with `-DEXACONSTIT_ENABLE_LONG_TESTS=ON`, then run + +```bash +ctest -L long --output-on-failure +``` + +This runs `test/test_restart_runs.py` on real simulations (Voce FCC crystal plasticity on a +4096-element mesh, 8 ranks by default via `EXACONSTIT_RESTART_RUN_TEST_NP`, capped at +`EXACONSTIT_TEST_MAX_MPI_TASKS`), which takes 10 to 15 minutes: +- **cyclic**: four ±1% strain cycles, chained every cycle and every two cycles, and restarted + from each past checkpoint of the uninterrupted reference; +- **monotonic**: 10% tension, chained halfway and restarted from the reference's halfway + checkpoint; +- **empty_ranks**: a 2-element mesh, so all but two ranks own no elements. + +Every checkpoint a restarted run writes is compared bitwise with the reference checkpoint of +the same step, and its volume-average output row by row. Each restarted run's log is checked to +confirm it really restarted from the expected checkpoint and ran only the remaining steps, and +a negative control restarts from a slightly perturbed checkpoint and requires the result to +differ. `python3 test_restart_runs.py --help` lists the options, including a smaller mesh, fewer +steps and fewer ranks for quick local checks, and the device (`--rtmodel`) and assembly +(`--assembly`) of the runs. + +GPU builds also register `test_restart_runs_gpu` (labels `restart;long;gpu`), which runs the same +cases with `rtmodel = "GPU"` and `EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY` (default `EA`, the +fastest GPU assembly for these runs). It checks that the restored host data reaches the device and +that GPU runs restart bitwise identically. Run only the GPU tests with `ctest -L gpu`. + +## Limitations + +- **Same rank count.** Restarting on a different number of ranks would need a + partition-independent layout and redistribution of the state; it is not supported. +- **One file per rank per checkpoint.** Very large runs produce many files. Axom's `IOManager` + can group them later without changing the commit protocol. +- **Conforming meshes only**, with one element geometry and one boundary geometry + (segment, triangle, quadrilateral, tetrahedron or hexahedron). Nonconforming and NURBS meshes + are rejected. +- **GPU testing so far is CUDA only.** The restored data is written on the host and marked + valid there, so it is copied to the device on first use. `test_restart_runs_gpu` passes + bitwise with PA and EA on an NVIDIA GPU (CUDA); it has not been run on an AMD GPU (HIP) yet. diff --git a/scripts/install/common/build_functions.sh b/scripts/install/common/build_functions.sh index 79b2a031..b56ed605 100644 --- a/scripts/install/common/build_functions.sh +++ b/scripts/install/common/build_functions.sh @@ -1,671 +1,27 @@ #!/usr/bin/env bash -# Common build functions for all ExaConstit dependencies - -# Logging wrapper -run_with_log() { - local log="$1"; shift - "$@" |& tee "$log" -} - -# Clone repository only if missing, initialize submodules on first clone -clone_if_missing() { - local repo="$1" branch="$2" dest="$3" - if [ ! -d "$dest/.git" ]; then - echo "Cloning ${dest}..." - git clone --branch "$branch" "$repo" "$dest" - cd "$dest" - if [ -f .gitmodules ]; then - git submodule update --init --recursive - fi - cd "$BASE_DIR" - else - echo "${dest} already exists, skipping clone." - fi -} - -# Optional: force submodule sync when explicitly requested -sync_submodules() { - local dest="$1" - if [ "${SYNC_SUBMODULES}" = "ON" ] && [ -f "$dest/.gitmodules" ]; then - echo "Syncing submodules in ${dest}..." - cd "$dest" - git submodule sync --recursive - git submodule update --init --recursive - cd "$BASE_DIR" - fi -} - -# Respect REBUILD flag when preparing build directories -prepare_build_dir() { - local dir="$1" - if [ "${REBUILD}" = "ON" ]; then - mkdir -p "$dir" - rm -rf "$dir"/* - echo "Cleaned build directory: ${dir}" - else - if [ ! -d "$dir" ]; then - mkdir -p "$dir" - echo "Created build directory: ${dir}" - else - echo "Reusing existing build directory: ${dir}" - fi - fi -} - -########################################### -# CAMP -########################################### -build_camp() { - echo "==========================================" - echo "Building CAMP" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/camp.git" "${CAMP_VER}" "${BASE_DIR}/camp" - sync_submodules "${BASE_DIR}/camp" - - prepare_build_dir "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - ) - fi - - run_with_log my_camp_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_camp_build make -j "${MAKE_JOBS}" - run_with_log my_camp_install make install - - CAMP_ROOT="${BASE_DIR}/camp/install_${BUILD_SUFFIX}" - export CAMP_ROOT - echo "CAMP installed to: ${CAMP_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# RAJA -########################################### -build_raja() { - echo "==========================================" - echo "Building RAJA" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/RAJA.git" "${RAJA_VER}" "${BASE_DIR}/RAJA" - sync_submodules "${BASE_DIR}/RAJA" - - prepare_build_dir "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DRAJA_ENABLE_TESTS=OFF - -DRAJA_ENABLE_EXAMPLES=OFF - -DRAJA_ENABLE_BENCHMARKS=OFF - -DRAJA_ENABLE_REPRODUCERS=OFF - -DRAJA_ENABLE_EXERCISES=OFF - -DRAJA_ENABLE_VECTORIZATION=OFF - -DRAJA_ENABLE_DOCUMENTATION=OFF - -DRAJA_USE_DOUBLE=ON - -DRAJA_TIMER=chrono - -DENABLE_OPENMP="${OPENMP_ON}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - ) - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DRAJA_USE_BARE_PTR=ON - ) - fi - fi - - run_with_log my_raja_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_raja_build make -j "${MAKE_JOBS}" - run_with_log my_raja_install make install - - RAJA_ROOT="${BASE_DIR}/RAJA/install_${BUILD_SUFFIX}" - export RAJA_ROOT - echo "RAJA installed to: ${RAJA_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# Umpire (GPU only) -########################################### -build_umpire() { - if [ "${BUILD_TYPE}" = "cpu" ]; then - echo "Skipping Umpire (not needed for CPU builds)" - return 0 - fi - - echo "==========================================" - echo "Building Umpire" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/Umpire.git" "${UMPIRE_VER}" "${BASE_DIR}/Umpire" - sync_submodules "${BASE_DIR}/Umpire" - - prepare_build_dir "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_MPI=OFF - -DUMPIRE_ENABLE_C=OFF - -DENABLE_FORTRAN=OFF - -DENABLE_GMOCK=OFF - -DUMPIRE_ENABLE_IPC_SHARED_MEMORY=OFF - -DUMPIRE_ENABLE_TOOLS=ON - -DUMPIRE_ENABLE_BACKTRACE=ON - -DUMPIRE_ENABLE_BACKTRACE_SYMBOLS=ON - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - run_with_log my_umpire_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_umpire_build make -j "${MAKE_JOBS}" - run_with_log my_umpire_install make install - - UMPIRE_ROOT="${BASE_DIR}/Umpire/install_${BUILD_SUFFIX}" - export UMPIRE_ROOT - - # Find fmt directory - FMT_DIR_CMAKE=$(find "${UMPIRE_ROOT}" -name 'fmtConfig.cmake' -print -quit || true) - if [ -n "${FMT_DIR_CMAKE}" ]; then - FMT_DIR=$(dirname "${FMT_DIR_CMAKE}") - else - FMT_DIR="${UMPIRE_ROOT}" - fi - export FMT_DIR - - # Find Umpire's CMake config dir (lib64 on RHEL-like systems, lib on Debian/Ubuntu) - UMPIRE_CMAKE_CONFIG=$(find "${UMPIRE_ROOT}" -name 'umpire-config.cmake' -print -quit || true) - if [ -n "${UMPIRE_CMAKE_CONFIG}" ]; then - UMPIRE_CMAKE_DIR=$(dirname "${UMPIRE_CMAKE_CONFIG}") - else - UMPIRE_CMAKE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" - fi - export UMPIRE_CMAKE_DIR - - echo "Umpire installed to: ${UMPIRE_ROOT}" - echo "fmt found at: ${FMT_DIR}" - cd "${BASE_DIR}" -} - -########################################### -# CHAI (GPU only) -########################################### -build_chai() { - if [ "${BUILD_TYPE}" = "cpu" ]; then - echo "Skipping CHAI (not needed for CPU builds)" - return 0 - fi - - echo "==========================================" - echo "Building CHAI" - echo "==========================================" - - clone_if_missing "https://github.com/LLNL/CHAI.git" "${CHAI_VER}" "${BASE_DIR}/CHAI" - sync_submodules "${BASE_DIR}/CHAI" - - prepare_build_dir "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_EXAMPLES=OFF - -DENABLE_DOCS=OFF - -DENABLE_GMOCK=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_MPI=OFF - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -DCHAI_ENABLE_RAJA_PLUGIN=ON - -DCHAI_ENABLE_RAJA_NESTED_TEST=OFF - -DCHAI_THIN_GPU_ALLOCATE="${CHAI_THIN_GPU_ALLOCATE}" - -DCHAI_ENABLE_PINNED="${CHAI_ENABLE_PINNED}" - -DCHAI_DISABLE_RM="${CHAI_DISABLE_RM}" - -DCHAI_ENABLE_PICK="${CHAI_ENABLE_PICK}" - -DCHAI_DEBUG="${CHAI_DEBUG}" - -DCHAI_ENABLE_GPU_SIMULATION_MODE="${CHAI_ENABLE_GPU_SIMULATION_MODE}" - -DCHAI_ENABLE_UM="${CHAI_ENABLE_UM}" - -DCHAI_ENABLE_MANAGED_PTR="${CHAI_ENABLE_MANAGED_PTR}" - -DCHAI_ENABLE_MANAGED_PTR_ON_GPU="${CHAI_ENABLE_MANAGED_PTR_ON_GPU}" - -Dfmt_DIR="${FMT_DIR}" - -Dumpire_DIR="${UMPIRE_ROOT}" - -DRAJA_DIR="${RAJA_ROOT}" - -Dcamp_DIR="${CAMP_ROOT}" - ) - - run_with_log my_chai_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_chai_build make -j "${MAKE_JOBS}" - run_with_log my_chai_install make install - - CHAI_ROOT="${BASE_DIR}/CHAI/install_${BUILD_SUFFIX}" - export CHAI_ROOT - echo "CHAI installed to: ${CHAI_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# ExaCMech -########################################### -build_exacmech() { - echo "==========================================" - echo "Building ExaCMech" - echo "==========================================" - - clone_if_missing "${EXACMECH_REPO}" "${EXACMECH_BRANCH}" "${BASE_DIR}/ExaCMech" - sync_submodules "${BASE_DIR}/ExaCMech" - - prepare_build_dir "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DENABLE_TESTS=OFF - -DENABLE_MINIAPPS=OFF - -DENABLE_OPENMP="${OPENMP_ON}" - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" - -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" != "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_${GPU_BACKEND}=ON - -DFMT_DIR="${FMT_DIR}" - -DUMPIRE_DIR="${UMPIRE_CMAKE_DIR}" - -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" - ) - fi - - run_with_log my_ecmech_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_ecmech_build make -j "${MAKE_JOBS}" - run_with_log my_ecmech_install make install - - ECMECH_ROOT="${BASE_DIR}/ExaCMech/install_${BUILD_SUFFIX}" - export ECMECH_ROOT - echo "ExaCMech installed to: ${ECMECH_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# Hypre -########################################### -build_hypre() { - echo "==========================================" - echo "Building Hypre" - echo "==========================================" - - if [ ! -d "${BASE_DIR}/hypre" ]; then - git clone https://github.com/hypre-space/hypre.git --branch "${HYPRE_VER}" --single-branch "${BASE_DIR}/hypre" - fi - - prepare_build_dir "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" - - run_with_log my_hypre_config cmake ../src \ - -DCMAKE_INSTALL_PREFIX=../src/hypre_${BUILD_SUFFIX}/ \ - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \ - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ - -DMPI_C_COMPILER="${MPI_C_COMPILER}" \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - - run_with_log my_hypre_build make -j "${MAKE_JOBS}" - run_with_log my_hypre_install make install - - HYPRE_ROOT="${BASE_DIR}/hypre/src/hypre_${BUILD_SUFFIX}" - export HYPRE_ROOT - echo "Hypre installed to: ${HYPRE_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# METIS -########################################### -build_metis() { - echo "==========================================" - echo "Building METIS" - echo "==========================================" - - if [ ! -d "${BASE_DIR}/metis-${METIS_VER}" ]; then - curl -o metis-${METIS_VER}.tar.gz "${METIS_URL}" - tar -xzf metis-${METIS_VER}.tar.gz - rm metis-${METIS_VER}.tar.gz - fi - - cd "${BASE_DIR}/metis-${METIS_VER}" - - # METIS doesn't have a proper incremental build, so always clean - make distclean 2>/dev/null || true - - prepare_build_dir "${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" - - run_with_log my_metis_config make config \ - prefix="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" \ - CC="${CMAKE_C_COMPILER}" \ - CXX="${CMAKE_CXX_COMPILER}" - - run_with_log my_metis_build make -j "${MAKE_JOBS}" - run_with_log my_metis_install make install - - METIS_ROOT="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" - export METIS_ROOT - echo "METIS installed to: ${METIS_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# MFEM batched gemv tuning (GPU builds) -########################################### -# MFEM's native batched matrix-vector kernels (used by element assembly) have -# tuning parameters with defaults per GPU vendor. MFEM's -# config/tune_batched_gemv.py measures them on the GPU at hand and writes a -# header that MFEM is rebuilt with (only one MFEM source file recompiles). +# Meta-loader for the ExaConstit build functions. # -# MFEM_GEMV_TUNING: -# AUTO (default) tune after MFEM is built and save the header as -# mfem/gemv_tuning__.hpp; later builds reuse it -# (MFEM_GEMV_RETUNE=ON tunes again) -# OFF use MFEM's defaults -# use this tuning header -# MFEM_GEMV_TUNING_LAUNCHER: command prefix that runs the benchmark on a GPU -# node, e.g. "flux run -n1 -g1" or "srun -n1 -G1", for builds on a node -# without a GPU. Without it the benchmark runs directly on the build node. -# MFEM_GEMV_TUNING_ARGS: extra arguments for tune_batched_gemv.py. - -# Saved tuning header for this build type and GPU architecture -mfem_gemv_tuning_file() { - local arch="${CMAKE_GPU_ARCHITECTURES}" - if [ "${GPU_BACKEND}" = "HIP" ]; then - arch="${MFEM_HIP_ARCHITECTURES:-${CMAKE_GPU_ARCHITECTURES}}" - fi - echo "${BASE_DIR}/mfem/gemv_tuning_${BUILD_SUFFIX}_${arch//[^A-Za-z0-9]/_}.hpp" -} - -# Tuning header to configure MFEM with (empty for MFEM's defaults) -mfem_gemv_tuning_header() { - local mode="${MFEM_GEMV_TUNING:-AUTO}" - if [ "${mode}" = "OFF" ]; then - echo "" - elif [ "${mode}" = "AUTO" ]; then - local hdr - hdr="$(mfem_gemv_tuning_file)" - if [ -f "${hdr}" ]; then echo "${hdr}"; else echo ""; fi - elif [ -f "${mode}" ]; then - echo "$(cd "$(dirname "${mode}")" && pwd)/$(basename "${mode}")" - else - echo "MFEM_GEMV_TUNING: no such file: ${mode}" >&2 - return 1 - fi -} - -# Run in the MFEM build directory after MFEM is installed. Never fails the -# build: if tuning is not possible, MFEM keeps its default parameters. -tune_mfem_gemv() { - [ "${MFEM_GEMV_TUNING:-AUTO}" = "AUTO" ] || return 0 - local hdr script - hdr="$(mfem_gemv_tuning_file)" - script="${BASE_DIR}/mfem/config/tune_batched_gemv.py" - if [ -f "${hdr}" ] && [ "${MFEM_GEMV_RETUNE:-OFF}" != "ON" ]; then - echo "Using saved batched gemv tuning: ${hdr}" - return 0 - fi - if [ ! -f "${script}" ]; then - echo "WARNING: ${script} not found (older MFEM?); skipping batched gemv tuning." - return 0 - fi - local launcher="${MFEM_GEMV_TUNING_LAUNCHER:-}" - echo "==========================================" - echo "Tuning MFEM batched gemv kernels" - echo "==========================================" - local python="${PYTHON_EXECUTABLE:-python3}" - [ -x "${python}" ] || python="python3" - local -a args=(--mfem-dir "${MFEM_ROOT}" - --work-dir "${PWD}/tune_batched_gemv" - --output "${hdr}.new") - [ -n "${launcher}" ] && args+=(--launcher "${launcher}") - local -a extra=() - read -r -a extra <<< "${MFEM_GEMV_TUNING_ARGS:-}" - if run_with_log my_mfem_tune "${python}" "${script}" "${args[@]}" \ - ${extra[@]+"${extra[@]}"}; then - mv "${hdr}.new" "${hdr}" - run_with_log my_mfem_config_tuned cmake -DMFEM_BATCHED_GEMV_TUNING="${hdr}" . - run_with_log my_mfem_build_tuned make -j "${MAKE_JOBS}" - run_with_log my_mfem_install_tuned make install - echo "MFEM rebuilt with batched gemv tuning: ${hdr}" - else - rm -f "${hdr}.new" - echo "WARNING: batched gemv tuning failed (see my_mfem_tune in $PWD);" - echo " MFEM keeps its default parameters." - if [ -z "${launcher}" ]; then - echo " If this node has no GPU, set MFEM_GEMV_TUNING_LAUNCHER" - echo " (e.g. \"flux run -n1 -g1\") to run the benchmark on a GPU node." - fi - fi -} - -########################################### -# MFEM -########################################### -build_mfem() { - echo "==========================================" - echo "Building MFEM" - echo "==========================================" - - clone_if_missing "${MFEM_REPO}" "${MFEM_BRANCH}" "${BASE_DIR}/mfem" - # Don't sync submodules for MFEM to preserve local changes - - prepare_build_dir "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DMFEM_USE_MPI=YES - -DMFEM_USE_SIMD=NO - -DMETIS_DIR="${METIS_ROOT}" - -DHYPRE_DIR="${HYPRE_ROOT}" - -DMFEM_USE_RAJA=YES - -DRAJA_DIR="${RAJA_ROOT}" - -DRAJA_REQUIRED_PACKAGES="camp" - -DMFEM_USE_CAMP=ON - -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" - -DMFEM_USE_OPENMP="${OPENMP_ON}" - -DMFEM_USE_ZLIB=YES - -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - ) - - if [ "${BUILD_TYPE}" = "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" - ) - else - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DMFEM_USE_${GPU_BACKEND}=ON - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - ) - - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DCMAKE_CUDA_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CXX_COMPILER}" - -DCMAKE_CUDA_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" - -DENABLE_CUDA=ON - # nvcc is the CXX compiler here; make CMake emit nvcc-style link options - -DCMAKE_USER_MAKE_RULES_OVERRIDE_CXX="${SCRIPT_DIR}/common/nvcc_cxx_rules_override.cmake" - ) - elif [ "${GPU_BACKEND}" = "HIP" ]; then - CMAKE_ARGS+=( - -DHIP_ARCH="${MFEM_HIP_ARCHITECTURES}" - -DCMAKE_HIP_ARCHITECTURES="${MFEM_HIP_ARCHITECTURES}" - ) - fi - local tuning - tuning="$(mfem_gemv_tuning_header)" - CMAKE_ARGS+=(-DMFEM_BATCHED_GEMV_TUNING="${tuning}") - fi - - run_with_log my_mfem_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_mfem_build make -j "${MAKE_JOBS}" - run_with_log my_mfem_install make install - - MFEM_ROOT="${BASE_DIR}/mfem/install_${BUILD_SUFFIX}" - export MFEM_ROOT - if [ "${BUILD_TYPE}" != "cpu" ]; then - tune_mfem_gemv - fi - echo "MFEM installed to: ${MFEM_ROOT}" - cd "${BASE_DIR}" -} - -########################################### -# ExaConstit -########################################### -build_exaconstit() { - echo "==========================================" - echo "Building ExaConstit" - echo "==========================================" - - clone_if_missing "${EXACONSTIT_REPO}" "${EXACONSTIT_BRANCH}" "${BASE_DIR}/ExaConstit" - sync_submodules "${BASE_DIR}/ExaConstit" - - prepare_build_dir "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" - cd "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" - - local CMAKE_ARGS=( - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" - -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" - -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" - -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" - -DENABLE_TESTS="${ENABLE_TESTS_EXACONSTIT}" - -DENABLE_OPENMP="${OPENMP_ON}" - -DENABLE_FORTRAN=OFF - -DENABLE_SNLS_V03=ON - -DCMAKE_INSTALL_PREFIX=../install_dir/ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" - -DMFEM_DIR="${MFEM_ROOT}/lib/cmake/mfem" - -DECMECH_DIR="${ECMECH_ROOT}" - -DSNLS_DIR="${ECMECH_ROOT}" - -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" - -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" - ) - - if [ "${BUILD_TYPE}" = "cpu" ]; then - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" - ) - else - CMAKE_ARGS+=( - -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" - -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" - -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" - -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" - -DENABLE_${GPU_BACKEND}=ON - -DFMT_DIR="${FMT_DIR}" - -DUMPIRE_DIR="${UMPIRE_CMAKE_DIR}" - -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" - ) - - if [ "${GPU_BACKEND}" = "CUDA" ]; then - CMAKE_ARGS+=( - -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" - -DBLT_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" - # nvcc is the CXX compiler here; make CMake emit nvcc-style link options - -DCMAKE_USER_MAKE_RULES_OVERRIDE_CXX="${SCRIPT_DIR}/common/nvcc_cxx_rules_override.cmake" - ) - elif [ "${GPU_BACKEND}" = "HIP" ]; then - CMAKE_ARGS+=( - -DCMAKE_HIP_FLAGS="${CMAKE_GPU_FLAGS}" - ) - fi - fi - - run_with_log my_exconstit_config cmake ../ "${CMAKE_ARGS[@]}" - run_with_log my_exconstit_build make -j "${MAKE_JOBS}" - - EXACONSTIT_ROOT="${BASE_DIR}/ExaConstit/install_dir" - export EXACONSTIT_ROOT - echo "==========================================" - echo "ExaConstit build complete!" - echo "Install prefix: ${EXACONSTIT_ROOT}" - echo "==========================================" - cd "${BASE_DIR}" -} - -########################################### -# Main orchestration function -########################################### -build_all_dependencies() { - build_camp - build_raja - build_umpire - build_chai - build_exacmech - build_hypre - build_metis - build_mfem - build_exaconstit -} \ No newline at end of file +# The build logic was split into a helpers file and three layer files +# grouped by dependency tier; this file simply sources them in +# dependency order so existing entry-point scripts (unix_*_install.sh) +# keep working unchanged. +# +# build_helpers.sh Shared helper functions +# (run_with_log, clone_if_missing, +# sync_submodules, prepare_build_dir). +# build_functions_common.sh BLT, CAMP, RAJA, Umpire, CHAI -- the +# shared portability stack. +# build_functions_mfem.sh Hypre, METIS, MFEM -- the FEM stack. +# build_functions_exaconstit.sh SNLS, ExaCMech, Axom, ExaConstit, +# plus the build_all_dependencies +# orchestrator. + +# Resolve our own location so each file sources its sibling. +_BUILD_FUNCTIONS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source "${_BUILD_FUNCTIONS_DIR}/build_helpers.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_common.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_mfem.sh" +source "${_BUILD_FUNCTIONS_DIR}/build_functions_exaconstit.sh" + +unset _BUILD_FUNCTIONS_DIR diff --git a/scripts/install/common/build_functions_common.sh b/scripts/install/common/build_functions_common.sh new file mode 100644 index 00000000..b164968e --- /dev/null +++ b/scripts/install/common/build_functions_common.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +# Common-stack build functions: BLT, CAMP, RAJA, Umpire, CHAI. +# +# These are the shared portability / utility libraries used by both +# the MFEM stack and the ExaConstit application stack. Helpers live +# in build_helpers.sh; the MFEM-stack and application-stack functions +# live in build_functions_mfem.sh and build_functions_exaconstit.sh +# respectively. +# +# Note: Umpire and CHAI are built on every platform now. The batch +# SNLS solvers depend on the full RAJA Portability Suite, and ExaCMech +# transitively links the same set, so making CHAI/Umpire available on +# CPU keeps the dependency graph uniform across CPU and GPU builds. + +########################################### +# BLT +########################################### +# BLT is a CMake-only build helper (header / macro / module library). +# It has no compile or install step. We clone it once and point every +# downstream LLNL/RADIUSS package at it via -DBLT_SOURCE_DIR=${BLT_ROOT}. +# This keeps every package on the same BLT version regardless of what +# their bundled submodule happens to point at. +build_blt() { + echo "==========================================" + echo "Cloning BLT (${BLT_VER})" + echo "==========================================" + + clone_if_missing "${BLT_REPO}" "${BLT_VER}" "${BASE_DIR}/blt" + + BLT_ROOT="${BASE_DIR}/blt" + export BLT_ROOT + echo "BLT available at: ${BLT_ROOT}" + echo "Downstream packages will consume it via -DBLT_SOURCE_DIR" + cd "${BASE_DIR}" +} + +########################################### +# CAMP +########################################### +build_camp() { + echo "==========================================" + echo "Building CAMP" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/camp.git" "${CAMP_VER}" "${BASE_DIR}/camp" + sync_submodules "${BASE_DIR}/camp" + + prepare_build_dir "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/camp/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_camp_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_camp_build make -j "${MAKE_JOBS}" + run_with_log my_camp_install make install + + CAMP_ROOT="${BASE_DIR}/camp/install_${BUILD_SUFFIX}" + export CAMP_ROOT + echo "CAMP installed to: ${CAMP_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# RAJA +########################################### +build_raja() { + echo "==========================================" + echo "Building RAJA" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/RAJA.git" "${RAJA_VER}" "${BASE_DIR}/RAJA" + sync_submodules "${BASE_DIR}/RAJA" + + prepare_build_dir "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/RAJA/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DRAJA_ENABLE_TESTS=OFF + -DRAJA_ENABLE_EXAMPLES=OFF + -DRAJA_ENABLE_BENCHMARKS=OFF + -DRAJA_ENABLE_REPRODUCERS=OFF + -DRAJA_ENABLE_EXERCISES=OFF + -DRAJA_ENABLE_VECTORIZATION=OFF + -DRAJA_ENABLE_DOCUMENTATION=OFF + -DRAJA_USE_DOUBLE=ON + -DRAJA_TIMER=chrono + -DENABLE_OPENMP="${OPENMP_ON}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DRAJA_USE_BARE_PTR=ON + ) + fi + fi + + run_with_log my_raja_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_raja_build make -j "${MAKE_JOBS}" + run_with_log my_raja_install make install + + RAJA_ROOT="${BASE_DIR}/RAJA/install_${BUILD_SUFFIX}" + export RAJA_ROOT + echo "RAJA installed to: ${RAJA_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Umpire +########################################### +# Built on both CPU and GPU. SNLS's batch solvers depend on Umpire, and +# we want batch solvers available regardless of platform. +build_umpire() { + echo "==========================================" + echo "Building Umpire" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/Umpire.git" "${UMPIRE_VER}" "${BASE_DIR}/Umpire" + sync_submodules "${BASE_DIR}/Umpire" + + prepare_build_dir "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/Umpire/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=OFF + -DUMPIRE_ENABLE_C=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_GMOCK=OFF + -DUMPIRE_ENABLE_IPC_SHARED_MEMORY=OFF + -DUMPIRE_ENABLE_TOOLS=ON + -DUMPIRE_ENABLE_BACKTRACE=ON + -DUMPIRE_ENABLE_BACKTRACE_SYMBOLS=ON + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_umpire_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_umpire_build make -j "${MAKE_JOBS}" + run_with_log my_umpire_install make install + + UMPIRE_ROOT="${BASE_DIR}/Umpire/install_${BUILD_SUFFIX}" + export UMPIRE_ROOT + + # Find fmt directory (Umpire vendors fmt and exports a CMake config for it) + FMT_DIR_CMAKE=$(find "${UMPIRE_ROOT}" -name 'fmtConfig.cmake' -print -quit || true) + if [ -n "${FMT_DIR_CMAKE}" ]; then + FMT_DIR=$(dirname "${FMT_DIR_CMAKE}") + else + FMT_DIR="${UMPIRE_ROOT}" + fi + export FMT_DIR + + # Find Umpire's CMake config dir (lib64 on RHEL-like systems, lib on Debian/Ubuntu) + UMPIRE_CMAKE_CONFIG=$(find "${UMPIRE_ROOT}" -name 'umpire-config.cmake' -print -quit || true) + if [ -n "${UMPIRE_CMAKE_CONFIG}" ]; then + UMPIRE_CMAKE_DIR=$(dirname "${UMPIRE_CMAKE_CONFIG}") + else + UMPIRE_CMAKE_DIR="${UMPIRE_ROOT}/lib64/cmake/umpire" + fi + export UMPIRE_CMAKE_DIR + + echo "Umpire installed to: ${UMPIRE_ROOT}" + echo "fmt found at: ${FMT_DIR}" + cd "${BASE_DIR}" +} + +########################################### +# CHAI +########################################### +# Built on both CPU and GPU. SNLS's batch solvers consume CHAI's +# ManagedArray and managed_ptr plumbing; on CPU CHAI's GPU-specific +# knobs (pinned, UM, managed_ptr_on_gpu, etc.) are OFF in the platform +# configs, but CHAI_ENABLE_MANAGED_PTR must stay ON since SNLS and +# ExaCMech use chai::managed_ptr on every platform. +build_chai() { + echo "==========================================" + echo "Building CHAI" + echo "==========================================" + + clone_if_missing "https://github.com/LLNL/CHAI.git" "${CHAI_VER}" "${BASE_DIR}/CHAI" + sync_submodules "${BASE_DIR}/CHAI" + + prepare_build_dir "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/CHAI/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_EXAMPLES=OFF + -DENABLE_DOCS=OFF + -DENABLE_GMOCK=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=OFF + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DCHAI_ENABLE_RAJA_PLUGIN=ON + -DCHAI_ENABLE_RAJA_NESTED_TEST=OFF + -DCHAI_THIN_GPU_ALLOCATE="${CHAI_THIN_GPU_ALLOCATE}" + -DCHAI_ENABLE_PINNED="${CHAI_ENABLE_PINNED}" + -DCHAI_DISABLE_RM="${CHAI_DISABLE_RM}" + -DCHAI_ENABLE_PICK="${CHAI_ENABLE_PICK}" + -DCHAI_DEBUG="${CHAI_DEBUG}" + -DCHAI_ENABLE_GPU_SIMULATION_MODE="${CHAI_ENABLE_GPU_SIMULATION_MODE}" + -DCHAI_ENABLE_UM="${CHAI_ENABLE_UM}" + -DCHAI_ENABLE_MANAGED_PTR="${CHAI_ENABLE_MANAGED_PTR}" + -DCHAI_ENABLE_MANAGED_PTR_ON_GPU="${CHAI_ENABLE_MANAGED_PTR_ON_GPU}" + -Dfmt_DIR="${FMT_DIR}" + -Dumpire_DIR="${UMPIRE_ROOT}" + -DRAJA_DIR="${RAJA_ROOT}" + -Dcamp_DIR="${CAMP_ROOT}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_chai_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_chai_build make -j "${MAKE_JOBS}" + run_with_log my_chai_install make install + + CHAI_ROOT="${BASE_DIR}/CHAI/install_${BUILD_SUFFIX}" + export CHAI_ROOT + echo "CHAI installed to: ${CHAI_ROOT}" + cd "${BASE_DIR}" +} diff --git a/scripts/install/common/build_functions_exaconstit.sh b/scripts/install/common/build_functions_exaconstit.sh new file mode 100644 index 00000000..6f5d5b1a --- /dev/null +++ b/scripts/install/common/build_functions_exaconstit.sh @@ -0,0 +1,465 @@ +#!/usr/bin/env bash +# ExaConstit application-stack build functions: SNLS, ExaCMech, HDF5, +# Conduit, Axom, and ExaConstit. Also defines the top-level +# build_all_dependencies orchestrator. +# +# Depends on the helpers in build_helpers.sh, the common stack defined +# in build_functions_common.sh (BLT, CAMP, RAJA, Umpire, CHAI), and +# MFEM defined in build_functions_mfem.sh. +# +# Axom lives here rather than in the common stack because it will +# eventually depend on MFEM, which puts it logically downstream of the +# MFEM-stack build file and alongside the other application-tier +# packages. HDF5 and Conduit live here because they are only needed +# for Axom's Sidre component (restart / checkpoint I/O). + +########################################### +# SNLS +########################################### +# Lifted out of ExaCMech and built standalone with the batch-solver +# option always enabled. Batch solvers require the full RAJA +# Portability Suite (RAJA + Umpire + CHAI + camp); since the common +# stack now builds Umpire and CHAI on every platform, this is uniform +# across CPU and GPU. +build_snls() { + echo "==========================================" + echo "Building SNLS" + echo "==========================================" + + clone_if_missing "${SNLS_REPO}" "${SNLS_VER}" "${BASE_DIR}/SNLS" + sync_submodules "${BASE_DIR}/SNLS" + + prepare_build_dir "${BASE_DIR}/SNLS/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/SNLS/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # Batch solvers ON everywhere -> needs the full Portability Suite. + -DUSE_BATCH_SOLVERS=ON + -DUSE_RAJA_ONLY=OFF + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + -DUMPIRE_DIR="${UMPIRE_CMAKE_DIR}" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + -DFMT_DIR="${FMT_DIR}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_snls_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_snls_build make -j "${MAKE_JOBS}" + run_with_log my_snls_install make install + + SNLS_ROOT="${BASE_DIR}/SNLS/install_${BUILD_SUFFIX}" + export SNLS_ROOT + echo "SNLS installed to: ${SNLS_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# ExaCMech +########################################### +# Consumes the standalone SNLS instead of its bundled submodule. +# ExaCMech's CMakeLists auto-sets its internal USE_BUILT_SNLS=ON when +# SNLS_DIR is defined, so we only need to pass SNLS_DIR -- no other +# external-SNLS toggle required. +# +# Because the standalone SNLS is built with USE_BATCH_SOLVERS=ON, it +# pulls CHAI / Umpire / fmt into ExaCMech's link line transitively. +# So FMT_DIR / UMPIRE_DIR / CHAI_DIR are passed unconditionally now, +# regardless of whether ExaCMech itself is being built with GPU support. +build_exacmech() { + echo "==========================================" + echo "Building ExaCMech" + echo "==========================================" + + clone_if_missing "${EXACMECH_REPO}" "${EXACMECH_BRANCH}" "${BASE_DIR}/ExaCMech" + sync_submodules "${BASE_DIR}/ExaCMech" + + prepare_build_dir "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/ExaCMech/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DENABLE_TESTS=OFF + -DENABLE_MINIAPPS=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # External SNLS: defining SNLS_DIR is sufficient; ExaCMech sets + # USE_BUILT_SNLS=ON internally when it sees this variable. + -DSNLS_DIR="${SNLS_ROOT}/lib/cmake/snls" + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + # SNLS was built with batch solvers, so ExaCMech needs the full + # Portability Suite resolved transitively even on CPU builds. + -DFMT_DIR="${FMT_DIR}" + -DUMPIRE_DIR="${UMPIRE_CMAKE_DIR}" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_${GPU_BACKEND}=ON + ) + fi + + run_with_log my_ecmech_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_ecmech_build make -j "${MAKE_JOBS}" + run_with_log my_ecmech_install make install + + ECMECH_ROOT="${BASE_DIR}/ExaCMech/install_${BUILD_SUFFIX}" + export ECMECH_ROOT + echo "ExaCMech installed to: ${ECMECH_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# HDF5 +########################################### +# Serial HDF5 used by Conduit's relay I/O, which is what Sidre's +# sidre_hdf5 protocol writes through. Sidre's parallel I/O (IOManager) +# writes one HDF5 file per rank group itself, so a parallel HDF5 build +# is not needed. Only the C library is built; Fortran / C++ / examples +# and tests are skipped. Tools stay on so h5dump / h5ls are available +# for inspecting restart files. +build_hdf5() { + echo "==========================================" + echo "Building HDF5" + echo "==========================================" + + clone_if_missing "${HDF5_REPO}" "${HDF5_VER}" "${BASE_DIR}/hdf5" + + prepare_build_dir "${BASE_DIR}/hdf5/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/hdf5/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + # Static only. With both, Conduit links hdf5-static and Axom hdf5-shared, + # so ExaConstit gets two copies of HDF5, and nvcc (the linker in CUDA + # builds) rejects the versioned libhdf5.so. that hdf5-shared names. + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + -DBUILD_TESTING=OFF + -DHDF5_ENABLE_PARALLEL=OFF + -DHDF5_BUILD_CPP_LIB=OFF + -DHDF5_BUILD_FORTRAN=OFF + -DHDF5_BUILD_JAVA=OFF + -DHDF5_BUILD_EXAMPLES=OFF + -DHDF5_BUILD_TOOLS=ON + -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF + -DHDF5_ENABLE_SZIP_SUPPORT=OFF + ) + + run_with_log my_hdf5_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_hdf5_build make -j "${MAKE_JOBS}" + run_with_log my_hdf5_install make install + + HDF5_ROOT="${BASE_DIR}/hdf5/install_${BUILD_SUFFIX}" + export HDF5_ROOT + echo "HDF5 installed to: ${HDF5_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Conduit +########################################### +# Hard dependency of Axom's Sidre component. Built with MPI (Axom +# requires conduit_relay_mpi when AXOM_ENABLE_MPI=ON) and HDF5 relay +# support. Conduit expects HDF5_DIR to be the HDF5 install prefix. +# +# Like Axom, Conduit's CMakeLists lives in src/, so the configure step +# points at ../src. +build_conduit() { + echo "==========================================" + echo "Building Conduit" + echo "==========================================" + + clone_if_missing "${CONDUIT_REPO}" "${CONDUIT_VER}" "${BASE_DIR}/conduit" + sync_submodules "${BASE_DIR}/conduit" + + prepare_build_dir "${BASE_DIR}/conduit/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/conduit/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DBLT_CXX_STD="c++${CMAKE_CXX_STANDARD}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DBUILD_SHARED_LIBS=OFF + -DENABLE_TESTS=OFF + -DENABLE_EXAMPLES=OFF + -DENABLE_UTILS=OFF + -DENABLE_DOCS=OFF + -DENABLE_PYTHON=OFF + -DENABLE_FORTRAN=OFF + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_MPI=ON + -DMPI_C_COMPILER="${MPI_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DHDF5_DIR="${HDF5_ROOT}" + ) + + run_with_log my_conduit_config cmake ../src "${CMAKE_ARGS[@]}" + run_with_log my_conduit_build make -j "${MAKE_JOBS}" + run_with_log my_conduit_install make install + + CONDUIT_ROOT="${BASE_DIR}/conduit/install_${BUILD_SUFFIX}" + export CONDUIT_ROOT + echo "Conduit installed to: ${CONDUIT_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# Axom +########################################### +# Built with the core component (always on) plus spin, primal, slam, +# and sidre. Slic is enabled explicitly because spin, sidre and other +# components rely on it for logging. Sidre is what the restart / +# checkpoint logic is built on; it pulls in Conduit (required) and +# HDF5 (for the sidre_hdf5 I/O protocol). +# +# Axom's CMakeLists lives in the src/ subdirectory, so the configure +# step points at ../src rather than ../ like the other packages. +build_axom() { + echo "==========================================" + echo "Building Axom" + echo "==========================================" + + clone_if_missing "${AXOM_REPO}" "${AXOM_VER}" "${BASE_DIR}/axom" + sync_submodules "${BASE_DIR}/axom" + + prepare_build_dir "${BASE_DIR}/axom/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/axom/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DBLT_CXX_STD="c++${CMAKE_CXX_STANDARD}" + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + # Disable everything by default, then turn on what we need. + -DAXOM_ENABLE_ALL_COMPONENTS=OFF + -DAXOM_ENABLE_SPIN=ON + -DAXOM_ENABLE_SLIC=ON + -DAXOM_ENABLE_SIDRE=ON + -DAXOM_ENABLE_INLET=OFF + -DAXOM_ENABLE_KLEE=OFF + -DAXOM_ENABLE_LUMBERJACK=ON + -DAXOM_ENABLE_MINT=OFF + -DAXOM_ENABLE_MIR=OFF + -DAXOM_ENABLE_MULTIMAT=OFF + -DAXOM_ENABLE_PRIMAL=ON + -DAXOM_ENABLE_QUEST=OFF + -DAXOM_ENABLE_SLAM=ON + # Build settings -- skip everything that isn't the library itself. + -DAXOM_ENABLE_TESTS=OFF + -DAXOM_ENABLE_EXAMPLES=OFF + -DAXOM_ENABLE_TUTORIALS=OFF + -DAXOM_ENABLE_DOCS=OFF + -DAXOM_ENABLE_TOOLS=OFF + -DENABLE_BENCHMARKS=OFF + -DENABLE_FORTRAN=OFF + # Parallelism / dependencies + # AXOM_ENABLE_MPI / AXOM_ENABLE_OPENMP are cmake_dependent_options + # gated on BLT's ENABLE_MPI / ENABLE_OPENMP, so those must be set + # too or Axom silently builds without MPI (no Sidre IOManager). + -DENABLE_MPI=ON + -DENABLE_OPENMP="${OPENMP_ON}" + -DAXOM_ENABLE_MPI=ON + -DAXOM_ENABLE_OPENMP="${OPENMP_ON}" + -DMPI_C_COMPILER="${MPI_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" + -DCAMP_DIR="${CAMP_ROOT}" + -DRAJA_DIR="${RAJA_ROOT}" + -DUMPIRE_DIR="${UMPIRE_ROOT}" + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + # Sidre dependencies + -DCONDUIT_DIR="${CONDUIT_ROOT}" + -DHDF5_DIR="${HDF5_ROOT}" + ) + + if [ "${BUILD_TYPE}" != "cpu" ]; then + # Spin's GPU paths run through RAJA -> Umpire memory plumbing. + # Like AXOM_ENABLE_MPI, AXOM_ENABLE_CUDA / AXOM_ENABLE_HIP are gated on + # BLT's ENABLE_CUDA / ENABLE_HIP. Without those, Axom still defines + # AXOM_USE_CUDA/HIP but compiles its sources with the host compiler, + # which fails in axom::Array's device-aware code. + CMAKE_ARGS+=( + -DENABLE_${GPU_BACKEND}=ON + -DAXOM_ENABLE_${GPU_BACKEND}=ON + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_FLAGS="${CMAKE_GPU_FLAGS}" + ) + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCUDA_TOOLKIT_ROOT_DIR="${CUDA_TOOLKIT_ROOT_DIR}" + ) + fi + fi + + run_with_log my_axom_config cmake ../src "${CMAKE_ARGS[@]}" + run_with_log my_axom_build make -j "${MAKE_JOBS}" + run_with_log my_axom_install make install + + AXOM_ROOT="${BASE_DIR}/axom/install_${BUILD_SUFFIX}" + export AXOM_ROOT + echo "Axom installed to: ${AXOM_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# ExaConstit +########################################### +# Like ExaCMech, the SNLS-batch transitive deps mean we pass FMT_DIR / +# UMPIRE_DIR / CHAI_DIR unconditionally now (previously GPU-only). +build_exaconstit() { + echo "==========================================" + echo "Building ExaConstit" + echo "==========================================" + + clone_if_missing "${EXACONSTIT_REPO}" "${EXACONSTIT_BRANCH}" "${BASE_DIR}/ExaConstit" + sync_submodules "${BASE_DIR}/ExaConstit" + + prepare_build_dir "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/ExaConstit/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" + -DENABLE_TESTS="${ENABLE_TESTS_EXACONSTIT}" + -DENABLE_OPENMP="${OPENMP_ON}" + -DENABLE_FORTRAN=OFF + -DENABLE_SNLS_V03=ON + -DCMAKE_INSTALL_PREFIX=../install_dir/ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + -DBLT_SOURCE_DIR="${BLT_ROOT}" + -DMFEM_DIR="${MFEM_ROOT}/lib/cmake/mfem" + -DECMECH_DIR="${ECMECH_ROOT}" + -DSNLS_DIR="${SNLS_ROOT}/lib/cmake/snls" + -DAXOM_DIR="${AXOM_ROOT}/lib/cmake" + -Daxom_DIR="${AXOM_ROOT}/lib/cmake" + -DRAJA_DIR="${RAJA_ROOT}/lib/cmake/raja" + -DCAMP_DIR="${CAMP_ROOT}/lib/cmake/camp" + # SNLS-batch transitive deps (now needed on CPU builds too). + -DFMT_DIR="${FMT_DIR}" + -DUMPIRE_DIR="${UMPIRE_CMAKE_DIR}" + -DCHAI_DIR="${CHAI_ROOT}/lib/cmake/chai" + ) + + if [ "${BUILD_TYPE}" = "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + ) + else + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + -DCMAKE_${GPU_BACKEND}_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_${GPU_BACKEND}_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DENABLE_${GPU_BACKEND}=ON + ) + + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" + -DBLT_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" + # nvcc is the CXX compiler here; make CMake emit nvcc-style link options + -DCMAKE_USER_MAKE_RULES_OVERRIDE_CXX="${SCRIPT_DIR}/common/nvcc_cxx_rules_override.cmake" + ) + elif [ "${GPU_BACKEND}" = "HIP" ]; then + CMAKE_ARGS+=( + -DCMAKE_HIP_FLAGS="${CMAKE_GPU_FLAGS}" + ) + fi + fi + + run_with_log my_exconstit_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_exconstit_build make -j "${MAKE_JOBS}" + + EXACONSTIT_ROOT="${BASE_DIR}/ExaConstit/install_dir" + export EXACONSTIT_ROOT + echo "==========================================" + echo "ExaConstit build complete!" + echo "Install prefix: ${EXACONSTIT_ROOT}" + echo "==========================================" + cd "${BASE_DIR}" +} + +########################################### +# Main orchestration function +########################################### +# Build order honors the dependency graph: +# 1. BLT (header-only build helper, must come first so every +# downstream package can point at it). +# 2. RAJA Portability Suite: CAMP -> RAJA -> Umpire -> CHAI +# (Umpire and CHAI now built on every platform). +# 3. MFEM stack: Hypre, METIS, MFEM. +# 4. Application stack: SNLS -> ExaCMech -> HDF5 -> Conduit -> Axom +# -> ExaConstit. SNLS and ExaCMech come first because the SNLS +# batch solver path is a hard dependency; HDF5 and Conduit feed +# Axom's Sidre component; Axom is placed before ExaConstit since +# ExaConstit consumes it (and Axom will eventually pick up MFEM). +build_all_dependencies() { + # Common stack + build_blt + build_camp + build_raja + build_umpire + build_chai + + # MFEM stack + build_hypre + build_metis + build_mfem + + # Application stack + build_snls + build_exacmech + build_hdf5 + build_conduit + build_axom + build_exaconstit +} diff --git a/scripts/install/common/build_functions_mfem.sh b/scripts/install/common/build_functions_mfem.sh new file mode 100644 index 00000000..820e2f67 --- /dev/null +++ b/scripts/install/common/build_functions_mfem.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# MFEM-stack build functions: Hypre, METIS, MFEM. +# +# Depends on the helpers in build_helpers.sh and the common stack +# defined in build_functions_common.sh (specifically RAJA / CAMP, +# which MFEM consumes). + +########################################### +# Hypre +########################################### +build_hypre() { + echo "==========================================" + echo "Building Hypre" + echo "==========================================" + + if [ ! -d "${BASE_DIR}/hypre" ]; then + git clone https://github.com/hypre-space/hypre.git --branch "${HYPRE_VER}" --single-branch "${BASE_DIR}/hypre" + fi + + prepare_build_dir "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/hypre/build_${BUILD_SUFFIX}" + + run_with_log my_hypre_config cmake ../src \ + -DCMAKE_INSTALL_PREFIX=../src/hypre_${BUILD_SUFFIX}/ \ + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" \ + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ + -DMPI_C_COMPILER="${MPI_C_COMPILER}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + + run_with_log my_hypre_build make -j "${MAKE_JOBS}" + run_with_log my_hypre_install make install + + HYPRE_ROOT="${BASE_DIR}/hypre/src/hypre_${BUILD_SUFFIX}" + export HYPRE_ROOT + echo "Hypre installed to: ${HYPRE_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# METIS +########################################### +build_metis() { + echo "==========================================" + echo "Building METIS" + echo "==========================================" + + if [ ! -d "${BASE_DIR}/metis-${METIS_VER}" ]; then + curl -o metis-${METIS_VER}.tar.gz "${METIS_URL}" + tar -xzf metis-${METIS_VER}.tar.gz + rm metis-${METIS_VER}.tar.gz + fi + + cd "${BASE_DIR}/metis-${METIS_VER}" + + # METIS doesn't have a proper incremental build, so always clean + make distclean 2>/dev/null || true + + prepare_build_dir "${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" + + run_with_log my_metis_config make config \ + prefix="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" \ + CC="${CMAKE_C_COMPILER}" \ + CXX="${CMAKE_CXX_COMPILER}" + + run_with_log my_metis_build make -j "${MAKE_JOBS}" + run_with_log my_metis_install make install + + METIS_ROOT="${BASE_DIR}/metis-${METIS_VER}/install_${BUILD_SUFFIX}" + export METIS_ROOT + echo "METIS installed to: ${METIS_ROOT}" + cd "${BASE_DIR}" +} + +########################################### +# MFEM batched gemv tuning (GPU builds) +########################################### +# MFEM's native batched matrix-vector kernels (used by element assembly) have +# tuning parameters with defaults per GPU vendor. MFEM's +# config/tune_batched_gemv.py measures them on the GPU at hand and writes a +# header that MFEM is rebuilt with (only one MFEM source file recompiles). +# +# MFEM_GEMV_TUNING: +# AUTO (default) tune after MFEM is built and save the header as +# mfem/gemv_tuning__.hpp; later builds reuse it +# (MFEM_GEMV_RETUNE=ON tunes again) +# OFF use MFEM's defaults +# use this tuning header +# MFEM_GEMV_TUNING_LAUNCHER: command prefix that runs the benchmark on a GPU +# node, e.g. "flux run -n1 -g1" or "srun -n1 -G1", for builds on a node +# without a GPU. Without it the benchmark runs directly on the build node. +# MFEM_GEMV_TUNING_ARGS: extra arguments for tune_batched_gemv.py. + +# Saved tuning header for this build type and GPU architecture +mfem_gemv_tuning_file() { + local arch="${CMAKE_GPU_ARCHITECTURES}" + if [ "${GPU_BACKEND}" = "HIP" ]; then + arch="${MFEM_HIP_ARCHITECTURES:-${CMAKE_GPU_ARCHITECTURES}}" + fi + echo "${BASE_DIR}/mfem/gemv_tuning_${BUILD_SUFFIX}_${arch//[^A-Za-z0-9]/_}.hpp" +} + +# Tuning header to configure MFEM with (empty for MFEM's defaults) +mfem_gemv_tuning_header() { + local mode="${MFEM_GEMV_TUNING:-AUTO}" + if [ "${mode}" = "OFF" ]; then + echo "" + elif [ "${mode}" = "AUTO" ]; then + local hdr + hdr="$(mfem_gemv_tuning_file)" + if [ -f "${hdr}" ]; then echo "${hdr}"; else echo ""; fi + elif [ -f "${mode}" ]; then + echo "$(cd "$(dirname "${mode}")" && pwd)/$(basename "${mode}")" + else + echo "MFEM_GEMV_TUNING: no such file: ${mode}" >&2 + return 1 + fi +} + +# Run in the MFEM build directory after MFEM is installed. Never fails the +# build: if tuning is not possible, MFEM keeps its default parameters. +tune_mfem_gemv() { + [ "${MFEM_GEMV_TUNING:-AUTO}" = "AUTO" ] || return 0 + local hdr script + hdr="$(mfem_gemv_tuning_file)" + script="${BASE_DIR}/mfem/config/tune_batched_gemv.py" + if [ -f "${hdr}" ] && [ "${MFEM_GEMV_RETUNE:-OFF}" != "ON" ]; then + echo "Using saved batched gemv tuning: ${hdr}" + return 0 + fi + if [ ! -f "${script}" ]; then + echo "WARNING: ${script} not found (older MFEM?); skipping batched gemv tuning." + return 0 + fi + local launcher="${MFEM_GEMV_TUNING_LAUNCHER:-}" + echo "==========================================" + echo "Tuning MFEM batched gemv kernels" + echo "==========================================" + local python="${PYTHON_EXECUTABLE:-python3}" + [ -x "${python}" ] || python="python3" + local -a args=(--mfem-dir "${MFEM_ROOT}" + --work-dir "${PWD}/tune_batched_gemv" + --output "${hdr}.new") + [ -n "${launcher}" ] && args+=(--launcher "${launcher}") + local -a extra=() + read -r -a extra <<< "${MFEM_GEMV_TUNING_ARGS:-}" + if run_with_log my_mfem_tune "${python}" "${script}" "${args[@]}" \ + ${extra[@]+"${extra[@]}"}; then + mv "${hdr}.new" "${hdr}" + run_with_log my_mfem_config_tuned cmake -DMFEM_BATCHED_GEMV_TUNING="${hdr}" . + run_with_log my_mfem_build_tuned make -j "${MAKE_JOBS}" + run_with_log my_mfem_install_tuned make install + echo "MFEM rebuilt with batched gemv tuning: ${hdr}" + else + rm -f "${hdr}.new" + echo "WARNING: batched gemv tuning failed (see my_mfem_tune in $PWD);" + echo " MFEM keeps its default parameters." + if [ -z "${launcher}" ]; then + echo " If this node has no GPU, set MFEM_GEMV_TUNING_LAUNCHER" + echo " (e.g. \"flux run -n1 -g1\") to run the benchmark on a GPU node." + fi + fi +} + +########################################### +# MFEM +########################################### +build_mfem() { + echo "==========================================" + echo "Building MFEM" + echo "==========================================" + + clone_if_missing "${MFEM_REPO}" "${MFEM_BRANCH}" "${BASE_DIR}/mfem" + # Don't sync submodules for MFEM to preserve local changes + + prepare_build_dir "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" + cd "${BASE_DIR}/mfem/build_${BUILD_SUFFIX}" + + local CMAKE_ARGS=( + -DMFEM_USE_MPI=YES + -DMFEM_USE_SIMD=NO + -DMETIS_DIR="${METIS_ROOT}" + -DHYPRE_DIR="${HYPRE_ROOT}" + -DMFEM_USE_RAJA=YES + -DRAJA_DIR="${RAJA_ROOT}" + -DRAJA_REQUIRED_PACKAGES="camp" + -DMFEM_USE_CAMP=ON + -Dcamp_DIR="${CAMP_ROOT}/lib/cmake/camp" + -DMFEM_USE_OPENMP="${OPENMP_ON}" + -DMFEM_USE_ZLIB=YES + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX}/ + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD}" + -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + ) + + if [ "${BUILD_TYPE}" = "cpu" ]; then + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${MPI_CXX_COMPILER}" + ) + else + CMAKE_ARGS+=( + -DCMAKE_CXX_COMPILER="${CMAKE_GPU_COMPILER}" + -DMPI_CXX_COMPILER="${MPI_CXX_COMPILER}" + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" + -DMFEM_USE_${GPU_BACKEND}=ON + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" + # Umpire, so that MFEM allocates from the same Umpire pools as SNLS and + # ExaCMech (ExaConstit's driver creates them and hands them to both) + -DMFEM_USE_UMPIRE=ON + -DUMPIRE_DIR="${UMPIRE_ROOT}" + -Dumpire_DIR="${UMPIRE_CMAKE_DIR}" + -Dfmt_DIR="${FMT_DIR}" + ) + + if [ "${GPU_BACKEND}" = "CUDA" ]; then + CMAKE_ARGS+=( + -DCMAKE_CUDA_COMPILER="${CMAKE_GPU_COMPILER}" + -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CXX_COMPILER}" + -DCMAKE_CUDA_ARCHITECTURES="${CMAKE_GPU_ARCHITECTURES}" + -DCMAKE_CUDA_FLAGS="${CMAKE_GPU_FLAGS}" + -DENABLE_CUDA=ON + # nvcc is the CXX compiler here; make CMake emit nvcc-style link options + -DCMAKE_USER_MAKE_RULES_OVERRIDE_CXX="${SCRIPT_DIR}/common/nvcc_cxx_rules_override.cmake" + ) + elif [ "${GPU_BACKEND}" = "HIP" ]; then + CMAKE_ARGS+=( + -DHIP_ARCH="${MFEM_HIP_ARCHITECTURES}" + -DCMAKE_HIP_ARCHITECTURES="${MFEM_HIP_ARCHITECTURES}" + ) + fi + local tuning + tuning="$(mfem_gemv_tuning_header)" + CMAKE_ARGS+=(-DMFEM_BATCHED_GEMV_TUNING="${tuning}") + fi + + run_with_log my_mfem_config cmake ../ "${CMAKE_ARGS[@]}" + run_with_log my_mfem_build make -j "${MAKE_JOBS}" + run_with_log my_mfem_install make install + + MFEM_ROOT="${BASE_DIR}/mfem/install_${BUILD_SUFFIX}" + export MFEM_ROOT + if [ "${BUILD_TYPE}" != "cpu" ]; then + tune_mfem_gemv + fi + echo "MFEM installed to: ${MFEM_ROOT}" + cd "${BASE_DIR}" +} diff --git a/scripts/install/common/build_helpers.sh b/scripts/install/common/build_helpers.sh new file mode 100644 index 00000000..8165e759 --- /dev/null +++ b/scripts/install/common/build_helpers.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Shared helper functions used by every build function. +# +# Kept separate from the build_functions_*.sh files so the per-library +# build logic stays focused on CMake invocations rather than the +# logging / cloning / build-dir-prep plumbing. + +########################################### +# Logging wrapper +########################################### +run_with_log() { + local log="$1"; shift + "$@" |& tee "$log" +} + +########################################### +# Clone repository only if missing, initialize submodules on first clone +########################################### +clone_if_missing() { + local repo="$1" branch="$2" dest="$3" + if [ ! -d "$dest/.git" ]; then + echo "Cloning ${dest}..." + git clone --branch "$branch" "$repo" "$dest" + cd "$dest" + if [ -f .gitmodules ]; then + git submodule update --init --recursive + fi + cd "$BASE_DIR" + else + echo "${dest} already exists, skipping clone." + fi +} + +########################################### +# Optional: force submodule sync when explicitly requested +########################################### +sync_submodules() { + local dest="$1" + if [ "${SYNC_SUBMODULES}" = "ON" ] && [ -f "$dest/.gitmodules" ]; then + echo "Syncing submodules in ${dest}..." + cd "$dest" + git submodule sync --recursive + git submodule update --init --recursive + cd "$BASE_DIR" + fi +} + +########################################### +# Respect REBUILD flag when preparing build directories +########################################### +prepare_build_dir() { + local dir="$1" + if [ "${REBUILD}" = "ON" ]; then + mkdir -p "$dir" + rm -rf "$dir"/* + echo "Cleaned build directory: ${dir}" + else + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + echo "Created build directory: ${dir}" + else + echo "Reusing existing build directory: ${dir}" + fi + fi +} diff --git a/scripts/install/common/dependency_versions.sh b/scripts/install/common/dependency_versions.sh index 1babd0a9..2bdb2025 100644 --- a/scripts/install/common/dependency_versions.sh +++ b/scripts/install/common/dependency_versions.sh @@ -1,7 +1,17 @@ #!/usr/bin/env bash # Central version control for all dependencies -# Portability libraries +########################################### +# Build infrastructure +########################################### +# BLT lifted out so all RADIUSS-stack packages share a single BLT and stay in sync. +# Each package below is pointed at this via -DBLT_SOURCE_DIR=${BLT_ROOT}. +export BLT_REPO="https://github.com/LLNL/blt.git" +export BLT_VER="v0.7.2" + +########################################### +# Portability libraries (RAJA Portability Suite) +########################################### export CAMP_VER="v2025.12.0" export RAJA_VER="v2025.12.2" #export UMPIRE_VER="v2025.09.0" @@ -10,11 +20,45 @@ export RAJA_VER="v2025.12.2" export UMPIRE_VER="v2025.12.0" export CHAI_VER="v2025.12.0" +########################################### +# SNLS (lifted out of ExaCMech so it can be built standalone with the +# RAJA Portability Suite and the batch-solver option always enabled) +########################################### +export SNLS_REPO="https://github.com/LLNL/SNLS.git" +export SNLS_VER="v0.4.4" + +########################################### +# Restart / checkpoint I/O stack (Axom Sidre) +########################################### +# Sidre requires Conduit, and Conduit's HDF5 relay provides the +# sidre_hdf5 I/O protocol. Stick with the HDF5 1.14 series; the 2.x +# series has caused CMake configuration issues downstream. +export HDF5_REPO="https://github.com/HDFGroup/hdf5.git" +export HDF5_VER="hdf5_1.14.6" +# Conduit v0.9.7 is the version Axom v0.15.0 was released against +# even though we use v0.14.0 below. +export CONDUIT_REPO="https://github.com/LLNL/conduit.git" +export CONDUIT_VER="v0.9.7" + +########################################### +# Axom (HPC utility library suite) +########################################### +# Built with core + slic + spin + primal + slam + sidre. Axom will +# eventually consume MFEM as well, which is why build_axom lives in the +# application-stack build file (build_functions_exaconstit.sh) rather +# than the common stack. +export AXOM_REPO="https://github.com/LLNL/axom.git" +export AXOM_VER="v0.14.0" + +########################################### # Material models +########################################### export EXACMECH_REPO="https://github.com/LLNL/ExaCMech.git" export EXACMECH_BRANCH="v0.4.3" +########################################### # FEM infrastructure +########################################### export HYPRE_VER="v3.1.0" export METIS_VER="5.1.0" export METIS_URL="https://mfem.github.io/tpls/metis-${METIS_VER}.tar.gz" @@ -22,10 +66,14 @@ export METIS_URL="https://mfem.github.io/tpls/metis-${METIS_VER}.tar.gz" export MFEM_REPO="https://github.com/rcarson3/mfem.git" export MFEM_BRANCH="exaconstit-dev" +########################################### # Main application +########################################### export EXACONSTIT_REPO="https://github.com/llnl/ExaConstit.git" export EXACONSTIT_BRANCH="exaconstit-dev" +########################################### # Build standards +########################################### export CMAKE_CXX_STANDARD="17" -export CMAKE_BUILD_TYPE="Release" \ No newline at end of file +export CMAKE_BUILD_TYPE="Release" diff --git a/scripts/install/common/preflight_checks.sh b/scripts/install/common/preflight_checks.sh index e0716520..92543b12 100644 --- a/scripts/install/common/preflight_checks.sh +++ b/scripts/install/common/preflight_checks.sh @@ -18,16 +18,16 @@ resolve_base_dir() { BASE_DIR=$(pwd -P) echo "Using current directory as build directory: ${BASE_DIR}" fi - + export BASE_DIR - + echo "==========================================" echo "Build Configuration:" echo " Base directory: ${BASE_DIR}" echo " All dependencies will be cloned and built here" echo "==========================================" echo "" - + # Optional: warn if running from ExaConstit source tree if [[ "${BASE_DIR}" == *"/ExaConstit"* ]]; then echo "⚠️ WARNING: You appear to be building inside the ExaConstit source tree." @@ -50,18 +50,18 @@ check_required_paths() { local missing=0 for p in "$@"; do if [[ "$p" == */bin/* ]]; then - if [ ! -x "$p" ]; then + if [ ! -x "$p" ]; then echo "ERROR: Missing executable: $p" >&2 missing=1 fi else - if [ ! -e "$p" ]; then + if [ ! -e "$p" ]; then echo "ERROR: Missing path: $p" >&2 missing=1 fi fi done - if [ "$missing" -ne 0 ]; then + if [ "$missing" -ne 0 ]; then echo "ERROR: Required paths missing. Exiting." >&2 exit 1 fi @@ -115,14 +115,17 @@ print_build_summary() { echo " Linker: ${CMAKE_EXE_LINKER_FLAGS}" echo "" echo "Key Versions:" + echo " BLT: ${BLT_VER}" echo " CAMP: ${CAMP_VER}" echo " RAJA: ${RAJA_VER}" - if [ "${BUILD_TYPE}" != "cpu" ]; then - echo " Umpire: ${UMPIRE_VER}" - echo " CHAI: ${CHAI_VER}" - fi + echo " Umpire: ${UMPIRE_VER}" + echo " CHAI: ${CHAI_VER}" echo " Hypre: ${HYPRE_VER}" echo " MFEM: ${MFEM_BRANCH}" + echo " SNLS: ${SNLS_VER}" + echo " HDF5: ${HDF5_VER}" + echo " Conduit: ${CONDUIT_VER}" + echo " Axom: ${AXOM_VER}" echo " ExaCMech: ${EXACMECH_BRANCH}" echo " ExaConstit: ${EXACONSTIT_BRANCH}" echo "==========================================" @@ -131,19 +134,19 @@ print_build_summary() { # Validate configuration before proceeding validate_configuration() { echo "Validating configuration..." - + # Check compilers exist check_required_paths "${CMAKE_C_COMPILER}" "${CMAKE_CXX_COMPILER}" - + if [ "${BUILD_TYPE}" != "cpu" ]; then check_required_paths "${CMAKE_GPU_COMPILER}" fi - + # Check MPI wrappers check_required_paths "${MPI_C_COMPILER}" "${MPI_CXX_COMPILER}" "${MPI_Fortran_COMPILER}" - + # Check required commands check_required_commands git cmake make curl tar - + echo "Configuration validation complete." -} \ No newline at end of file +} diff --git a/scripts/install/configs/cpu_intel_config.sh b/scripts/install/configs/cpu_intel_config.sh index a0dfb046..51f4fe60 100644 --- a/scripts/install/configs/cpu_intel_config.sh +++ b/scripts/install/configs/cpu_intel_config.sh @@ -71,7 +71,7 @@ export CMAKE_GPU_ARCHITECTURES="" export CMAKE_GPU_FLAGS="" ########################################### -# CHAI Options (Not Used in CPU Build) +# CHAI Options (CPU build: GPU-specific features disabled) ########################################### export CHAI_DISABLE_RM="OFF" export CHAI_THIN_GPU_ALLOCATE="OFF" @@ -80,5 +80,7 @@ export CHAI_ENABLE_PICK="OFF" export CHAI_DEBUG="OFF" export CHAI_ENABLE_GPU_SIMULATION_MODE="OFF" export CHAI_ENABLE_UM="OFF" -export CHAI_ENABLE_MANAGED_PTR="OFF" +# managed_ptr is required on CPU too: SNLS (batch solvers) and ExaCMech +# use chai::managed_ptr whenever SNLS is built with the RAJA Portability Suite. +export CHAI_ENABLE_MANAGED_PTR="ON" export CHAI_ENABLE_MANAGED_PTR_ON_GPU="OFF" \ No newline at end of file diff --git a/scripts/install/configs/cpu_mac_config.sh b/scripts/install/configs/cpu_mac_config.sh index b2598c87..3119f1f7 100644 --- a/scripts/install/configs/cpu_mac_config.sh +++ b/scripts/install/configs/cpu_mac_config.sh @@ -97,7 +97,7 @@ export CMAKE_GPU_ARCHITECTURES="" export CMAKE_GPU_FLAGS="" ########################################### -# CHAI Options (Not Used in CPU Build) +# CHAI Options (CPU build: GPU-specific features disabled) ########################################### export CHAI_DISABLE_RM="OFF" export CHAI_THIN_GPU_ALLOCATE="OFF" @@ -106,7 +106,9 @@ export CHAI_ENABLE_PICK="OFF" export CHAI_DEBUG="OFF" export CHAI_ENABLE_GPU_SIMULATION_MODE="OFF" export CHAI_ENABLE_UM="OFF" -export CHAI_ENABLE_MANAGED_PTR="OFF" +# managed_ptr is required on CPU too: SNLS (batch solvers) and ExaCMech +# use chai::managed_ptr whenever SNLS is built with the RAJA Portability Suite. +export CHAI_ENABLE_MANAGED_PTR="ON" export CHAI_ENABLE_MANAGED_PTR_ON_GPU="OFF" ########################################### diff --git a/scripts/restart/compare_restart_checkpoints.cpp b/scripts/restart/compare_restart_checkpoints.cpp new file mode 100644 index 00000000..95c1db77 --- /dev/null +++ b/scripts/restart/compare_restart_checkpoints.cpp @@ -0,0 +1,530 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +// Diagnostic and test tool: compares two ExaConstit checkpoint directories +// (/cycle_/, see doc/restart.md) group by group and view by view. +// +// Usage: compare_restart_checkpoints [--rtol ] +// +// Both directories must hold the same set of rank files. The commit file and every rank file +// are loaded in full, including the arrays stored as external views, and every group and view +// is compared: names, types, sizes and contents. A quadrature function stored as `copy_of` +// another one (see RestartCheckpoint::WriteQuadratureFunctions()) is resolved to that +// function's values first, so a begin/end pair stored as a copy in one checkpoint and as full +// values in the other still compares equal, as it should, since both restore the same state. +// +// Without --rtol the comparison is bitwise. With --rtol, double arrays may differ by +// rtol * max(|a|, |b|) per entry; everything else must still match exactly. +// +// Exit status: 0 = the checkpoints match, 1 = they differ, 2 = usage or I/O error. +// test/test_restart_runs.py relies on the exit status and on the last line of the output, +// which is always a one-line summary. + +namespace fs = std::filesystem; + +namespace { + +using axom::sidre::DataStore; +using axom::sidre::Group; +using axom::sidre::View; + +/** + * @brief Running totals of a comparison + */ +struct ComparisonTally { + /** @brief Relative tolerance for double arrays (0 = bitwise) */ + double rtol = 0.0; + /** @brief Number of views (or resolved quadrature functions) compared */ + long views = 0; + /** @brief Number of mismatches found */ + long mismatches = 0; + /** @brief Largest absolute difference seen between double entries */ + double max_abs = 0.0; + /** @brief Largest relative difference seen between double entries */ + double max_rel = 0.0; +}; + +/** + * @brief Encode a registry name the same way restart_checkpoint.cpp does + * + * @param name Registry name + * @return The name with '%' encoded as "%25" and '/' as "%2F" + * + * @details Needed to find the group a `copy_of` entry refers to, since `copy_of` holds the + * plain registry name while the groups are named with the encoded form. + */ +std::string SidreSafeName(const std::string& name) { + std::string encoded; + for (const char character : name) { + if (character == '%') { + encoded += "%25"; + } else if (character == '/') { + encoded += "%2F"; + } else { + encoded += character; + } + } + return encoded; +} + +/** + * @brief A Sidre file loaded in full, including the data of its external views + * + * @details Group::load() restores an external view as a description only, so this allocates + * a buffer for every external view, points the view at it, and then reads the data with + * Group::loadExternalData(), the same two-step read restart_checkpoint.cpp uses. Zero-length + * external views are described but have no data, so they are removed before the read and + * remembered by path, so the sets of empty arrays in two files can still be compared. + */ +class LoadedFile { +public: + /** + * @brief Load a Sidre file and all of its external data + * + * @param path File to load + * @param protocol Sidre protocol of the file ("sidre_hdf5" or "sidre_json") + * @return True if the file and its external data were read + */ + bool Load(const fs::path& path, const char* protocol) { + Group* root = m_store.getRoot(); + if (!root->load(path.string(), protocol)) { + return false; + } + std::vector> empty_views; + BindExternalViews(root, empty_views); + for (auto& [group, name] : empty_views) { + group->destroyView(name); + } + return m_external_buffers.empty() || root->loadExternalData(path.string()); + } + + /** + * @brief Root group of the loaded file + * + * @return The root group + */ + Group* Root() { + return m_store.getRoot(); + } + + /** + * @brief Paths of the zero-length external views removed during loading + * + * @return The set of paths + */ + const std::set& EmptyExternalViews() const { + return m_empty_paths; + } + +private: + /** + * @brief Point every non-empty external view at a newly allocated buffer + * + * @param group Group to process, recursively + * @param empty_views Receives the zero-length external views, to be removed by the caller + */ + void BindExternalViews(Group* group, std::vector>& empty_views) { + for (View& view : group->views()) { + if (!view.isExternal()) { + continue; + } + const std::size_t bytes = static_cast(view.getTotalBytes()); + if (bytes == 0) { + empty_views.emplace_back(group, view.getName()); + m_empty_paths.insert(view.getPathName()); + continue; + } + m_external_buffers.emplace_back(bytes); + view.setExternalDataPtr(m_external_buffers.back().data()); + } + for (Group& child : group->groups()) { + BindExternalViews(&child, empty_views); + } + } + + /** @brief DataStore holding the loaded file */ + DataStore m_store; + /** @brief Buffers the external views were pointed at */ + std::vector> m_external_buffers; + /** @brief Paths of the zero-length external views */ + std::set m_empty_paths; +}; + +/** + * @brief Print one mismatch and count it + * + * @param tally Running totals + * @param file File the mismatch was found in + * @param path Path of the group or view within the file + * @param message Description of the mismatch + */ +void ReportMismatch(ComparisonTally& tally, + const std::string& file, + const std::string& path, + const std::string& message) { + std::cout << " MISMATCH [" << file << "] " << path << ": " << message << std::endl; + ++tally.mismatches; +} + +/** + * @brief Compare two double arrays + * + * @param a First array + * @param b Second array + * @param count Number of entries in each + * @param tally Running totals (tolerance in, largest differences updated) + * @param message Receives a description of the differences when the arrays do not match + * @return True if the arrays match: bitwise when tally.rtol is 0, otherwise every entry within + * rtol * max(|a|, |b|) + * + * @details Entries with identical bits always match, which also makes two identical NaNs + * match. The description gives the number of differing entries, the largest absolute and + * relative differences, and the first differing entry, printed with 17 significant digits so + * the two values can be told apart. + */ +bool CompareDoubles(const double* a, + const double* b, + const long count, + ComparisonTally& tally, + std::string& message) { + if (count == 0) { + return true; + } + if (tally.rtol == 0.0 && + std::memcmp(a, b, static_cast(count) * sizeof(double)) == 0) { + return true; + } + long differing = 0; + long first_differing = -1; + double max_abs = 0.0; + double max_rel = 0.0; + for (long i = 0; i < count; ++i) { + if (std::memcmp(&a[i], &b[i], sizeof(double)) == 0) { + continue; + } + const double difference = std::abs(a[i] - b[i]); + const double scale = std::max({std::abs(a[i]), std::abs(b[i]), 1e-300}); + max_abs = std::max(max_abs, difference); + max_rel = std::max(max_rel, difference / scale); + if (tally.rtol == 0.0 || !(difference <= tally.rtol * scale)) { + if (first_differing < 0) { + first_differing = i; + } + ++differing; + } + } + tally.max_abs = std::max(tally.max_abs, max_abs); + tally.max_rel = std::max(tally.max_rel, max_rel); + if (differing == 0) { + return true; + } + std::ostringstream description; + description << differing << " of " << count << " entries differ (max abs " << std::scientific + << std::setprecision(3) << max_abs << ", max rel " << max_rel << "; first at " + << first_differing << ": " << std::defaultfloat << std::setprecision(17) + << a[first_differing] << " vs " << b[first_differing] << ")"; + message = description.str(); + return false; +} + +/** + * @brief Compare two views with the same path + * + * @param a View from the first file + * @param b View from the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Strings must be equal, other views must have the same type and number of entries, + * double arrays are compared with CompareDoubles(), and everything else byte for byte. + */ +void CompareViews(View* a, View* b, const std::string& file, ComparisonTally& tally) { + ++tally.views; + const std::string path = a->getPathName(); + if (a->getTypeID() != b->getTypeID()) { + ReportMismatch(tally, file, path, "the types differ"); + return; + } + if (a->isString() || b->isString()) { + if (!a->isString() || !b->isString() || std::string(a->getString()) != b->getString()) { + ReportMismatch(tally, + file, + path, + std::string("the strings differ: '") + + (a->isString() ? a->getString() : "?") + "' vs '" + + (b->isString() ? b->getString() : "?") + "'"); + } + return; + } + if (a->getNumElements() != b->getNumElements()) { + ReportMismatch(tally, + file, + path, + "the entry counts differ: " + std::to_string(a->getNumElements()) + " vs " + + std::to_string(b->getNumElements())); + return; + } + const long count = static_cast(a->getNumElements()); + if (count == 0) { + return; + } + const void* data_a = a->getVoidPtr(); + const void* data_b = b->getVoidPtr(); + if (data_a == nullptr || data_b == nullptr) { + ReportMismatch(tally, file, path, "no data was loaded"); + return; + } + if (a->getTypeID() == axom::sidre::FLOAT64_ID) { + std::string message; + if (!CompareDoubles(static_cast(data_a), + static_cast(data_b), + count, + tally, + message)) { + ReportMismatch(tally, file, path, message); + } + return; + } + if (std::memcmp(data_a, data_b, static_cast(a->getTotalBytes())) != 0) { + ReportMismatch(tally, file, path, "the contents differ"); + } +} + +/** + * @brief The values view of a quadrature function entry, following `copy_of` if present + * + * @param qfuncs The `qfuncs/` group + * @param entry One quadrature function's group + * @return The view holding the function's values, or nullptr if it cannot be found + */ +View* ResolveQuadratureFunctionValues(Group* qfuncs, Group* entry) { + if (entry->hasView("copy_of")) { + const std::string source = SidreSafeName(entry->getView("copy_of")->getString()); + if (!qfuncs->hasGroup(source)) { + return nullptr; + } + Group* source_entry = qfuncs->getGroup(source); + return source_entry->hasView("values") ? source_entry->getView("values") : nullptr; + } + return entry->hasView("values") ? entry->getView("values") : nullptr; +} + +/** + * @brief Compare the `qfuncs/` groups of two rank files + * + * @param a `qfuncs/` group of the first file + * @param b `qfuncs/` group of the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Both files must have the same quadrature functions with the same space name, vdim + * and value count. The values are compared after resolving `copy_of`, so how each file chose + * to store a function does not matter, only what it restores. A function with no values and + * no `copy_of` in both files is an empty function whose zero-length view was removed while + * loading, and has nothing to compare. + */ +void CompareQuadratureFunctions(Group* a, + Group* b, + const std::string& file, + ComparisonTally& tally) { + std::set names; + for (Group& group : a->groups()) { + names.insert(group.getName()); + } + for (Group& group : b->groups()) { + names.insert(group.getName()); + } + for (const std::string& name : names) { + const std::string path = a->getPathName() + "/" + name; + if (!a->hasGroup(name) || !b->hasGroup(name)) { + ReportMismatch(tally, + file, + path, + std::string("only in the ") + (a->hasGroup(name) ? "first" : "second") + + " checkpoint"); + continue; + } + Group* entry_a = a->getGroup(name); + Group* entry_b = b->getGroup(name); + for (const char* metadata : {"qspace_name", "vdim", "value_count"}) { + if (entry_a->hasView(metadata) != entry_b->hasView(metadata)) { + ReportMismatch( + tally, file, path + "/" + metadata, "present in only one checkpoint"); + } else if (entry_a->hasView(metadata)) { + CompareViews(entry_a->getView(metadata), entry_b->getView(metadata), file, tally); + } + } + + const bool empty_a = !entry_a->hasView("copy_of") && !entry_a->hasView("values"); + const bool empty_b = !entry_b->hasView("copy_of") && !entry_b->hasView("values"); + if (empty_a && empty_b) { + continue; + } + View* values_a = ResolveQuadratureFunctionValues(a, entry_a); + View* values_b = ResolveQuadratureFunctionValues(b, entry_b); + if (values_a == nullptr || values_b == nullptr) { + ReportMismatch(tally, file, path, "the values could not be resolved"); + continue; + } + ++tally.views; + if (values_a->getNumElements() != values_b->getNumElements()) { + ReportMismatch(tally, file, path, "the value counts differ"); + continue; + } + std::string message; + if (!CompareDoubles(static_cast(values_a->getVoidPtr()), + static_cast(values_b->getVoidPtr()), + static_cast(values_a->getNumElements()), + tally, + message)) { + ReportMismatch(tally, file, path, message); + } + } +} + +/** + * @brief Compare two groups recursively + * + * @param a Group from the first file + * @param b Group from the second file + * @param file File name, for the mismatch messages + * @param tally Running totals + * + * @details Both groups must have the same views and child groups. The top-level `qfuncs/` + * group is compared with CompareQuadratureFunctions() so that `copy_of` entries are resolved. + */ +void CompareGroups(Group* a, Group* b, const std::string& file, ComparisonTally& tally) { + std::set views; + for (View& view : a->views()) { + views.insert(view.getName()); + } + for (View& view : b->views()) { + views.insert(view.getName()); + } + for (const std::string& name : views) { + if (!a->hasView(name) || !b->hasView(name)) { + ReportMismatch(tally, + file, + a->getPathName() + "/" + name, + std::string("view only in the ") + + (a->hasView(name) ? "first" : "second") + " checkpoint"); + continue; + } + CompareViews(a->getView(name), b->getView(name), file, tally); + } + + std::set groups; + for (Group& group : a->groups()) { + groups.insert(group.getName()); + } + for (Group& group : b->groups()) { + groups.insert(group.getName()); + } + for (const std::string& name : groups) { + if (!a->hasGroup(name) || !b->hasGroup(name)) { + ReportMismatch(tally, + file, + a->getPathName() + "/" + name, + std::string("group only in the ") + + (a->hasGroup(name) ? "first" : "second") + " checkpoint"); + continue; + } + if (name == "qfuncs" && a->isRoot()) { + CompareQuadratureFunctions(a->getGroup(name), b->getGroup(name), file, tally); + } else { + CompareGroups(a->getGroup(name), b->getGroup(name), file, tally); + } + } +} + +/** + * @brief Rank files in a checkpoint directory + * + * @param directory Checkpoint directory + * @return Names of the rank_*.hdf5 files, sorted + */ +std::vector RankFiles(const fs::path& directory) { + std::vector files; + std::error_code error; + for (const auto& item : fs::directory_iterator(directory, error)) { + const std::string name = item.path().filename().string(); + if (name.rfind("rank_", 0) == 0 && item.path().extension() == ".hdf5") { + files.push_back(name); + } + } + std::sort(files.begin(), files.end()); + return files; +} + +} // namespace + +int main(int argc, char** argv) { + ComparisonTally tally; + std::vector directories; + for (int i = 1; i < argc; ++i) { + const std::string argument = argv[i]; + if (argument == "--rtol" && i + 1 < argc) { + tally.rtol = std::atof(argv[++i]); + } else { + directories.push_back(argument); + } + } + if (directories.size() != 2) { + std::cerr << "usage: " << argv[0] << " [--rtol ]" + << std::endl; + return 2; + } + const fs::path directory_a = directories[0]; + const fs::path directory_b = directories[1]; + + const std::vector files_a = RankFiles(directory_a); + const std::vector files_b = RankFiles(directory_b); + if (files_a.empty() || files_a != files_b) { + std::cout << "ERROR: the rank files differ or are missing (" << files_a.size() << " in " + << directory_a.string() << ", " << files_b.size() << " in " + << directory_b.string() << ")" << std::endl; + return 2; + } + + { + LoadedFile commit_a; + LoadedFile commit_b; + if (!commit_a.Load(directory_a / "commit.json", "sidre_json") || + !commit_b.Load(directory_b / "commit.json", "sidre_json")) { + std::cout << "ERROR: unable to load commit.json in both directories" << std::endl; + return 2; + } + CompareGroups(commit_a.Root(), commit_b.Root(), "commit.json", tally); + } + for (const std::string& file : files_a) { + LoadedFile file_a; + LoadedFile file_b; + if (!file_a.Load(directory_a / file, "sidre_hdf5") || + !file_b.Load(directory_b / file, "sidre_hdf5")) { + std::cout << "ERROR: unable to load " << file << " in both directories" << std::endl; + return 2; + } + if (file_a.EmptyExternalViews() != file_b.EmptyExternalViews()) { + ReportMismatch(tally, file, "/", "the sets of empty arrays differ"); + } + CompareGroups(file_a.Root(), file_b.Root(), file, tally); + } + + std::cout << (tally.mismatches == 0 ? "MATCH" : "DIFFER") << ": " << files_a.size() + << " rank files, " << tally.views << " views compared, " << tally.mismatches + << " mismatches (max abs " << std::scientific << std::setprecision(3) << tally.max_abs + << ", max rel " << tally.max_rel << ")" << (tally.rtol == 0.0 ? ", bitwise" : "") + << std::endl; + return tally.mismatches == 0 ? 0 : 1; +} diff --git a/scripts/restart/inspect_restart_rank_file.cpp b/scripts/restart/inspect_restart_rank_file.cpp new file mode 100644 index 00000000..761ee9f1 --- /dev/null +++ b/scripts/restart/inspect_restart_rank_file.cpp @@ -0,0 +1,54 @@ +#include +#include +#include + +#include "axom/sidre.hpp" + +// Diagnostic tool: lists the top-level groups of an ExaConstit checkpoint rank file and how +// each quadrature function is stored in it, either as values (with their count) or as +// `copy_of` another function (see RestartCheckpoint::WriteQuadratureFunctions()). +// +// Usage: inspect_restart_rank_file /cycle_/rank_.hdf5 +// +// Only the file's structure is read; the stored arrays themselves are not loaded. + +int main(int argc, char** argv) { + if (argc < 2) { + std::cerr << "usage: " << argv[0] << " " << std::endl; + return EXIT_FAILURE; + } + + axom::sidre::DataStore store; + axom::sidre::Group* root = store.getRoot(); + if (!root->load(argv[1], "sidre_hdf5")) { + std::cerr << "Unable to load " << argv[1] << " as a Sidre HDF5 file" << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Top-level groups:"; + for (axom::sidre::Group& group : root->groups()) { + std::cout << " " << group.getName(); + } + std::cout << std::endl; + + if (!root->hasGroup("qfuncs")) { + std::cout << "No qfuncs/ group; this is not an ExaConstit checkpoint rank file" + << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Quadrature functions:" << std::endl; + for (axom::sidre::Group& entry : root->getGroup("qfuncs")->groups()) { + std::string storage; + if (entry.hasView("copy_of")) { + storage = std::string("copy_of ") + entry.getView("copy_of")->getString(); + } else if (entry.hasView("values")) { + storage = "values " + std::to_string(entry.getView("values")->getNumElements()) + + " doubles"; + } else { + storage = "(no values)"; + } + std::cout << " " << entry.getName() << ": " << storage << std::endl; + } + return EXIT_SUCCESS; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af8cd1ab..ccf94fab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,8 @@ set(EXACONSTIT_HEADERS postprocessing/postprocessing_driver.hpp postprocessing/mechanics_lightup.hpp sim_state/simulation_state.hpp + sim_state/exa_sidre_parmesh.hpp + sim_state/restart_checkpoint.hpp solvers/mechanics_solver.hpp solvers/trust_region_solver.hpp utilities/dynamic_function_loader.hpp @@ -53,12 +55,15 @@ set(EXACONSTIT_SOURCES options/option_material.cpp options/option_mesh.cpp options/option_post_processing.cpp + options/option_restart.cpp options/option_solvers.cpp options/option_time.cpp postprocessing/postprocessing_driver.cpp postprocessing/projection_class.cpp postprocessing/mechanics_lightup.cpp sim_state/simulation_state.cpp + sim_state/exa_sidre_parmesh.cpp + sim_state/restart_checkpoint.cpp solvers/mechanics_solver.cpp solvers/trust_region_solver.cpp utilities/mechanics_kernels.cpp @@ -114,6 +119,10 @@ if(ENABLE_CALIPER) list(APPEND EXACONSTIT_DEPENDS caliper) endif() +if(ENABLE_AXOM) + list(APPEND EXACONSTIT_DEPENDS axom) +endif() + if(UNIX AND NOT APPLE AND TARGET Threads::Threads) # Check if we determined explicit linking is needed if(EXACONSTIT_THREADS_EXPLICIT_LINK) @@ -132,6 +141,9 @@ set(EXACONSTIT_DEFINES HAVE_EXACONSTIT) if(ENABLE_CALIPER) list(APPEND EXACONSTIT_DEFINES HAVE_CALIPER) endif() +if(ENABLE_AXOM) + list(APPEND EXACONSTIT_DEFINES EXACONSTIT_USE_AXOM) +endif() #------------------------------------------------------------------------------ # Includes #------------------------------------------------------------------------------ diff --git a/src/mechanics_driver.cpp b/src/mechanics_driver.cpp index 4f3efcad..339bde8f 100644 --- a/src/mechanics_driver.cpp +++ b/src/mechanics_driver.cpp @@ -88,6 +88,11 @@ #include "SNLS_config.h" #if defined(SNLS_RAJA_PORT_SUITE) +#include "SNLS_device_forall.h" +#include "SNLS_memory_manager.h" + +#include +#include #include #endif @@ -203,6 +208,38 @@ int main(int argc, char* argv[]) { device_config = "raja-hip"; #endif } + /* + * Umpire Memory Pools: + * - MFEM and SNLS (which ExaCMech allocates through) share one host and one device + * Umpire pool instead of each creating their own. + * - SNLS's execution strategy follows rtmodel, so that runs on the CPU allocate the + * ExaCMech model data on the host. + */ +#if defined(SNLS_RAJA_PORT_SUITE) + if (toml_opt.solvers.rtmodel == RTModel::GPU) { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::GPU); + } else if (toml_opt.solvers.rtmodel == RTModel::OPENMP) { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::OPENMP); + } else { + snls::Device::GetInstance().SetBackend(snls::ExecutionStrategy::CPU); + } +#if defined(MFEM_USE_UMPIRE) + { + auto& rm = umpire::ResourceManager::getInstance(); + auto host_pool = rm.makeAllocator( + "ExaConstit_HOST_pool", rm.getAllocator("HOST")); + mfem::MemoryManager::SetUmpireHostAllocatorName("ExaConstit_HOST_pool"); + snls::memoryManager::getInstance().setHostAllocator(host_pool.getId()); +#if defined(__snls_gpu_active__) + auto device_pool = rm.makeAllocator( + "ExaConstit_DEVICE_pool", rm.getAllocator("DEVICE")); + mfem::MemoryManager::SetUmpireDeviceAllocatorName("ExaConstit_DEVICE_pool"); + snls::memoryManager::getInstance().setDeviceAllocator(device_pool.getId()); +#endif + } +#endif +#endif + /* * MFEM Device Configuration: * - Configure device memory management for host/device data movement @@ -318,6 +355,39 @@ int main(int argc, char* argv[]) { * - Performs material state updates and post-processing at each step */ int ti = 0; + if (sim_state->IsRestartRun()) { + // Restart phase 2. Every object above has been built exactly as in a fresh run, so + // the checkpoint now overwrites the evolving state (coordinates, velocity, state + // variables, stresses and time). Whatever the constructors initialized, such as + // ExaCMechModel::InitStateVars(), is simply replaced. + sim_state->RestoreFromCheckpoint(); + ti = static_cast(sim_state->GetRestartDriverStep()); + + // Bring the boundary conditions to the segment that was active after step ti. + // + // SystemDriver's constructor set up the boundary conditions of step 1. In an + // uninterrupted run, each later update step s calls GetUpdateStep(s), which makes s + // BCManager's current step, and then UpdateEssBdr() rebuilds the boundary data and + // essential DOFs for it (followed by UpdateVelocity() and SolveInit()). + // GetUpdateStep(s) has no other side effects and only changes the current step when + // s is an update step, so replaying it for s = 1..ti leaves BCManager on the last + // update step <= ti, and UpdateEssBdr() then rebuilds that segment's boundary data. + // + // UpdateVelocity() and SolveInit() are not repeated. UpdateVelocity() is called by + // the loop before every Solve(). SolveInit() was the corrector applied at the start of + // that segment, and its effect is already part of the restored velocity. If ti + 1 + // is itself an update step, the loop below handles it exactly as an uninterrupted + // run would. + int last_bc_update_step = 0; + for (int step = 1; step <= ti; ++step) { + if (BCManager::GetInstance().GetUpdateStep(step)) { + last_bc_update_step = step; + } + } + if (last_bc_update_step > 0) { + oper.UpdateEssBdr(); + } + } auto v_sol = sim_state->GetPrimalField(); while (!sim_state->IsFinished()) { ti++; @@ -367,6 +437,7 @@ int main(int argc, char* argv[]) { sim_state->FinishCycle(); oper.UpdateModel(); post_process.Update(ti, sim_state->GetTrueCycleTime()); + sim_state->WriteCheckpointIfDue(static_cast(ti)); } // end loop over time steps /** diff --git a/src/mfem_expt/partial_qfunc.hpp b/src/mfem_expt/partial_qfunc.hpp index 5e6bb83c..d53d82de 100644 --- a/src/mfem_expt/partial_qfunc.hpp +++ b/src/mfem_expt/partial_qfunc.hpp @@ -165,6 +165,35 @@ class PartialQuadratureFunction : public QuadratureFunction { */ PartialQuadratureFunction& operator=(const QuadratureFunction& qf); + /** + * @brief Copy constructor: a new function on the same partial space with a copy of the data. + * + * @details Declared (defaulted) explicitly because the copy assignment below is + * user-provided; the behaviour is that of the former implicit copy constructor. + */ + PartialQuadratureFunction(const PartialQuadratureFunction&) = default; + + /** + * @brief Copy assignment with value semantics: copies the DATA only. + * @param rhs Function on an identically laid-out partial space (same vdim and size) + * @return Reference to this PartialQuadratureFunction + * + * @details The implicitly-declared version relied on mfem::QuadratureFunction's + * implicitly-declared copy assignment, which is deprecated because QuadratureFunction has a + * user-provided copy constructor (-Wdeprecated-copy), and it also re-pointed this function's + * space members at the source's. It is reached from SimulationState::SetupModelVariables + * (every registered begin/end pair) and ExaCMechModel::InitStateVars; both copy between + * functions on the same space, for which data-only copying is exactly the intended operation. + */ + PartialQuadratureFunction& operator=(const PartialQuadratureFunction& rhs) { + MFEM_ASSERT(rhs.GetVDim() == vdim && rhs.Size() == this->Size(), + "PartialQuadratureFunction copy assignment requires identical layouts"); + if (this != &rhs) { + QuadratureFunction::operator=(static_cast(rhs)); + } + return *this; + } + /** * @brief Fill a global QuadratureFunction with data from this partial function. * diff --git a/src/options.toml b/src/options.toml index 96f8e127..2b47b330 100644 --- a/src/options.toml +++ b/src/options.toml @@ -335,13 +335,6 @@ grain_file = "grains.txt" # 3. Fixed (if [Time.Fixed] exists or as default) time_type = "custom" - # ===== Restart Options (Optional) ===== - # Continue from a previous simulation - # Currently ignored but will be coming in a near-future release - # restart = true - # restart_time = 0.5 # Time to restart from - # restart_cycle = 100 # Cycle number to restart from - # ===== Automatic Time Stepping ===== # Adjusts time step based on convergence behavior # After each converged step dt is multiplied by @@ -642,7 +635,7 @@ grain_file = "grains.txt" # [output_directory]/[basename]/ <- Main simulation folder # [output_directory]/[basename]/avg_*.txt <- Volume average files # [output_directory]/[basename]/visualizations/ <- Visualization files - # [output_directory]/[basename]/restart/ <- Restart files (if enabled) + # [output_directory]/[basename]/restart/ <- Checkpoints ([Restart]) # # Example: If output_directory = "./results" and basename = "tensile_test": # ./results/tensile_test/ <- All outputs go here @@ -900,4 +893,56 @@ grain_file = "grains.txt" # Number of elements [X_elements, Y_elements, Z_elements] # Total elements = product of these numbers - nxyz = [10, 10, 10] \ No newline at end of file + nxyz = [10, 10, 10] + +# ===================================== +# CHECKPOINT / RESTART +# ===================================== +# Writes binary checkpoints during a run and restarts later runs from them. +# Requires ExaConstit built with Axom (Axom with Sidre, Conduit and HDF5; the +# install scripts build all three). See doc/restart.md for the full details. +# +# A restart needs only the checkpoint and this options file: the mesh, grain, +# region-mapping, orientation and state-variable files are not read. +# A restart must use the same number of MPI ranks, the same mesh order and the +# same materials (names, model types and state variable counts) as the run that +# wrote the checkpoint. The final time, time steps, solver settings, output +# settings and material properties may change; boundary conditions for steps +# already completed must stay the same. + +[Restart] + # Write checkpoints during this run + write = false + + # Write a checkpoint every this many completed time steps + # 0 = only write the final checkpoint (if write_final = true) + interval = 0 + + # Also write a checkpoint when the simulation finishes + write_final = true + + # Number of checkpoints to keep on disk; older ones are removed + # 0 = keep every checkpoint + keep_last = 2 + + # Directory checkpoints are written to, one subdirectory per checkpoint + # (cycle_). A relative path is inside the run's + # output folder, [output_directory]/[basename] (see [PostProcessing.volume_averages] + # output_directory), next to the other outputs: by default + # ./results//restart. A restart with the same output_directory and + # basename finds its checkpoints there. + directory = "restart" + + # Restart this run from a checkpoint + read = false + + # Which checkpoint to restart from, by number of completed steps + # If omitted, the newest complete checkpoint is used + # read_cycle = 100 + + # Read the checkpoint from this directory instead of `directory` + # Useful for restarting from another run's checkpoints without touching + # them; this run still writes its own checkpoints to `directory`. A relative + # path is relative to the directory ExaConstit is started in (not the output + # folder), e.g. another run's checkpoints: + # read_directory = "results/other_run/restart" diff --git a/src/options/option_material.cpp b/src/options/option_material.cpp index d93a06f4..cabef2eb 100644 --- a/src/options/option_material.cpp +++ b/src/options/option_material.cpp @@ -297,15 +297,16 @@ std::vector MaterialOptions::from_toml_array(const toml::value& return materials; } -bool GrainInfo::validate() const { - // Implement validation logic - if (!orientation_file) { +bool GrainInfo::validate(bool check_input_files) const { + // On a restart the orientations come from the checkpointed state variables, so the + // orientation file is neither required nor read. + if (check_input_files && !orientation_file) { WARNING_0_OPT("Error: Grain table was provided without providing an orientation file this " "is required"); return false; } - if (orientation_file) { + if (check_input_files && orientation_file) { if (!std::filesystem::exists(*orientation_file)) { std::ostringstream err; err << "Error: Orientation file does not exist provided value: " << *orientation_file; @@ -354,8 +355,10 @@ bool MaterialProperties::validate() const { return true; } -bool StateVariables::validate() const { - if (static_cast(num_vars) != initial_values.size()) { +bool StateVariables::validate(bool check_input_files) const { + // initial_values only seed a fresh run; on a restart the state comes from the checkpoint + // (a deleted state file leaves initial_values empty after StateVariables::from_toml). + if (check_input_files && static_cast(num_vars) != initial_values.size()) { WARNING_0_OPT("Error: StateVariables num_vars != initial_values.size()"); return false; } @@ -422,7 +425,7 @@ bool MaterialModelOptions::validate() const { return true; } -bool MaterialOptions::validate() const { +bool MaterialOptions::validate(bool check_input_files) const { std::string mat_name = material_name + "_" + std::to_string(region_id); if (mech_type == MechType::NOTYPE) { @@ -448,7 +451,7 @@ bool MaterialOptions::validate() const { WARNING_0_OPT(err.str()); return false; } - if (!state_vars.validate()) { + if (!state_vars.validate(check_input_files)) { std::ostringstream err; err << "Error: Material table for material_name_region# " << mat_name << " the State_Vars table had errors"; @@ -464,7 +467,7 @@ bool MaterialOptions::validate() const { } if (grain_info) { - if (!grain_info->validate()) { + if (!grain_info->validate(check_input_files)) { std::ostringstream err; err << "Error: Material table for material_name_region# " << mat_name << " the Grain table had errors"; diff --git a/src/options/option_mesh.cpp b/src/options/option_mesh.cpp index 684f1384..d6cfece6 100644 --- a/src/options/option_mesh.cpp +++ b/src/options/option_mesh.cpp @@ -61,7 +61,7 @@ MeshOptions MeshOptions::from_toml(const toml::value& toml_input) { return options; } -bool MeshOptions::validate() const { +bool MeshOptions::validate(bool check_input_files) const { if (mesh_type == MeshType::NOTYPE) { WARNING_0_OPT("Error: Mesh table was not provided an appropriate mesh type"); return false; @@ -87,8 +87,9 @@ bool MeshOptions::validate() const { } } - // Check that mesh file exists for CUBIT or OTHER mesh types - if ((mesh_type == MeshType::FILE) && !mesh_file.empty()) { + // Check that mesh file exists for CUBIT or OTHER mesh types. A restart rebuilds the mesh + // from the checkpoint, so the file is not needed (or checked) then. + if (check_input_files && (mesh_type == MeshType::FILE) && !mesh_file.empty()) { if (!fs::exists(mesh_file)) { std::ostringstream err; err << "Error: Mesh file '" << mesh_file << "' does not exist." << std::endl; diff --git a/src/options/option_parser_v2.cpp b/src/options/option_parser_v2.cpp index 4f030d74..e301e5a9 100644 --- a/src/options/option_parser_v2.cpp +++ b/src/options/option_parser_v2.cpp @@ -203,6 +203,20 @@ void ExaOptions::parse_options(const std::string& filename, int my_id) { resolved_str.resize(static_cast(length)); MPI_Bcast(resolved_str.data(), length, MPI_CHAR, 0, MPI_COMM_WORLD); output_dir = resolved_str; + + // Checkpoints are part of this run's output: a relative [Restart] directory is taken inside + // the run's output folder, /, next to its logs, volume averages + // and visualizations. read_directory names checkpoints written elsewhere (often by another + // run), so a relative read_directory stays relative to the working directory. + if (restart.directory.is_relative()) { + restart.directory = fs::path(output_dir) / basename / restart.directory; + } + // Checked here rather than in RestartOptions::validate(), which runs before the directory + // is resolved. Which checkpoint inside it is used, and whether it is complete, is decided + // when the simulation starts. + VERIFY_0_OPT(!restart.read || fs::is_directory(restart.GetReadDirectory()), + "Restart table has `read` = true, but the checkpoint directory '" + << restart.GetReadDirectory().string() << "' does not exist."); } void ExaOptions::parse_from_toml(const toml::value& toml_input) { @@ -242,6 +256,7 @@ void ExaOptions::parse_from_toml(const toml::value& toml_input) { // Parse component sections parse_mesh_options(toml_input); parse_time_options(toml_input); + parse_restart_options(toml_input); parse_solver_options(toml_input); parse_boundary_options(toml_input); parse_visualization_options(toml_input); @@ -277,6 +292,12 @@ void ExaOptions::parse_time_options(const toml::value& toml_input) { time = TimeOptions::from_toml(toml::find(toml_input, "Time")); } +void ExaOptions::parse_restart_options(const toml::value& toml_input) { + if (toml_input.contains("Restart")) { + restart = RestartOptions::from_toml(toml::find(toml_input, "Restart")); + } +} + void ExaOptions::parse_solver_options(const toml::value& toml_input) { if (toml_input.contains("Solvers")) { solvers = SolverOptions::from_toml(toml::find(toml_input, "Solvers")); @@ -466,7 +487,11 @@ void ExaOptions::load_post_processing_file() { bool ExaOptions::validate() { // Basic validation - could be expanded with more comprehensive checks - if (!mesh.validate()) + // On a restart, the mesh, grain, region-mapping, orientation and state-variable inputs + // are replaced by the checkpoint and may no longer exist. + const bool inputs_from_checkpoint = restart.read; + + if (!mesh.validate(!inputs_from_checkpoint)) return false; if (!time.validate()) return false; @@ -476,6 +501,8 @@ bool ExaOptions::validate() { return false; if (!boundary_conditions.validate()) return false; + if (!restart.validate()) + return false; // Check that we have at least one material if (materials.empty()) { @@ -483,19 +510,7 @@ bool ExaOptions::validate() { return false; } - if (materials.size() > 1) { - if (!region_mapping_file) { - WARNING_0_OPT("Error: region_mapping_file was not provided even though multiple " - "materials were asked for."); - return false; - } else if (mesh.mesh_type == MeshType::AUTO && !grain_file) { - WARNING_0_OPT("Error: region_mapping_file was provided but no grain_file was provided " - "when using auto mesh."); - return false; - } - } - - if (materials.size() > 1) { + if (materials.size() > 1 && !inputs_from_checkpoint) { if (!region_mapping_file) { WARNING_0_OPT("Error: region_mapping_file was not provided even though multiple " "materials were asked for."); @@ -509,7 +524,7 @@ bool ExaOptions::validate() { size_t index = 1; for (auto& mat : materials) { - if (!mat.validate()) + if (!mat.validate(!inputs_from_checkpoint)) return false; // Update the region_id value after validating // everything so to make it easier for users to @@ -630,31 +645,34 @@ bool ExaOptions::validate() { if (!post_processing.validate()) return false; - if (region_mapping_file) { - if (!std::filesystem::exists(*region_mapping_file)) { - std::ostringstream err; - err << "Error: Region mapping file: " << *region_mapping_file << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (!inputs_from_checkpoint) { + if (region_mapping_file) { + if (!std::filesystem::exists(*region_mapping_file)) { + std::ostringstream err; + err << "Error: Region mapping file: " << *region_mapping_file << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } - } - if (grain_file) { - if (!std::filesystem::exists(*grain_file)) { - std::ostringstream err; - err << "Error: Grain file provided at top level: " << *grain_file << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (grain_file) { + if (!std::filesystem::exists(*grain_file)) { + std::ostringstream err; + err << "Error: Grain file provided at top level: " << *grain_file + << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } - } - if (orientation_file) { - if (!std::filesystem::exists(*orientation_file)) { - std::ostringstream err; - err << "Error: Orientation file provided at top level: " << *orientation_file - << " does not exist"; - WARNING_0_OPT(err.str()); - return false; + if (orientation_file) { + if (!std::filesystem::exists(*orientation_file)) { + std::ostringstream err; + err << "Error: Orientation file provided at top level: " << *orientation_file + << " does not exist"; + WARNING_0_OPT(err.str()); + return false; + } } } @@ -678,6 +696,21 @@ void ExaOptions::print_options() const { std::cout << "\nSimulation Information:\n"; std::cout << " Base name: " << basename << "\n"; std::cout << " Version: " << version << "\n"; + if (restart.write || restart.read) { + std::cout << " Restart: write=" << (restart.write ? "yes" : "no") + << " interval=" << restart.interval + << " write_final=" << (restart.write_final ? "yes" : "no") + << " keep_last=" << restart.keep_last + << " read=" << (restart.read ? "yes" : "no"); + if (restart.read_cycle) { + std::cout << " read_cycle=" << *restart.read_cycle; + } + std::cout << " directory=" << restart.directory.string(); + if (restart.read && restart.read_directory) { + std::cout << " read_directory=" << restart.read_directory->string(); + } + std::cout << "\n"; + } // Print each component print_mesh_options(); @@ -773,12 +806,6 @@ void ExaOptions::print_time_options() const { } else { std::cout << " Type: none (time_type has no matching [Time.*] section)\n"; } - - if (time.restart) { - std::cout << " Restart enabled:\n"; - std::cout << " Restart time: " << time.restart_time << "\n"; - std::cout << " Restart cycle: " << time.restart_cycle << "\n"; - } } void ExaOptions::print_solver_options() const { diff --git a/src/options/option_parser_v2.hpp b/src/options/option_parser_v2.hpp index 5699b3e2..772efed6 100644 --- a/src/options/option_parser_v2.hpp +++ b/src/options/option_parser_v2.hpp @@ -181,8 +181,9 @@ struct MeshOptions { */ bool periodicity = false; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static MeshOptions from_toml(const toml::value& toml_input); @@ -222,8 +223,9 @@ struct GrainInfo { */ int num_grains = 0; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static GrainInfo from_toml(const toml::value& toml_input); @@ -274,8 +276,9 @@ struct StateVariables { */ std::vector initial_values; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static StateVariables from_toml(const toml::value& toml_input); @@ -441,8 +444,9 @@ struct MaterialOptions { */ double temperature = 298.0; - // Validation - bool validate() const; + // Validation (check_input_files = false skips the checks on input files a restart does + // not read, since they may no longer exist) + bool validate(bool check_input_files = true) const; // Conversion from toml static MaterialOptions from_toml(const toml::value& toml_input); @@ -562,21 +566,6 @@ struct TimeOptions { */ std::optional custom_time; - /** - * @brief Whether this is a restart simulation - */ - bool restart = false; - - /** - * @brief Time to restart from (if restart is enabled) - */ - double restart_time = 0.0; - - /** - * @brief Cycle number to restart from (if restart is enabled) - */ - size_t restart_cycle = 0; - /** * @brief Determine which time stepping mode is active * @@ -1352,6 +1341,82 @@ struct PostProcessingOptions { const LightUpOptions* get_light_up_config_for_region(int region_id) const; }; +/** + * @brief Checkpoint and restart configuration ([Restart] table) + * + * @details Checkpoints are binary Axom Sidre files, one per MPI rank, written to + * `directory/cycle_/`. A restart needs only the checkpoint and the options + * file; the mesh, grain, region-mapping, orientation and state-variable files are not read. + * Writing or reading checkpoints requires ExaConstit built with Axom (EXACONSTIT_USE_AXOM). + * See doc/restart.md for the full description. + */ +struct RestartOptions { + /** + * @brief Whether to write checkpoints during the run + */ + bool write = false; + + /** + * @brief Number of completed cycles between checkpoints (0 disables interval checkpoints) + */ + int interval = 0; + + /** + * @brief Whether to also write a checkpoint when the simulation finishes + */ + bool write_final = true; + + /** + * @brief Number of complete checkpoints to keep on disk; older ones are removed (0 keeps + * all of them) + */ + int keep_last = 2; + + /** + * @brief Whether to restart this run from a checkpoint + */ + bool read = false; + + /** + * @brief Completed-cycle count of the checkpoint to restart from; the newest complete + * checkpoint if unset + */ + std::optional read_cycle; + + /** + * @brief Directory checkpoints are written to (and read from unless read_directory is set) + * + * @details A relative path is taken inside the run's output folder, + * / (ExaOptions::parse_options() replaces it by the full path). + */ + std::filesystem::path directory = "restart"; + + /** + * @brief Directory to read the restart checkpoint from instead of directory + * + * @details Lets a continuation restart from another run's checkpoints while writing its own + * to directory. A relative path is relative to the working directory. Writing, superseding and + * keep_last pruning only ever act on directory, so the other run's checkpoints are never + * renamed or removed. + */ + std::optional read_directory; + + /** + * @brief Directory a restart reads from + * + * @return read_directory if it is set, otherwise directory + */ + const std::filesystem::path& GetReadDirectory() const { + return read_directory ? *read_directory : directory; + } + + // Validation + bool validate() const; + + // Conversion from toml + static RestartOptions from_toml(const toml::value& toml_input); +}; + /** * @brief Main options class for ExaConstit simulation configuration */ @@ -1402,6 +1467,11 @@ class ExaOptions { */ PostProcessingOptions post_processing; + /** + * @brief Checkpoint/restart options + */ + RestartOptions restart; + /** * @brief Paths to external material configuration files */ @@ -1464,6 +1534,12 @@ class ExaOptions { */ void parse_time_options(const toml::value& toml_input); + /** + * @brief Parse checkpoint and restart options from TOML input + * @param toml_input TOML value containing the [Restart] table + */ + void parse_restart_options(const toml::value& toml_input); + /** * @brief Parse solver options from TOML input * @param toml_input TOML value containing solver configuration diff --git a/src/options/option_restart.cpp b/src/options/option_restart.cpp new file mode 100644 index 00000000..e91b8115 --- /dev/null +++ b/src/options/option_restart.cpp @@ -0,0 +1,94 @@ +#include "options/option_parser_v2.hpp" +#include "options/option_util.hpp" + +RestartOptions RestartOptions::from_toml(const toml::value& toml_input) { + RestartOptions options; + + if (toml_input.contains("write")) { + options.write = toml::find(toml_input, "write"); + } + + if (toml_input.contains("interval")) { + options.interval = toml::find(toml_input, "interval"); + } + + if (toml_input.contains("write_final")) { + options.write_final = toml::find(toml_input, "write_final"); + } + + if (toml_input.contains("keep_last")) { + options.keep_last = toml::find(toml_input, "keep_last"); + } + + if (toml_input.contains("read")) { + options.read = toml::find(toml_input, "read"); + } + + if (toml_input.contains("read_cycle")) { + options.read_cycle = toml::find(toml_input, "read_cycle"); + } + + if (toml_input.contains("directory")) { + options.directory = toml::find(toml_input, "directory"); + } + + if (toml_input.contains("read_directory")) { + options.read_directory = toml::find(toml_input, "read_directory"); + } + + return options; +} + +bool RestartOptions::validate() const { + // Checkpoints are Axom Sidre files, so a build without Axom can neither write nor read + // them. Without this check the request would be silently ignored. +#if !defined(EXACONSTIT_USE_AXOM) + if (write || read) { + WARNING_0_OPT("Error: The Restart table asks to write or read checkpoints, but ExaConstit " + "was built without Axom (configure with AXOM_DIR pointing at an Axom " + "built with HDF5)."); + return false; + } +#endif + + if (interval < 0) { + WARNING_0_OPT("Error: Restart table has `interval` set to a value less than 0 (use 0 to " + "disable interval checkpoints)."); + return false; + } + + if (keep_last < 0) { + WARNING_0_OPT("Error: Restart table has `keep_last` set to a value less than 0 (use 0 to " + "keep every checkpoint)."); + return false; + } + + if ((write || read) && directory.empty()) { + WARNING_0_OPT("Error: Restart table has an empty `directory`."); + return false; + } + + if (read_directory.has_value() && read_directory->empty()) { + WARNING_0_OPT("Error: Restart table has an empty `read_directory`."); + return false; + } + + // Whether the checkpoint directory exists is checked in ExaOptions::parse_options(), once a + // relative `directory` has been placed inside the output folder. + + // The remaining cases are allowed but probably not what was intended. + if (read_directory.has_value() && !read) { + WARNING_0_OPT("Restart table sets `read_directory` but `read` = false, so it is ignored."); + } + + if (read_cycle.has_value() && !read) { + WARNING_0_OPT("Restart table sets `read_cycle` but `read` = false, so it is ignored."); + } + + if (write && interval == 0 && !write_final) { + WARNING_0_OPT("Restart table has `write` = true, but `interval` = 0 and `write_final` = " + "false, so no checkpoints will be written."); + } + + return true; +} diff --git a/src/options/option_time.cpp b/src/options/option_time.cpp index cf2a760b..7a614296 100644 --- a/src/options/option_time.cpp +++ b/src/options/option_time.cpp @@ -125,19 +125,6 @@ void TimeOptions::determine_time_type(const std::optional& request TimeOptions TimeOptions::from_toml(const toml::value& toml_input) { TimeOptions options; - // Check for restart options - if (toml_input.contains("restart")) { - options.restart = toml::find(toml_input, "restart"); - } - - if (toml_input.contains("restart_time")) { - options.restart_time = toml::find(toml_input, "restart_time"); - } - - if (toml_input.contains("restart_cycle")) { - options.restart_cycle = toml::find(toml_input, "restart_cycle"); - } - // Check for nested time stepping sections if (toml_input.contains("Auto")) { options.auto_time = AutoTimeOptions::from_toml(toml::find(toml_input, "Auto")); diff --git a/src/postprocessing/postprocessing_driver.cpp b/src/postprocessing/postprocessing_driver.cpp index 10f9d838..d70cec3c 100644 --- a/src/postprocessing/postprocessing_driver.cpp +++ b/src/postprocessing/postprocessing_driver.cpp @@ -12,6 +12,9 @@ #include "SNLS_linalg.h" #include +#include +#include +#include namespace fs = std::filesystem; namespace { @@ -1512,6 +1515,12 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { paraview.SetLevelsOfDetail(options.mesh.order); paraview.SetDataFormat(mfem::VTKFormat::BINARY); paraview.SetHighOrderOutput(false); + // Restart: keep the .pvd entries written before the restart time + paraview.UseRestartMode(m_sim_state->IsRestartRun()); + if (m_sim_state->IsRestartRun()) { + m_restart_pvd_files.push_back(output_dir_vizs / + (output_dir_vizs.filename().string() + ".pvd")); + } dcs_keys.push_back(key); } #ifdef MFEM_USE_ADIOS2 @@ -1534,9 +1543,14 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { dcs->RegisterField(disp_key, value.get()); } } - dcs->SetCycle(0); - dcs->SetTime(0.0); - dcs->Save(); + // The initial state, written once by the run that starts at t = 0. A restart + // keeps the files written before it and adds to them from its first output + // step on, so it must not overwrite cycle 0. + if (!m_sim_state->IsRestartRun()) { + dcs->SetCycle(0); + dcs->SetTime(0.0); + dcs->Save(); + } } } } @@ -1572,6 +1586,12 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { paraview.SetLevelsOfDetail(options.mesh.order); paraview.SetDataFormat(mfem::VTKFormat::BINARY); paraview.SetHighOrderOutput(false); + // Restart: keep the .pvd entries written before the restart time + paraview.UseRestartMode(m_sim_state->IsRestartRun()); + if (m_sim_state->IsRestartRun()) { + m_restart_pvd_files.push_back(output_dir_vizs / + (output_dir_vizs.filename().string() + ".pvd")); + } dcs_keys.push_back(key); } #ifdef MFEM_USE_ADIOS2 @@ -1595,14 +1615,60 @@ void PostProcessingDriver::InitializeDataCollections(ExaOptions& options) { dcs->RegisterField(disp_key, value.get()); } } - dcs->SetCycle(0); - dcs->SetTime(0.0); - dcs->Save(); + // The initial state, written only by the run that starts at t = 0 (see above) + if (!m_sim_state->IsRestartRun()) { + dcs->SetCycle(0); + dcs->SetTime(0.0); + dcs->Save(); + } } } } +namespace { + +/** + * @brief Remove the entries of cycle first_cycle and later from a ParaView collection file + * + * @param pvd_file The .pvd file (nothing is done if it does not exist) + * @param first_cycle First cycle to remove + * + * Lines other than DataSet entries (the header and closing tags) are kept. MFEM's restart mode + * then appends the new entries after the kept ones. + */ +void TrimParaViewCollection(const fs::path& pvd_file, const int first_cycle) { + std::ifstream in(pvd_file); + if (!in) { + return; + } + static const std::regex cycle_regex("file=\"Cycle(\\d+)"); + std::ostringstream kept; + std::string line; + std::smatch match; + while (std::getline(in, line)) { + if (std::regex_search(line, match, cycle_regex) && std::stoi(match[1]) >= first_cycle) { + continue; + } + kept << line << "\n"; + } + in.close(); + std::ofstream out(pvd_file, std::ios::trunc); + out << kept.str(); +} + +} // namespace + void PostProcessingDriver::UpdateDataCollections(const int step, const double time) { + // First save of a restart run: drop the ParaView entries this run writes again (written by + // rank 0, which alone writes the .pvd files) + if (!m_restart_pvd_files.empty()) { + if (m_mpi_rank == 0) { + for (const auto& pvd_file : m_restart_pvd_files) { + TrimParaViewCollection(pvd_file, step); + } + } + m_restart_pvd_files.clear(); + } for (auto& [tmp, dcs] : m_map_dcs) { dcs->SetCycle(step); dcs->SetTime(time); diff --git a/src/postprocessing/postprocessing_driver.hpp b/src/postprocessing/postprocessing_driver.hpp index 7be26c8f..fcbe4a29 100644 --- a/src/postprocessing/postprocessing_driver.hpp +++ b/src/postprocessing/postprocessing_driver.hpp @@ -965,6 +965,18 @@ class PostProcessingDriver { */ std::map> m_map_dcs; + /** + * @brief ParaView collection files (.pvd) to trim before the first save of a restart run + * + * A restart keeps the visualization files written before it. ParaView's restart mode + * (ParaViewDataCollection::UseRestartMode) keeps the .pvd entries whose time is below the + * current time, but it compares with the times printed in the file, so an entry of the cycle + * being written again (a restart from an earlier checkpoint) can look earlier and stop the + * run. UpdateDataCollections() therefore first removes, by cycle, the entries from the cycle + * it writes on. Empty after that first save, and in runs that are not restarts. + */ + std::vector m_restart_pvd_files; + /** * @brief Registered projection operations * diff --git a/src/sim_state/exa_sidre_parmesh.cpp b/src/sim_state/exa_sidre_parmesh.cpp new file mode 100644 index 00000000..3a896823 --- /dev/null +++ b/src/sim_state/exa_sidre_parmesh.cpp @@ -0,0 +1,1127 @@ +// Compiled only when ExaConstit is built with Axom, so this file can always be listed in +// CMake; without EXACONSTIT_USE_AXOM it is an empty translation unit. +#if defined(EXACONSTIT_USE_AXOM) + +#include "sim_state/exa_sidre_parmesh.hpp" + +#include "sim_state/restart_sidre_utils.hpp" +#include "utilities/mechanics_log.hpp" +#include "utilities/unified_logger.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +// ========================================================================= +// BACKGROUND: HOW AN mfem::ParMesh DESCRIBES ITS PARALLEL STRUCTURE +// ========================================================================= +// +// Everything in this file writes or rebuilds the pieces listed here, so it is worth reading +// once before changing anything below. References are to MFEM's mesh/pmesh.cpp, +// general/communication.cpp and fem/pfespace.cpp. +// +// Local mesh. Each rank holds an ordinary serial mfem::Mesh for its own elements: local +// vertices, local elements, local boundary elements, and the local edges and faces MFEM +// derives from the element connectivity (FinalizeTopology()). Vertices, edges and faces on +// the interface between ranks exist on every rank that touches them, each rank with its own +// local index for them. +// +// Communication groups (ParMesh::gtopo, an mfem::GroupTopology). A group is a set of ranks +// that share at least one vertex, edge or face. Group 0 is always this rank on its own and +// shares nothing. Every other group has at least two members and a local index +// 1..NGroups()-1 on each member rank; the same group generally has different local indices +// on different ranks. Each group has a master rank (GroupTopology::Create() picks the +// smallest rank in the group), and the master owns the group's shared DOFs when the finite +// element spaces assign true DOFs. Inside a GroupTopology, ranks are referred to by "local +// processor" (lproc) numbers: this rank is lproc 0 and the other ranks are numbered in the +// order they first appear when walking the groups in local group order +// (GroupTopology::ProcToLProc()). The local group order therefore also fixes the neighbor +// numbering and the order of the messages the group communicators exchange. +// +// Shared entities. The shared vertices are stored as local vertex indices (svert_lvert). +// The shared edges and faces are stored as vertex tuples (shared_edges, shared_trias, +// shared_quads) in a "stored orientation" that every rank in the group agrees on, together +// with the local edge or face each one corresponds to (sedge_ledge, sface_lface). +// +// Group tables (group_svert, group_sedge, group_stria, group_squad). mfem::Table objects in +// compressed-row (CSR) form with one row per communication group 1..N: row r describes group +// r+1, since group 0 shares nothing. Row r lists the indices, into the shared-entity arrays +// above, of the entities group r+1 shares. +// +// How the finite element spaces use this. ParFiniteElementSpace builds its parallel DOF +// numbering from these structures. For each group it collects the DOFs on the group's shared +// entities, in the order the entities appear in the group's row, and the group's master +// assigns them true DOFs that the other members then receive by position. Two consequences +// drive the whole design of this file: +// 1. Entity i of a group on one rank is matched with entity i of the same group on every +// other member, so the order of the entities within each group must be identical on +// every member rank. +// 2. A shared edge or face with several interior DOFs lists them in a specific order, and +// GetSharedEdgeDofs() / GetSharedTriangleDofs() / GetSharedQuadrilateralDofs() permute +// each rank's local DOF order by the orientation that GroupEdge() / GroupTriangle() / +// GroupQuadrilateral() report, so that every rank lists them in the stored order. A +// wrong stored orientation silently matches the wrong DOFs across ranks. It only shows +// up once a shared entity has two or more interior DOFs (H1 order >= 3 on hex faces), +// and the mesh otherwise looks perfectly valid. +// +// MFEM normally builds all of this in the partitioning ParMesh constructor, which walks the +// global serial mesh in global entity order; that is what makes the per-group entity order +// and the stored orientations agree across ranks. A restarted run has no global mesh, so the +// writer below records exactly what each rank holds and the reader rebuilds it unchanged. + +namespace exaconstit { + +namespace { + +using axom::sidre::Group; +using axom::sidre::IndexType; +using axom::sidre::View; +using restart_detail::ReadSidreScalar; +using restart_detail::VerifyOnAllRanks; + +static_assert(std::is_same::value, + "Restart mesh I/O stores coordinates as FLOAT64; build MFEM in double precision."); + +/** + * @brief Conduit Blueprint shape name for an MFEM geometry + * + * @param geometry MFEM geometry type (mfem::Geometry::Type) + * @return Blueprint shape name, or nullptr if the restart mesh I/O does not support the + * geometry + */ +const char* ShapeNameFromGeometry(const int geometry) { + switch (geometry) { + case mfem::Geometry::SEGMENT: + return "line"; + case mfem::Geometry::TRIANGLE: + return "tri"; + case mfem::Geometry::SQUARE: + return "quad"; + case mfem::Geometry::TETRAHEDRON: + return "tet"; + case mfem::Geometry::CUBE: + return "hex"; + default: + return nullptr; + } +} + +/** + * @brief Create an integer array view owned by Sidre + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param count Number of entries + * @return Pointer to the allocated entries, or nullptr when count is 0 + * + * @details The arrays are owned by Sidre, never external views of the mesh's memory, because + * Group::load() only restores the description of an external view and not its data, so the + * reader would get nothing back. Zero-length arrays are described but not allocated, which + * is why callers must not dereference the returned pointer when count is 0. Ranks with no + * boundary elements, or groups with no shared edges, produce such arrays all the time. + */ +int* CreateOwnedIntArray(Group* group, const std::string& path, const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::INT_ID, 0); + return nullptr; + } + return group->createViewAndAllocate(path, axom::sidre::INT_ID, count)->getData(); +} + +/** + * @brief Create a double array view owned by Sidre + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param count Number of entries + * @return Pointer to the allocated entries, or nullptr when count is 0 + * + * @details See CreateOwnedIntArray() for why the arrays are owned and why zero-length arrays + * are not allocated. + */ +double* CreateOwnedDoubleArray(Group* group, const std::string& path, const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return nullptr; + } + return group->createViewAndAllocate(path, axom::sidre::FLOAT64_ID, count)->getData(); +} + +/** + * @brief Look up an array written by CreateOwnedIntArray() or CreateOwnedDoubleArray() + * + * @tparam ValueType int or double + * @param group Group that holds the view + * @param path Path of the view relative to the group + * @param expected_count Number of entries the array must have + * @return Pointer to the entries (nullptr when the array is empty) and whether the view + * exists, is owned by Sidre, has the expected type and has expected_count entries + * + * @details The type and ownership checks catch a checkpoint written by a different layout + * version or a corrupted file. Without them a wrong-typed array would be reinterpreted + * silently, and an external view would come back with a null data pointer. + */ +template +std::pair +ReadOwnedArray(Group* group, const std::string& path, const IndexType expected_count) { + const axom::sidre::TypeID expected_type = std::is_same::value + ? axom::sidre::FLOAT64_ID + : axom::sidre::INT_ID; + if (!group->hasView(path)) { + return {nullptr, false}; + } + View* view = group->getView(path); + if (view->getNumElements() != expected_count) { + return {nullptr, false}; + } + if (expected_count == 0) { + return {nullptr, true}; + } + if (view->getTypeID() != expected_type || view->isExternal()) { + return {nullptr, false}; + } + return {view->getData(), true}; +} + +/** + * @brief Look up an owned integer array of unknown length that holds fixed-size tuples + * + * @param group Group that holds the view + * @param path Path of the view relative to the group + * @param components Number of integers per tuple (for example 2 for an edge) + * @param ok Set to false if the view is missing, malformed, or its length is not a multiple + * of components; left unchanged otherwise, so one flag can collect several checks + * @return Pointer to the entries (nullptr when empty or malformed) and the number of tuples + * + * @details Used for the adjacency-set arrays, whose lengths are not recorded separately. + */ +std::pair +ReadIntTuples(Group* group, const std::string& path, const int components, bool& ok) { + if (!group->hasView(path)) { + ok = false; + return {nullptr, 0}; + } + View* view = group->getView(path); + const IndexType scalar_count = view->getNumElements(); + if (scalar_count % components != 0) { + ok = false; + return {nullptr, 0}; + } + if (scalar_count == 0) { + return {nullptr, 0}; + } + if (view->getTypeID() != axom::sidre::INT_ID || view->isExternal()) { + ok = false; + return {nullptr, 0}; + } + return {view->getData(), scalar_count / components}; +} + +/** + * @brief Agree across ranks on the element and boundary element geometries + * + * @param mesh Parallel mesh being written + * @return Mesh-wide element geometry and boundary element geometry; the boundary geometry is + * -1 if no rank has boundary elements + * + * @details Collective. The reader needs the geometry to size the connectivity arrays and to + * create elements, and it has to be the same on every rank. A rank deep inside the domain may + * own no boundary elements, though, and a rank could in principle own no elements at all, so + * a rank cannot simply look at its own first element. Each rank reports what it has, using -1 + * for "none", and an MPI_Allreduce with MPI_MAX picks the mesh-wide value (valid geometry + * values are all >= 0). + * + * The same pass also checks that no rank mixes geometries, that every rank agrees with the + * mesh-wide value, and that the geometry is one the Blueprint layout here supports. Any + * violation aborts on every rank, since a single connectivity array per topology cannot + * describe mixed meshes. + */ +std::pair AgreeOnMeshGeometries(const mfem::ParMesh& mesh) { + int local_element_geometry = -1; + int local_boundary_geometry = -1; + bool local_mixed_geometry = false; + for (int element_index = 0; element_index < mesh.GetNE(); ++element_index) { + const int geometry = mesh.GetElementGeometry(element_index); + if (local_element_geometry < 0) { + local_element_geometry = geometry; + } else if (geometry != local_element_geometry) { + local_mixed_geometry = true; + } + } + for (int boundary_index = 0; boundary_index < mesh.GetNBE(); ++boundary_index) { + const int geometry = mesh.GetBdrElementGeometry(boundary_index); + if (local_boundary_geometry < 0) { + local_boundary_geometry = geometry; + } else if (geometry != local_boundary_geometry) { + local_mixed_geometry = true; + } + } + + std::array local_geometries = {local_element_geometry, local_boundary_geometry}; + std::array global_geometries = {-1, -1}; + MPI_Allreduce( + local_geometries.data(), global_geometries.data(), 2, MPI_INT, MPI_MAX, mesh.GetComm()); + const int element_geometry = global_geometries[0]; + const int boundary_geometry = global_geometries[1]; + + const bool local_consistent = + !local_mixed_geometry && + (local_element_geometry < 0 || local_element_geometry == element_geometry) && + (local_boundary_geometry < 0 || local_boundary_geometry == boundary_geometry) && + ShapeNameFromGeometry(element_geometry) != nullptr && + (boundary_geometry < 0 || ShapeNameFromGeometry(boundary_geometry) != nullptr); + VerifyOnAllRanks(local_consistent, + mesh.GetComm(), + "the mesh mixes element geometries or uses a geometry other than segment, " + "triangle, quadrilateral, tetrahedron or hexahedron", + "Unable to write the mesh to the restart checkpoint"); + return {element_geometry, boundary_geometry}; +} + +/** + * @brief Write the vertex coordinates under `coordsets/coords` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * + * @details One array per axis (x, y and, in 3D, z), each indexed by local vertex number, as + * Blueprint's explicit coordsets expect. The local vertex numbering has to be preserved + * exactly: element connectivity, the shared vertices and the shared edge and face tuples all + * refer to vertices by these indices. + * + * These are the mesh's vertex positions, not its Nodes. For a high-order or deformed mesh the + * vertices only carry the element corners. SimulationState's coordinate fields are restored + * separately from the checkpoint, so nothing is lost. + */ +void WriteCoordinates(const mfem::ParMesh& mesh, Group* mesh_group) { + const std::array axis_names = {"x", "y", "z"}; + mesh_group->createViewString("coordsets/coords/type", "explicit"); + for (int axis = 0; axis < mesh.SpaceDimension(); ++axis) { + double* axis_values = CreateOwnedDoubleArray( + mesh_group, std::string("coordsets/coords/values/") + axis_names[axis], mesh.GetNV()); + for (int vertex_index = 0; vertex_index < mesh.GetNV(); ++vertex_index) { + axis_values[vertex_index] = mesh.GetVertex(vertex_index)[axis]; + } + } +} + +/** + * @brief Write the element connectivity and element attributes + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param element_geometry Mesh-wide element geometry from AgreeOnMeshGeometries() + * + * @details The connectivity is one flat array, vertices_per_element entries per element, in + * local element order and in each element's own vertex order. Both orders matter. The finite + * element spaces number their local DOFs element by element following each element's vertex + * order, so a different element order or a rotated vertex list would put restored values on + * the wrong DOFs even though the mesh looks the same. That is also why the reader must not + * let MFEM "fix" element orientations (see the reader's Finalize() call). + * + * In ExaConstit the element attributes hold the region IDs once SimulationState has been set + * up (it overwrites the grain IDs read from the mesh file), and that is what a restarted run + * uses to rebuild its regions. The grain IDs are checkpointed separately by the caller. + */ +void WriteVolumeTopology(const mfem::ParMesh& mesh, Group* mesh_group, const int element_geometry) { + const int num_elements = mesh.GetNE(); + const int vertices_per_element = mfem::Geometry::NumVerts[element_geometry]; + mesh_group->createViewString("topologies/mesh/type", "unstructured"); + mesh_group->createViewString("topologies/mesh/coordset", "coords"); + mesh_group->createViewString("topologies/mesh/elements/shape", + ShapeNameFromGeometry(element_geometry)); + + int* connectivity = CreateOwnedIntArray(mesh_group, + "topologies/mesh/elements/connectivity", + static_cast(num_elements) * + vertices_per_element); + for (int element_index = 0; element_index < num_elements; ++element_index) { + const int* element_vertices = mesh.GetElement(element_index)->GetVertices(); + for (int local_vertex = 0; local_vertex < vertices_per_element; ++local_vertex) { + connectivity[static_cast(element_index) * vertices_per_element + + local_vertex] = element_vertices[local_vertex]; + } + } + + mesh_group->createViewString("fields/mesh_material_attribute/association", "element"); + mesh_group->createViewString("fields/mesh_material_attribute/topology", "mesh"); + int* element_attributes = CreateOwnedIntArray( + mesh_group, "fields/mesh_material_attribute/values", num_elements); + for (int element_index = 0; element_index < num_elements; ++element_index) { + element_attributes[element_index] = mesh.GetAttribute(element_index); + } +} + +/** + * @brief Write the boundary element connectivity and boundary attributes + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param boundary_geometry Mesh-wide boundary geometry from AgreeOnMeshGeometries(); nothing + * is written when it is -1 (no rank has boundary elements) + * + * @details Boundary elements are stored explicitly rather than regenerated on read, because + * their attributes are what the boundary conditions refer to, and MFEM's generated boundary + * would carry default attributes. A rank with no boundary elements still writes the + * (zero-length) arrays whenever some other rank has boundary elements, so every rank's group + * has the same structure. + */ +void WriteBoundaryTopology(const mfem::ParMesh& mesh, + Group* mesh_group, + const int boundary_geometry) { + if (boundary_geometry < 0) { + return; + } + const int num_boundary_elements = mesh.GetNBE(); + const int vertices_per_boundary_element = mfem::Geometry::NumVerts[boundary_geometry]; + mesh_group->createViewString("topologies/mesh/boundary_topology", "boundary"); + mesh_group->createViewString("topologies/boundary/type", "unstructured"); + mesh_group->createViewString("topologies/boundary/coordset", "coords"); + mesh_group->createViewString("topologies/boundary/elements/shape", + ShapeNameFromGeometry(boundary_geometry)); + + int* boundary_connectivity = CreateOwnedIntArray(mesh_group, + "topologies/boundary/elements/connectivity", + static_cast(num_boundary_elements) * + vertices_per_boundary_element); + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + const int* boundary_vertices = mesh.GetBdrElement(boundary_index)->GetVertices(); + for (int local_vertex = 0; local_vertex < vertices_per_boundary_element; ++local_vertex) { + boundary_connectivity[static_cast(boundary_index) * + vertices_per_boundary_element + + local_vertex] = boundary_vertices[local_vertex]; + } + } + + mesh_group->createViewString("fields/boundary_material_attribute/association", "element"); + mesh_group->createViewString("fields/boundary_material_attribute/topology", "boundary"); + int* boundary_attributes = CreateOwnedIntArray( + mesh_group, "fields/boundary_material_attribute/values", num_boundary_elements); + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + boundary_attributes[boundary_index] = mesh.GetBdrAttribute(boundary_index); + } +} + +/** + * @brief Write the communication groups and their shared entities under `adjsets/mesh` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * + * @details Writes one Sidre group per communication group g = 1..NGroups()-1 (see the + * background notes at the top of this file for what a communication group is). Each holds: + * - `local_group_index`: g itself. The reader puts the groups back in this order. The order + * determines the lproc numbering and the message order of the group communicators, so + * rebuilding it exactly keeps the restarted run's parallel communication, and therefore + * its floating-point reduction order, identical to the original run's. The reader does not + * rely on Sidre or HDF5 returning the groups in creation order. + * - `neighbors`: the MPI ranks of the other group members. + * - `values`: the shared vertices, as local vertex indices. + * - `edges` (2D and 3D): the shared edges as vertex pairs, in the stored orientation. + * - `triangles`, `quadrilaterals` (3D): the shared faces as vertex tuples, in the stored + * orientation. + * + * The shared entities are written in the order the group's table row lists them, which the + * partitioning ParMesh constructor made identical on every member rank (it walks the global + * entities in global order). The reader rebuilds each row in that same order, so the + * per-group order agreement survives the round trip. + * + * Every group is written, including groups that share only vertices, so the rebuilt mesh has + * exactly the original set of groups. The group name `g_` is the + * same on every member rank, which makes the files easier to inspect; nothing depends on it. + * + * **Recovering the stored orientation.** ParMesh does not expose the stored edge and face + * tuples directly. GroupEdge(), GroupTriangle() and GroupQuadrilateral() return the local + * edge or face index and the orientation o of the stored entity relative to that local + * entity, which is what ParFiniteElementSpace consumes. Writing the local entity's vertices + * instead of the stored ones would be wrong whenever o is not the identity. Every rank would + * rebuild its shared entity in its own local orientation, every rank would then report o as + * the identity, and ranks whose local orientations differ would list the interior DOFs of + * that edge or face in different orders. So the stored tuple is rebuilt from the local one: + * - Edges: GetEdgeVertices() always returns a local edge as (smaller, larger), and + * GroupEdge() reports o = +1 exactly when the stored edge's first vertex is the smaller one + * (`o = (v[0] < v[1]) ? +1 : -1`). Hence stored = (o > 0) ? (smaller, larger) : (larger, + * smaller). + * - Faces: GroupTriangle() / GroupQuadrilateral() compute + * o = GetTriOrientation() / GetQuadOrientation(base = local face, test = stored face), and + * those return the o for which test[Orient[o][i]] == base[i] for every i + * (Geometry::Constants<...>::Orient in fem/geom.hpp). So the stored face is obtained by + * scattering: stored[Orient[o][i]] = local_face[i]. + * On read, the rebuilt mesh holds these stored tuples, and GroupEdge(), GroupTriangle() and + * GroupQuadrilateral() return the same orientations as in the writing run. + */ +void WriteAdjacencySets(const mfem::ParMesh& mesh, Group* mesh_group) { + const int dimension = mesh.Dimension(); + Group* adjset_group = mesh_group->createGroup("adjsets/mesh"); + adjset_group->createViewString("association", "vertex"); + adjset_group->createViewString("topology", "mesh"); + Group* communication_groups = adjset_group->createGroup("groups"); + + mfem::Array local_entity_vertices; + const auto& triangle_orient = mfem::Geometry::Constants::Orient; + const auto& quad_orient = mfem::Geometry::Constants::Orient; + + // Group 0 is this rank on its own and shares nothing, so it is not written. + for (int group_index = 1; group_index < mesh.GetNGroups(); ++group_index) { + const std::string group_name = "g" + + std::to_string(mesh.gtopo.GetGroupMasterRank(group_index)) + + "_" + + std::to_string(mesh.gtopo.GetGroupMasterGroup(group_index)); + Group* shared_group = communication_groups->createGroup(group_name); + shared_group->createViewScalar("local_group_index", group_index); + + // Neighbor ranks. GetGroup() lists the members as lproc numbers; this rank is always + // lproc 0, and every other lproc is translated back to its MPI rank, since lproc + // numbers mean nothing outside this GroupTopology. + const int member_count = mesh.gtopo.GetGroupSize(group_index); + const int* member_lprocs = mesh.gtopo.GetGroup(group_index); + int* neighbor_ranks = CreateOwnedIntArray(shared_group, "neighbors", member_count - 1); + int neighbor_slot = 0; + for (int member = 0; member < member_count; ++member) { + if (member_lprocs[member] != 0) { + neighbor_ranks[neighbor_slot++] = mesh.gtopo.GetNeighborRank(member_lprocs[member]); + } + } + + // Shared vertices: a vertex has no orientation, so its local index is all there is. + const int shared_vertex_count = mesh.GroupNVertices(group_index); + int* shared_vertices = CreateOwnedIntArray(shared_group, "values", shared_vertex_count); + for (int i = 0; i < shared_vertex_count; ++i) { + shared_vertices[i] = mesh.GroupVertex(group_index, i); + } + + if (dimension >= 2) { + // Shared edges, converted from the local (smaller, larger) order to the stored + // order using GroupEdge()'s orientation (see the function documentation). + const int shared_edge_count = mesh.GroupNEdges(group_index); + int* shared_edges_out = CreateOwnedIntArray( + shared_group, "edges", 2 * static_cast(shared_edge_count)); + for (int i = 0; i < shared_edge_count; ++i) { + int local_edge = -1; + int orientation = 0; + mesh.GroupEdge(group_index, i, local_edge, orientation); + mesh.GetEdgeVertices(local_edge, local_entity_vertices); + const int smaller = local_entity_vertices[0]; + const int larger = local_entity_vertices[1]; + shared_edges_out[2 * i + 0] = (orientation > 0) ? smaller : larger; + shared_edges_out[2 * i + 1] = (orientation > 0) ? larger : smaller; + } + } + + if (dimension >= 3) { + // Shared faces, converted from the local face's vertex order to the stored order + // by scattering through Orient[o] (see the function documentation). A group can + // share both triangles and quadrilaterals only in mixed meshes, which are not + // supported, but both lists are always written so the layout does not depend on + // the element type. + const int shared_triangle_count = mesh.GroupNTriangles(group_index); + int* shared_triangles_out = CreateOwnedIntArray( + shared_group, "triangles", 3 * static_cast(shared_triangle_count)); + for (int i = 0; i < shared_triangle_count; ++i) { + int local_face = -1; + int orientation = 0; + mesh.GroupTriangle(group_index, i, local_face, orientation); + mesh.GetFaceVertices(local_face, local_entity_vertices); + for (int j = 0; j < 3; ++j) { + shared_triangles_out[3 * i + triangle_orient[orientation][j]] = + local_entity_vertices[j]; + } + } + + const int shared_quad_count = mesh.GroupNQuadrilaterals(group_index); + int* shared_quads_out = CreateOwnedIntArray( + shared_group, "quadrilaterals", 4 * static_cast(shared_quad_count)); + for (int i = 0; i < shared_quad_count; ++i) { + int local_face = -1; + int orientation = 0; + mesh.GroupQuadrilateral(group_index, i, local_face, orientation); + mesh.GetFaceVertices(local_face, local_entity_vertices); + for (int j = 0; j < 4; ++j) { + shared_quads_out[4 * i + quad_orient[orientation][j]] = + local_entity_vertices[j]; + } + } + } + } +} + +/** + * @brief Write the ExaConstit metadata and validation counts under `exaconstit/` + * + * @param mesh Parallel mesh being written + * @param mesh_group Group being written into + * @param element_geometry Mesh-wide element geometry + * @param boundary_geometry Mesh-wide boundary geometry (-1 if there are no boundary elements) + * + * @details The format version lets the reader reject layouts it does not understand. The + * dimensions and geometries are what the reader needs to size and create the mesh, and the + * topology counts are compared after the rebuild (see MeshTopologyCounts). The vertex, + * element and boundary element counts also give the expected lengths of the stored arrays. + */ +void WriteMeshMetadata(const mfem::ParMesh& mesh, + Group* mesh_group, + const int element_geometry, + const int boundary_geometry) { + Group* metadata = mesh_group->createGroup("exaconstit"); + metadata->createViewScalar("format_version", MESH_CHECKPOINT_FORMAT_VERSION); + metadata->createViewScalar("dimension", mesh.Dimension()); + metadata->createViewScalar("space_dimension", mesh.SpaceDimension()); + metadata->createViewScalar("element_geometry", element_geometry); + metadata->createViewScalar("boundary_geometry", boundary_geometry); + const MeshTopologyCounts counts = CountMeshTopology(mesh); + metadata->createViewScalar("num_vertices", counts.num_vertices); + metadata->createViewScalar("num_elements", counts.num_elements); + metadata->createViewScalar("num_boundary_elements", counts.num_boundary_elements); + metadata->createViewScalar("num_edges", counts.num_edges); + metadata->createViewScalar("num_faces", counts.num_faces); + metadata->createViewScalar("num_communication_groups", counts.num_communication_groups); +} + +} // namespace + +// ========================================================================= +// TOPOLOGY COUNTS +// ========================================================================= + +bool MeshTopologyCounts::Matches(const MeshTopologyCounts& other) const { + return num_vertices == other.num_vertices && num_elements == other.num_elements && + num_boundary_elements == other.num_boundary_elements && num_edges == other.num_edges && + num_faces == other.num_faces && + num_communication_groups == other.num_communication_groups; +} + +MeshTopologyCounts CountMeshTopology(const mfem::ParMesh& mesh) { + MeshTopologyCounts counts; + counts.num_vertices = mesh.GetNV(); + counts.num_elements = mesh.GetNE(); + counts.num_boundary_elements = mesh.GetNBE(); + counts.num_edges = mesh.GetNEdges(); + counts.num_faces = mesh.GetNFaces(); + // GetNGroups() includes group 0 (this rank alone), which shares nothing. + counts.num_communication_groups = mesh.GetNGroups() - 1; + return counts; +} + +MeshTopologyCounts ReadStoredMeshTopologyCounts(Group* mesh_group) { + Group* metadata = mesh_group->getGroup("exaconstit"); + MeshTopologyCounts counts; + counts.num_vertices = ReadSidreScalar(metadata, "num_vertices"); + counts.num_elements = ReadSidreScalar(metadata, "num_elements"); + counts.num_boundary_elements = ReadSidreScalar(metadata, "num_boundary_elements"); + counts.num_edges = ReadSidreScalar(metadata, "num_edges"); + counts.num_faces = ReadSidreScalar(metadata, "num_faces"); + counts.num_communication_groups = ReadSidreScalar(metadata, + "num_communication_groups"); + return counts; +} + +// ========================================================================= +// WRITER +// ========================================================================= + +/** + * @details Implementation notes. The writer records exactly what this rank's ParMesh holds, + * with no renumbering of any kind, because the reader has to reproduce every numbering and + * ordering listed in the background notes at the top of this file. Each piece is written by + * its own helper, documented above: + * - AgreeOnMeshGeometries(): the only collective step (MPI_Allreduce). + * - WriteCoordinates(): local vertex positions. + * - WriteVolumeTopology(): element connectivity and attributes (region IDs). + * - WriteBoundaryTopology(): boundary element connectivity and attributes. + * - WriteAdjacencySets(): communication groups and shared entities in stored orientation. + * - WriteMeshMetadata(): format version, dimensions, geometries and validation counts. + * + * Nonconforming and NURBS meshes keep extra structures (NCMesh, NURBSExtension) that this + * layout does not describe, so they are rejected up front rather than written incompletely. + */ +void WriteParMeshBlueprint(const mfem::ParMesh& mesh, Group* mesh_group) { + CALI_CXX_MARK_SCOPE("exaconstit::WriteParMeshBlueprint"); + + MFEM_VERIFY(mesh_group != nullptr, "WriteParMeshBlueprint was given a null mesh group"); + MFEM_VERIFY_0(!mesh.Nonconforming() && mesh.NURBSext == nullptr, + "Restart checkpoints do not support nonconforming or NURBS meshes"); + + const auto [element_geometry, boundary_geometry] = AgreeOnMeshGeometries(mesh); + WriteCoordinates(mesh, mesh_group); + WriteVolumeTopology(mesh, mesh_group, element_geometry); + WriteBoundaryTopology(mesh, mesh_group, boundary_geometry); + WriteAdjacencySets(mesh, mesh_group); + WriteMeshMetadata(mesh, mesh_group, element_geometry, boundary_geometry); +} + +// ========================================================================= +// READER +// ========================================================================= + +/** + * @details Implementation notes. This replaces MFEM's partitioning ParMesh constructor for a + * restarted run. That constructor builds the local mesh piece and the parallel structures + * from the global serial mesh. Here they are rebuilt from what each rank wrote, using the same + * sequence of protected Mesh / ParMesh calls the MFEM constructors use, in the same order. + * Read the background notes at the top of this file first; the steps below refer to them. + * + * The order of the steps is not arbitrary: + * - The communicator has to be set before the first collective call. + * - Every array is checked before anything is built, so a bad checkpoint aborts on every rank + * instead of crashing one rank halfway through construction while the others wait in a + * collective call. + * - FinalizeTopology() has to run before the shared structures are rebuilt, and Finalize() + * after, because Finalize() looks up the local edge and face of every shared entity in the + * edge and face tables FinalizeTopology() built. + * + * Any change here should be checked with test_restart_mesh_roundtrip, which compares the + * rebuilt mesh with the original on 1 to 4 ranks, including shared-entity orientations and the + * true-DOF layout of H1 spaces up to order 4. + */ +ExaSidreParMesh::ExaSidreParMesh(Group* mesh_group, MPI_Comm comm) { + CALI_CXX_MARK_SCOPE("exaconstit::ExaSidreParMesh"); + + MFEM_VERIFY(mesh_group != nullptr, "ExaSidreParMesh was given a null mesh group"); + MFEM_VERIFY(comm != MPI_COMM_NULL, "ExaSidreParMesh was given a null communicator"); + const std::string abort_summary = "Unable to rebuild the mesh from the restart checkpoint"; + + // --------------------------------------------------------------------- + // Step 0: communicator + // + // The ParMesh base class was default constructed, which leaves MyComm, NRanks, MyRank + // and gtopo's communicator unset. They must be set before the first collective call: + // FinalizeTopology() calls ParMesh::HasBoundaryElements() and ParMesh::SetAttributes(), + // both of which use MyComm, and BuildSharedStructures() uses gtopo's communicator. The + // MFEM constructors do the same thing first. + // --------------------------------------------------------------------- + MyComm = comm; + gtopo.SetComm(comm); + MPI_Comm_size(MyComm, &NRanks); + MPI_Comm_rank(MyComm, &MyRank); + + // --------------------------------------------------------------------- + // Step 1: metadata + // + // The format version comes first, since nothing else can be interpreted without it. The + // dimensions, geometries and local sizes then determine how many entries each array must + // have. A boundary geometry of -1 means no rank had boundary elements when the checkpoint + // was written, which is only consistent if this rank has none either. + // --------------------------------------------------------------------- + const bool has_metadata = mesh_group->hasGroup("exaconstit"); + int format_version = -1; + if (has_metadata) { + format_version = ReadSidreScalar(mesh_group->getGroup("exaconstit"), "format_version"); + } + VerifyOnAllRanks(has_metadata && format_version == MESH_CHECKPOINT_FORMAT_VERSION, + comm, + "the mesh group has no exaconstit/ metadata or an unsupported " + "format_version (" + + std::to_string(format_version) + ", expected " + + std::to_string(MESH_CHECKPOINT_FORMAT_VERSION) + ")", + abort_summary); + + Group* metadata = mesh_group->getGroup("exaconstit"); + const int dimension = ReadSidreScalar(metadata, "dimension"); + const int space_dimension = ReadSidreScalar(metadata, "space_dimension"); + const int element_geometry = ReadSidreScalar(metadata, "element_geometry"); + const int boundary_geometry = ReadSidreScalar(metadata, "boundary_geometry"); + const MeshTopologyCounts stored_counts = ReadStoredMeshTopologyCounts(mesh_group); + const int num_vertices = static_cast(stored_counts.num_vertices); + const int num_elements = static_cast(stored_counts.num_elements); + const int num_boundary_elements = static_cast(stored_counts.num_boundary_elements); + + const bool geometry_supported = element_geometry >= 0 && + element_geometry < mfem::Geometry::NUM_GEOMETRIES && + (boundary_geometry < 0 || + boundary_geometry < mfem::Geometry::NUM_GEOMETRIES) && + (boundary_geometry >= 0 || num_boundary_elements == 0); + VerifyOnAllRanks(geometry_supported, + comm, + "the mesh metadata has an invalid element or boundary geometry", + abort_summary); + + const int vertices_per_element = mfem::Geometry::NumVerts[element_geometry]; + const int vertices_per_boundary_element = boundary_geometry >= 0 + ? mfem::Geometry::NumVerts[boundary_geometry] + : 0; + + // --------------------------------------------------------------------- + // Step 2: check every array before building anything + // + // Each array must exist, be owned by Sidre (an external view would have no data after + // load), have the expected type, and have exactly the length the metadata implies. The + // adjacency-set arrays are checked in BuildSharedStructures(); only the presence of their + // parent group is checked here. All results are combined into one flag and agreed on by + // every rank, so either every rank goes on to build the mesh or every rank aborts. + // --------------------------------------------------------------------- + bool arrays_ok = true; + std::array axis_values = {nullptr, nullptr, nullptr}; + const std::array axis_names = {"x", "y", "z"}; + for (int axis = 0; axis < space_dimension; ++axis) { + const auto [values, ok] = ReadOwnedArray( + mesh_group, std::string("coordsets/coords/values/") + axis_names[axis], num_vertices); + axis_values[axis] = values; + arrays_ok = arrays_ok && ok; + } + const auto [element_connectivity, connectivity_ok] = ReadOwnedArray( + mesh_group, + "topologies/mesh/elements/connectivity", + static_cast(num_elements) * vertices_per_element); + const auto [element_attributes, element_attributes_ok] = ReadOwnedArray( + mesh_group, "fields/mesh_material_attribute/values", num_elements); + arrays_ok = arrays_ok && connectivity_ok && element_attributes_ok; + + const int* boundary_connectivity = nullptr; + const int* boundary_attributes = nullptr; + if (boundary_geometry >= 0) { + const auto [connectivity, boundary_connectivity_ok] = ReadOwnedArray( + mesh_group, + "topologies/boundary/elements/connectivity", + static_cast(num_boundary_elements) * vertices_per_boundary_element); + const auto [attributes, boundary_attributes_ok] = ReadOwnedArray( + mesh_group, "fields/boundary_material_attribute/values", num_boundary_elements); + boundary_connectivity = connectivity; + boundary_attributes = attributes; + arrays_ok = arrays_ok && boundary_connectivity_ok && boundary_attributes_ok; + } + arrays_ok = arrays_ok && mesh_group->hasGroup("adjsets/mesh/groups"); + VerifyOnAllRanks(arrays_ok, + comm, + "the mesh arrays are missing, external, or have unexpected sizes or types", + abort_summary); + + // --------------------------------------------------------------------- + // Step 3: build the local (serial) mesh piece + // + // InitMesh() only reserves space; AddVertex(), AddElement() and AddBdrElement() then + // append in order, so local vertex i, element i and boundary element i get exactly the + // indices they had in the writing run. Every value is copied into storage MFEM owns: + // AddVertex() copies the coordinates, and SetVertices() copies the vertex indices into + // the element MFEM allocated with NewElement(). Nothing points into the Sidre buffers + // afterwards. The Element objects are owned by the mesh once added and are deleted with + // it. + // + // InitMesh() takes the counts from the metadata, but AddVertex() and AddElement() grow + // the arrays as needed, so the counts are only a size hint; the array checks in step 2 + // are what guarantee the loops below stay in bounds. + // --------------------------------------------------------------------- + InitMesh(dimension, space_dimension, num_vertices, num_elements, num_boundary_elements); + for (int vertex_index = 0; vertex_index < num_vertices; ++vertex_index) { + mfem::real_t coordinates[3] = {0.0, 0.0, 0.0}; + for (int axis = 0; axis < space_dimension; ++axis) { + coordinates[axis] = axis_values[axis][vertex_index]; + } + AddVertex(coordinates); + } + for (int element_index = 0; element_index < num_elements; ++element_index) { + mfem::Element* element = NewElement(element_geometry); + element->SetVertices(element_connectivity + + static_cast(element_index) * vertices_per_element); + element->SetAttribute(element_attributes[element_index]); + AddElement(element); + } + for (int boundary_index = 0; boundary_index < num_boundary_elements; ++boundary_index) { + mfem::Element* boundary_element = NewElement(boundary_geometry); + boundary_element->SetVertices(boundary_connectivity + + static_cast(boundary_index) * + vertices_per_boundary_element); + boundary_element->SetAttribute(boundary_attributes[boundary_index]); + AddBdrElement(boundary_element); + } + + // --------------------------------------------------------------------- + // Step 4: local topology + // + // FinalizeTopology() sets the local mesh type (SetMeshGen()), builds the element-to-face + // and element-to-edge tables, generates the local faces and edges with their local + // numbering, and sets the attribute lists. The shared-entity lookups in Finalize() (step + // 6) need these edge and face tables. + // + // generate_bdr is false because the boundary was read from the checkpoint. Generating it + // would add boundary elements with default attributes, which the boundary conditions + // would not recognize. Two calls inside are collective in a ParMesh and are reached on + // every rank: HasBoundaryElements() is an MPI_Allreduce and is evaluated before + // generate_bdr in `!HasBoundaryElements() && generate_bdr`, and SetAttributes() is + // ParMesh::SetAttributes(), which gathers the attribute lists from every rank so that + // `attributes` and `bdr_attributes` are mesh-wide. + // --------------------------------------------------------------------- + const bool generate_boundary_elements = false; + FinalizeTopology(generate_boundary_elements); + + // --------------------------------------------------------------------- + // Step 5: parallel structure + // + // FinalizeTopology() computed `meshgen` (which element families are present) from this + // rank's elements only. ReduceMeshGen() combines it across ranks with a bitwise OR, so + // every rank, including one with no elements, sees the mesh-wide value that MFEM's + // parallel code expects. The MFEM constructors do the same. + // + // BuildSharedStructures() then rebuilds the communication groups and the shared-entity + // tables; see its implementation notes. + // --------------------------------------------------------------------- + ReduceMeshGen(); + BuildSharedStructures(mesh_group->getGroup("adjsets/mesh/groups"), dimension); + + // --------------------------------------------------------------------- + // Step 6: finalize + // + // ParMesh::Finalize() calls Mesh::Finalize() and then FinalizeParTopo(), which fills in + // sedge_ledge and sface_lface (the local edge and face of every shared edge and face) by + // looking up each stored vertex tuple in the local vertex-to-vertex and face tables. + // ParMesh::Finalize() also saves and restores the mesh-wide meshgen around + // Mesh::Finalize(), which may recompute it from the local elements. + // + // Both flags must stay false. Either one would rewrite element vertex lists that were + // stored exactly as the writing run had them, and that run's mesh was already finalized: + // - fix_orientation = true lets CheckElementOrientation() reorder the vertices of any + // element it considers inverted. + // - refine = true runs MarkForRefinement(), which for tetrahedral meshes reorders every + // tet's vertices to mark its refinement edge and can rotate the shared triangles. + // Either change renumbers local DOFs relative to the writing run and can break the stored + // shared-face orientations, so the restored fields would land on the wrong DOFs. + // --------------------------------------------------------------------- + const bool refine = false; + const bool fix_orientation = false; + Finalize(refine, fix_orientation); +} + +/** + * @details Implementation notes. This rebuilds everything listed under "Communication + * groups", "Shared entities" and "Group tables" in the background notes at the top of this + * file, from the per-group arrays WriteAdjacencySets() wrote. It does the same job as the + * group-building part of MFEM's partitioning ParMesh constructor (FindSharedFaces(), + * FindSharedEdges(), FindSharedVertices(), gtopo.Create(), BuildFaceGroup(), BuildEdgeGroup(), + * BuildVertexGroup() and the shared-element builders), except that the groups and entities + * are read rather than discovered. + * + * Three properties carry over from the writing run, and the rest of the parallel machinery + * depends on each of them: + * - Local group order: group g is rebuilt with local index g (from `local_group_index`). This + * fixes the lproc numbering and the communication pattern, so the restarted run exchanges + * data exactly as the original run did. + * - Per-group entity order: each table row lists its entities in the stored order, which is + * the order every member rank agrees on. + * - Stored orientation: the shared edge and face tuples are taken verbatim, so GroupEdge(), + * GroupTriangle() and GroupQuadrilateral() report the original orientations. + * + * `sedge_ledge` and `sface_lface` are only sized here. Their values depend on the local edge + * and face numbering, and Finalize() computes them from the stored tuples. + */ +void ExaSidreParMesh::BuildSharedStructures(Group* adjset_groups, const int dimension) { + CALI_CXX_MARK_SCOPE("exaconstit::ExaSidreParMesh::BuildSharedStructures"); + + // Pointers into the Sidre arrays of one communication group, plus their tuple counts. + struct SharedGroupArrays { + const int* neighbor_ranks = nullptr; + IndexType neighbor_count = 0; + const int* vertices = nullptr; + IndexType vertex_count = 0; + const int* edges = nullptr; + IndexType edge_count = 0; + const int* triangles = nullptr; + IndexType triangle_count = 0; + const int* quadrilaterals = nullptr; + IndexType quadrilateral_count = 0; + }; + + // --------------------------------------------------------------------- + // Read the stored groups into their original local order + // + // shared_groups[g - 1] receives the group whose stored local_group_index is g, no matter + // in what order Sidre returns the groups. Each index 1..group_count has to appear exactly + // once; a missing, duplicated or out-of-range index means the adjacency sets are corrupt. + // The tuple sizes (1 for vertices and neighbors, 2 for edges, 3 for triangles, 4 for + // quadrilaterals) are checked by ReadIntTuples(). + // --------------------------------------------------------------------- + const int group_count = static_cast(adjset_groups->getNumGroups()); + std::vector shared_groups(static_cast(group_count)); + std::vector group_filled(static_cast(group_count), false); + bool arrays_ok = true; + for (Group& group : adjset_groups->groups()) { + const int local_group_index = group.hasView("local_group_index") + ? ReadSidreScalar(&group, "local_group_index") + : -1; + if (local_group_index < 1 || local_group_index > group_count || + group_filled[static_cast(local_group_index - 1)]) { + arrays_ok = false; + continue; + } + group_filled[static_cast(local_group_index - 1)] = true; + SharedGroupArrays& arrays = shared_groups[static_cast(local_group_index - 1)]; + std::tie(arrays.neighbor_ranks, + arrays.neighbor_count) = ReadIntTuples(&group, "neighbors", 1, arrays_ok); + std::tie(arrays.vertices, + arrays.vertex_count) = ReadIntTuples(&group, "values", 1, arrays_ok); + if (dimension >= 2) { + std::tie(arrays.edges, + arrays.edge_count) = ReadIntTuples(&group, "edges", 2, arrays_ok); + } + if (dimension >= 3) { + std::tie(arrays.triangles, + arrays.triangle_count) = ReadIntTuples(&group, "triangles", 3, arrays_ok); + std::tie(arrays.quadrilaterals, arrays.quadrilateral_count) = ReadIntTuples( + &group, "quadrilaterals", 4, arrays_ok); + } + } + + // --------------------------------------------------------------------- + // Rebuild the group topology + // + // GroupTopology::Create() takes the groups as a list of rank sets, in local group order, + // with this rank alone first (group 0). Each IntegerSet must contain MPI ranks in sorted + // order: Create() picks each group's master with ListOfIntegerSets::PickElementInSet(), + // which returns the set's first entry, so sorting is what makes the master the smallest + // rank, the same choice MFEM's partitioning constructor makes on every member rank. + // + // ListOfIntegerSets::Insert() returns the index of an equal set if one is already in the + // list instead of adding a new one. Two stored groups with the same members would + // therefore collapse into one and shift every later group's index, so the returned index + // is checked against the expected one. A valid mesh never has duplicate groups. + // + // Create() exchanges group information with each neighbor rank (each non-master member + // learns the group's index on its master), so every rank must reach it. That is why the + // array checks above are agreed on across ranks first. The MPI tag is the one MFEM's + // partitioning constructor uses for the same call. + // --------------------------------------------------------------------- + mfem::ListOfIntegerSets communication_sets; + { + mfem::IntegerSet local_only_set; + mfem::Array& local_only = local_only_set; + local_only.Append(MyRank); + communication_sets.Insert(local_only_set); + } + for (int group_offset = 0; group_offset < group_count; ++group_offset) { + const SharedGroupArrays& arrays = shared_groups[static_cast(group_offset)]; + mfem::IntegerSet member_set; + mfem::Array& members = member_set; + members.Append(MyRank); + members.Append(arrays.neighbor_ranks, static_cast(arrays.neighbor_count)); + members.Sort(); + const int inserted_index = communication_sets.Insert(member_set); + arrays_ok = arrays_ok && (inserted_index == group_offset + 1); + } + VerifyOnAllRanks(arrays_ok, + MyComm, + "the adjacency-set arrays are malformed, have missing or duplicate local " + "group indices, or contain duplicate communication groups", + "Unable to rebuild the mesh from the restart checkpoint"); + const int group_topology_mpi_tag = 822; + gtopo.Create(communication_sets, group_topology_mpi_tag); + + // --------------------------------------------------------------------- + // Size the shared-entity arrays and group tables + // + // The shared-entity arrays (svert_lvert, shared_edges, shared_trias, shared_quads) hold + // every group's entities back to back, group 1 first. The group tables have one row per + // group 1..N (row r is group r+1), and each row lists the indices of that group's + // entities in those arrays. Since the arrays are filled group by group in row order, each + // row's indices are simply consecutive. + // + // ParFiniteElementSpace::GetGroupComm() calls GroupNTriangles() and GroupNQuadrilaterals() + // for every group in every dimension, so below 3D the triangle and quadrilateral tables + // still need group_count (empty) rows instead of being left unsized. The partitioning + // constructor also sizes them in every dimension (BuildFaceGroup()). + // --------------------------------------------------------------------- + IndexType total_shared_vertices = 0; + IndexType total_shared_edges = 0; + IndexType total_shared_faces = 0; + for (const SharedGroupArrays& arrays : shared_groups) { + total_shared_vertices += arrays.vertex_count; + total_shared_edges += arrays.edge_count; + total_shared_faces += arrays.triangle_count + arrays.quadrilateral_count; + } + + svert_lvert.SetSize(static_cast(total_shared_vertices)); + group_svert.SetDims(group_count, static_cast(total_shared_vertices)); + if (dimension >= 2) { + sedge_ledge.SetSize(static_cast(total_shared_edges)); + shared_edges.SetSize(static_cast(total_shared_edges)); + group_sedge.SetDims(group_count, static_cast(total_shared_edges)); + } + if (dimension >= 3) { + sface_lface.SetSize(static_cast(total_shared_faces)); + group_stria.MakeI(group_count); + group_squad.MakeI(group_count); + } else { + group_stria.SetSize(group_count, 0); + group_squad.SetSize(group_count, 0); + } + + // --------------------------------------------------------------------- + // Fill the shared-entity arrays and group tables, group by group in local order + // + // Vertex and edge tables are written directly in compressed-row form. Table::SetDims() + // allocated I (row offsets, size group_count + 1) and J (column entries) and set I[0] = 0; + // after row r is filled, I[r + 1] is set to the running total, which is both the end of + // row r and the start of row r + 1. J receives consecutive indices into svert_lvert and + // shared_edges. + // + // The face tables use mfem::Table's two-pass construction instead, because triangles and + // quadrilaterals are counted per row first: MakeI() above zeroed the per-row counts, + // AddColumnsInRow() records each row's count here, and MakeJ() below turns the counts + // into offsets. + // + // Within each group the entities are appended in stored order, which is the order every + // member rank agrees on, and the edge and face tuples are copied verbatim, keeping their + // stored orientation. Each shared edge becomes an mfem::Segment owned by the mesh, + // with attribute 1 as in MFEM's own constructors. Shared faces are plain vertex tuples + // (Vert3 / Vert4). + // --------------------------------------------------------------------- + int vertices_added = 0; + int edges_added = 0; + for (int row = 0; row < group_count; ++row) { + const SharedGroupArrays& arrays = shared_groups[static_cast(row)]; + + for (IndexType i = 0; i < arrays.vertex_count; ++i) { + group_svert.GetJ()[vertices_added] = vertices_added; + svert_lvert[vertices_added] = arrays.vertices[i]; + ++vertices_added; + } + group_svert.GetI()[row + 1] = vertices_added; + + if (dimension >= 2) { + for (IndexType i = 0; i < arrays.edge_count; ++i) { + group_sedge.GetJ()[edges_added] = edges_added; + shared_edges[edges_added] = new mfem::Segment(arrays.edges + 2 * i, 1); + ++edges_added; + } + group_sedge.GetI()[row + 1] = edges_added; + } + + if (dimension >= 3) { + for (IndexType i = 0; i < arrays.triangle_count; ++i) { + const int* v = arrays.triangles + 3 * i; + shared_trias.Append(Vert3(v[0], v[1], v[2])); + } + for (IndexType i = 0; i < arrays.quadrilateral_count; ++i) { + const int* v = arrays.quadrilaterals + 4 * i; + shared_quads.Append(Vert4(v[0], v[1], v[2], v[3])); + } + group_stria.AddColumnsInRow(row, static_cast(arrays.triangle_count)); + group_squad.AddColumnsInRow(row, static_cast(arrays.quadrilateral_count)); + } + } + + if (dimension >= 3) { + // sface_lface holds the triangles first and then the quadrilaterals, so its size has + // to equal the two lists combined for FinalizeParTopo() to index it correctly. + MFEM_VERIFY(shared_trias.Size() + shared_quads.Size() == sface_lface.Size(), + "Shared face count mismatch while rebuilding the mesh from the restart " + "checkpoint"); + // Row offsets from the per-row counts, then consecutive column entries: triangle + // (or quadrilateral) j of the whole list is entry j of the table. + group_stria.MakeJ(); + for (int i = 0; i < shared_trias.Size(); ++i) { + group_stria.GetJ()[i] = i; + } + group_squad.MakeJ(); + for (int i = 0; i < shared_quads.Size(); ++i) { + group_squad.GetJ()[i] = i; + } + } +} + +} // namespace exaconstit + +#endif // EXACONSTIT_USE_AXOM diff --git a/src/sim_state/exa_sidre_parmesh.hpp b/src/sim_state/exa_sidre_parmesh.hpp new file mode 100644 index 00000000..54e986f5 --- /dev/null +++ b/src/sim_state/exa_sidre_parmesh.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include "mfem.hpp" + +#include +#include +#include + +namespace axom { +namespace sidre { +class Group; +} // namespace sidre +} // namespace axom + +namespace exaconstit { + +/** + * @brief Version of the mesh subtree layout written by WriteParMeshBlueprint() + * + * Stored in `mesh/exaconstit/format_version` of every checkpoint rank file. Bump it whenever + * the layout written by WriteParMeshBlueprint() changes, so older checkpoints are rejected + * with a clear message instead of being misread. + */ +inline constexpr int MESH_CHECKPOINT_FORMAT_VERSION = 1; + +/** + * @brief Per-rank topology counts used to validate a reconstructed mesh + * + * @details Edges and faces are not stored in a checkpoint; MFEM regenerates them from the + * element connectivity. Comparing these counts with the ones recorded at write time checks + * that the regenerated local topology matches the original. + */ +struct MeshTopologyCounts { + /** @brief Number of local vertices (Mesh::GetNV) */ + std::int64_t num_vertices = 0; + /** @brief Number of local elements (Mesh::GetNE) */ + std::int64_t num_elements = 0; + /** @brief Number of local boundary elements (Mesh::GetNBE) */ + std::int64_t num_boundary_elements = 0; + /** @brief Number of local edges (Mesh::GetNEdges) */ + std::int64_t num_edges = 0; + /** @brief Number of local faces (Mesh::GetNFaces) */ + std::int64_t num_faces = 0; + /** @brief Number of communication groups, not counting group 0 (this rank alone) */ + std::int64_t num_communication_groups = 0; + + /** + * @brief Check whether every count agrees with another set of counts + * + * @param other Counts to compare against + * @return True if all six counts are equal + */ + bool Matches(const MeshTopologyCounts& other) const; +}; + +/** + * @brief Count the local topology of a parallel mesh + * + * @param mesh Parallel mesh to count + * @return Topology counts of this rank's piece of the mesh + * + * @details Purely local; no MPI communication takes place. + */ +MeshTopologyCounts CountMeshTopology(const mfem::ParMesh& mesh); + +/** + * @brief Read the topology counts recorded by WriteParMeshBlueprint() + * + * @param mesh_group Sidre group that WriteParMeshBlueprint() wrote into + * @return Topology counts stored under `exaconstit/` in the group + */ +MeshTopologyCounts ReadStoredMeshTopologyCounts(axom::sidre::Group* mesh_group); + +/** + * @brief Write this rank's piece of a parallel mesh into a Sidre group + * + * @param mesh Parallel mesh to write + * @param mesh_group Empty Sidre group to write into + * + * @details Writes the local vertices, elements, boundary elements and attributes, plus the + * description of how this piece connects to the other ranks' pieces (communication groups + * and shared vertices, edges and faces), so that ExaSidreParMesh can rebuild the identical + * ParMesh on the same rank. The layout follows the Conduit Blueprint mesh conventions, with + * ExaConstit additions under `adjsets/` and `exaconstit/`. All arrays are owned by Sidre. + * + * Collective over mesh.GetComm(). + * + * @pre The mesh is conforming and not NURBS, all elements share one geometry and all boundary + * elements share one geometry (segment, triangle, quadrilateral, tetrahedron or hexahedron), + * and mfem::real_t is double. Violations abort on every rank. + */ +void WriteParMeshBlueprint(const mfem::ParMesh& mesh, axom::sidre::Group* mesh_group); + +/** + * @brief Parallel mesh rebuilt from the output of WriteParMeshBlueprint() + * + * @details Restarted runs use this in place of reading the original mesh file, which may no + * longer exist. The result is a finalized, conforming mfem::ParMesh identical to the one that + * was written: the same local vertex and element numbering, element vertex order, + * attributes, communication groups, shared entities and shared-entity orientation. Finite + * element spaces built on it therefore number their local and true DOFs exactly as in the + * writing run. + * + * Invariants callers must respect: + * - The run must use the same number of MPI ranks as the writing run, and rank r must read + * the group that rank r wrote. + * - The mesh has no Nodes grid function. Code that needs high-order or deformed coordinates + * must supply them (SimulationState overlays the checkpointed coordinates). + * + * This is a subclass only because the members needed to rebuild a ParMesh from its pieces + * are protected; it adds nothing beyond its constructor. + */ +class ExaSidreParMesh : public mfem::ParMesh { +public: + /** + * @brief Rebuild a parallel mesh from a checkpoint's mesh group + * + * @param mesh_group Group written by WriteParMeshBlueprint(). Non-const because Sidre's + * typed accessor View::getData() is non-const. + * @param comm Communicator with the same number of ranks as the writing run + * + * @details Collective over comm. Every array is checked on every rank before anything is + * built, and a bad checkpoint aborts on every rank. Nothing points into the Sidre buffers + * once the constructor returns, so the DataStore holding mesh_group may be destroyed + * right away. + */ + ExaSidreParMesh(axom::sidre::Group* mesh_group, MPI_Comm comm); + + ExaSidreParMesh(const ExaSidreParMesh&) = delete; + ExaSidreParMesh& operator=(const ExaSidreParMesh&) = delete; + ~ExaSidreParMesh() override = default; + + /** + * @brief Rebuild a parallel mesh and return it as a shared pointer + * + * @param mesh_group Group written by WriteParMeshBlueprint() + * @param comm Communicator with the same number of ranks as the writing run + * @return Shared pointer to the rebuilt mesh, ready to assign to SimulationState's mesh + */ + static std::shared_ptr Create(axom::sidre::Group* mesh_group, MPI_Comm comm) { + return std::make_shared(mesh_group, comm); + } + +private: + /** + * @brief Rebuild the communication groups and the shared-entity tables + * + * @param adjset_groups The `adjsets/mesh/groups` group written by WriteParMeshBlueprint() + * @param dimension Topological dimension of the mesh + * + * @details Collective over the mesh communicator. Must be called after ReduceMeshGen() + * and before Finalize(). + */ + void BuildSharedStructures(axom::sidre::Group* adjset_groups, int dimension); +}; + +} // namespace exaconstit diff --git a/src/sim_state/restart_checkpoint.cpp b/src/sim_state/restart_checkpoint.cpp new file mode 100644 index 00000000..8058f736 --- /dev/null +++ b/src/sim_state/restart_checkpoint.cpp @@ -0,0 +1,1979 @@ +// Compiled only when ExaConstit is built with Axom, so this file can always be listed in +// CMake; without EXACONSTIT_USE_AXOM it is an empty translation unit. +#if defined(EXACONSTIT_USE_AXOM) + +#include "sim_state/restart_checkpoint.hpp" + +#include "sim_state/exa_sidre_parmesh.hpp" +#include "sim_state/restart_sidre_utils.hpp" +#include "sim_state/simulation_state.hpp" +#include "utilities/mechanics_log.hpp" +#include "utilities/unified_logger.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" + +#if !defined(AXOM_USE_HDF5) +#error "ExaConstit restart uses the sidre_hdf5 protocol; build Axom with HDF5 (AXOM_USE_HDF5)." +#endif + +// ========================================================================= +// BACKGROUND: WHAT A CHECKPOINT CONTAINS AND HOW A RESTART USES IT +// ========================================================================= +// +// Read this once before changing anything below; every function in this file implements a +// piece of it. doc/restart.md has the user-facing description and the full design record. +// +// Goal. A restarted run must continue exactly as if the original run had never stopped, +// using only the checkpoint and the options file. The mesh, grain, region-mapping, +// orientation and state-variable files may no longer exist. For a run with the same number of +// ranks, the restarted run is bitwise identical to the uninterrupted one +// (test/test_restart_runs.py checks this on real runs). +// +// Rank file layout. Every rank writes one Sidre DataStore to +// /cycle_/rank_.hdf5 with these groups: +// +// restart_info/ format version, rank count, this rank, completed cycles, driver step, +// completed time, next dt, time-step type, mesh order, ExaConstit version +// mesh/ this rank's piece of the mesh (exa_sidre_parmesh.cpp), plus +// fields/grain_id (the grain ID of every local element) +// fields/ mesh_ref/values reference coordinates (owned copy, see below) +// mesh_current/values current coordinates +// primal_field/values velocity true DOFs +// mesh_space/ sizes and a DOF-layout hash of the coordinate space +// qspaces/ validation data for each quadrature space (order, size, element subset) +// qfuncs/ each quadrature function: its space, vdim and size, plus either its +// values or `copy_of = ` (see WriteQuadratureFunctions()) +// materials/ name, model type, state-variable count and properties of every region +// +// Registry names are made into single Sidre path segments with SidreSafeName(), because +// Sidre uses '/' as its path separator. +// +// Owned and external views. Sidre can either copy data into memory it owns ("owned" view) +// or record a pointer to memory someone else owns ("external" view). The large arrays +// (current coordinates, velocity, quadrature functions) are written as external views +// pointing straight at SimulationState's host data, so writing a checkpoint copies nothing in +// memory. On read, Group::load() restores an external view as a description only (type and +// size, no data). The reader then points each one at the SimulationState buffer it should +// fill (View::setExternalDataPtr()) and a single Group::loadExternalData() call reads all of +// them from the file straight into place. Small and metadata arrays are owned, and so is +// mesh_ref, because it is needed in phase 1, before the buffers phase 2 fills even exist. +// +// Commit protocol. Every rank saves its file, the ranks agree that all saves worked, and only +// then does rank 0 write commit.json (to a temporary name, then renamed, which is atomic on +// POSIX file systems). A checkpoint directory without commit.json is incomplete and is never +// selected for a restart, so a crash in the middle of a write can never be restarted from. +// +// Two-phase restart. A fresh run builds every object from the input files: SimulationState, +// then SystemDriver (the operator, the material models and the solvers), then +// PostProcessingDriver. A restarted run builds them in exactly the same way and then +// overwrites the evolving state with the checkpoint ("build first, overlay last"). Anything a +// constructor initializes (for example ExaCMechModel::InitStateVars()) is simply replaced, +// and quadrature functions registered by any component are restored as long as they exist +// by then. The one unavoidable exception is that SimulationState's constructor would read +// input files that may be gone, so on restart runs it takes those few things from the +// checkpoint instead (phase 1): +// - the mesh (ReconstructMesh()), since every space is built on it; +// - the reference coordinates (RestoreReferenceConfiguration()), see that function for why +// the exact bits matter even during construction; +// - the grain IDs and region IDs (ReadElementGrainIds() and the element attributes), since +// they decide which region spaces and functions the constructor creates; +// - the orientation and state-variable initialization is skipped, since phase 2 overwrites +// that state anyway. +// Phase 2 (RestoreState()) is the overlay itself, called from the driver once every object +// exists. +// +// Ranks with no elements. Every function here must work on a rank that owns no elements, and +// such a rank must still reach every collective call. On such a rank the mesh arrays, the +// coordinate and velocity vectors and the global quadrature functions all have zero length, +// and the region quadrature spaces and functions do not exist at all (SimulationState only +// creates them where the region has elements). Zero-length arrays are written as described +// but unallocated views and are never dereferenced; their data pointers may be null. Each +// rank only ever reads the file it wrote, so a region that does not exist on a rank is absent +// from both the checkpoint and the restarted run on that rank. test_restart_mesh_roundtrip +// and test_restart_runs both include cases with empty ranks. +// +// DataStore lifetimes. A DataStore that finds Axom's logger (SLIC) uninitialized initializes +// it and finalizes it again in its destructor. Two DataStores alive at the same time could +// therefore leave the second one logging through a finalized logger, so every DataStore in +// this file is destroyed before the next one is created: the write store before the commit +// store, and the commit store before the read store. + +namespace fs = std::filesystem; + +namespace exaconstit { + +namespace { + +using axom::sidre::DataStore; +using axom::sidre::Group; +using axom::sidre::IndexType; +using axom::sidre::View; +using restart_detail::AllRanksTrue; +using restart_detail::ReadSidreScalar; + +/** @brief File name of the commit marker that makes a checkpoint directory usable */ +constexpr const char* COMMIT_FILE_NAME = "commit.json"; +/** @brief Sidre protocol of the commit file (small, human readable) */ +constexpr const char* COMMIT_PROTOCOL = "sidre_json"; +/** @brief Sidre protocol of the rank files */ +constexpr const char* RANK_FILE_PROTOCOL = "sidre_hdf5"; +/** @brief Number of digits of the cycle count in a checkpoint directory name */ +constexpr std::size_t CYCLE_DIGITS = 10; + +/** + * @brief Name of the checkpoint directory for a completed-cycle count + * + * @param completed_cycles Number of completed cycles the checkpoint holds + * @return `cycle_` followed by the count, zero-padded to CYCLE_DIGITS digits + * + * @details Zero padding makes the directories sort in cycle order by name as well. + */ +std::string CycleDirectoryName(const std::size_t completed_cycles) { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "cycle_%010zu", completed_cycles); + return std::string(buffer); +} + +/** + * @brief Name of a rank's file inside a checkpoint directory + * + * @param rank MPI rank + * @return `rank_` followed by the rank, zero-padded to 6 digits, and `.hdf5` + */ +std::string RankFileName(const int rank) { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "rank_%06d.hdf5", rank); + return std::string(buffer); +} + +/** + * @brief Parse a checkpoint directory name + * + * @param name Directory name (not a path) + * @param completed_cycles Receives the cycle count when the name matches + * @return True if the name is exactly `cycle_` followed by CYCLE_DIGITS digits + * + * @details The match is exact, so renamed directories (`cycle_.superseded`) and anything + * else a user leaves in the checkpoint directory are never mistaken for checkpoints. + */ +bool ParseCycleDirectoryName(const std::string& name, std::size_t& completed_cycles) { + const std::string prefix = "cycle_"; + if (name.size() != prefix.size() + CYCLE_DIGITS || + name.compare(0, prefix.size(), prefix) != 0) { + return false; + } + for (std::size_t i = prefix.size(); i < name.size(); ++i) { + if (name[i] < '0' || name[i] > '9') { + return false; + } + } + completed_cycles = static_cast(std::stoull(name.substr(prefix.size()))); + return true; +} + +/** + * @brief A checkpoint directory found on disk + */ +struct CheckpointDirectoryEntry { + /** @brief Completed cycles the checkpoint holds (from its name) */ + std::size_t completed_cycles = 0; + /** @brief Path of the checkpoint directory */ + fs::path path; + /** @brief Whether the directory has a commit file, i.e. is complete */ + bool committed = false; +}; + +/** + * @brief List the checkpoint directories under a directory, oldest first + * + * @param directory Checkpoint root directory + * @return One entry per `cycle_` subdirectory, sorted by completed cycles; empty if + * the directory does not exist + * + * @details Only rank 0 calls this. File system errors are treated as "no checkpoints" rather + * than thrown, since the callers decide what a missing checkpoint means. + */ +std::vector ListCheckpointDirectories(const fs::path& directory) { + std::vector entries; + std::error_code error; + if (!fs::is_directory(directory, error)) { + return entries; + } + for (const auto& item : fs::directory_iterator(directory, error)) { + std::size_t completed_cycles = 0; + if (item.is_directory(error) && + ParseCycleDirectoryName(item.path().filename().string(), completed_cycles)) { + entries.push_back( + {completed_cycles, item.path(), fs::exists(item.path() / COMMIT_FILE_NAME, error)}); + } + } + std::sort(entries.begin(), entries.end(), [](const auto& lhs, const auto& rhs) { + return lhs.completed_cycles < rhs.completed_cycles; + }); + return entries; +} + +/** + * @brief Turn a SimulationState registry name into a single Sidre path segment + * + * @param name Registry name (quadrature space or quadrature function name) + * @return The name with '%' encoded as "%25" and '/' as "%2F" + * + * @details Sidre treats '/' as a path separator, so a name containing it would otherwise + * create nested groups. The same encoding is used on write and read, so the round trip only + * needs the two to agree; scripts/restart/compare_restart_checkpoints.cpp uses it too. + */ +std::string SidreSafeName(const std::string& name) { + std::string encoded; + encoded.reserve(name.size()); + for (const char character : name) { + if (character == '/') { + encoded += "%2F"; + } else if (character == '%') { + encoded += "%25"; + } else { + encoded += character; + } + } + return encoded; +} + +/** + * @brief Broadcast a string from rank 0 + * + * @param value On rank 0 the string to send; on every other rank receives it + * @param comm Communicator to broadcast over + * + * @details Collective. The length is broadcast first so the receivers can size the string. + */ +void BroadcastString(std::string& value, MPI_Comm comm) { + int length = static_cast(value.size()); + MPI_Bcast(&length, 1, MPI_INT, 0, comm); + value.resize(static_cast(length)); + if (length > 0) { + MPI_Bcast(value.data(), length, MPI_CHAR, 0, comm); + } +} + +/** + * @brief 64-bit FNV-1a hash + * + * @details A simple, deterministic, order-sensitive hash with no dependencies. It is used to + * fingerprint arrays that are too large to store just for validation (DOF layouts and element + * subsets). It is not cryptographic; it only has to make an accidental match between two + * different layouts very unlikely. + */ +class Fnv1a64 { +public: + /** + * @brief Add raw bytes to the hash + * + * @param data Bytes to add + * @param byte_count Number of bytes + */ + void AddBytes(const void* data, const std::size_t byte_count) { + const auto* bytes = static_cast(data); + for (std::size_t i = 0; i < byte_count; ++i) { + m_state ^= static_cast(bytes[i]); + m_state *= 1099511628211ULL; + } + } + + /** + * @brief Current hash value + * + * @return The hash of every byte added so far + */ + std::uint64_t Value() const { + return m_state; + } + +private: + /** @brief Running hash state, starting from the FNV-1a 64-bit offset basis */ + std::uint64_t m_state = 1469598103934665603ULL; +}; + +/** + * @brief Hash of an integer array's contents + * + * @param values Array to hash + * @return FNV-1a hash of the entries in order; the hash of nothing for an empty array + * + * @details Empty arrays (for example the element subset of the global quadrature space on a + * rank with no elements) are never read, since their host pointer may be null. + */ +std::uint64_t HashIntArray(const mfem::Array& values) { + Fnv1a64 hasher; + if (values.Size() > 0) { + hasher.AddBytes(values.HostRead(), static_cast(values.Size()) * sizeof(int)); + } + return hasher.Value(); +} + +/** + * @brief Hash of a grid function read element by element in each element's DOF order + * + * @param field Grid function to hash + * @return FNV-1a hash of the values visited element by element + * + * @details For each local element, the field's values are visited through + * GetElementVDofs(), i.e. in the order the element's DOFs are numbered locally. If the + * rebuilt space numbers the local DOFs differently from the writing run, the stored + * coordinate vector lands on different DOFs and the visited sequence, and so the hash, + * changes. Coordinates at distinct nodes are distinct, so a renumbering cannot reproduce the + * same sequence by accident. + * + * MFEM encodes a DOF with reversed orientation as a negative index -1 - dof whose value must + * be negated; that convention is followed so the hash matches what an element actually sees. + * On a rank with no elements nothing is visited and the hash is that of an empty sequence, + * the same in both runs. + */ +std::uint64_t ElementLocalFieldHash(const mfem::ParGridFunction& field) { + const mfem::ParFiniteElementSpace* space = field.ParFESpace(); + const double* values = field.HostRead(); + mfem::Array element_vdofs; + Fnv1a64 hasher; + for (int element_index = 0; element_index < space->GetNE(); ++element_index) { + space->GetElementVDofs(element_index, element_vdofs); + for (const int signed_vdof : element_vdofs) { + const int vdof = (signed_vdof >= 0) ? signed_vdof : (-1 - signed_vdof); + const double value = (signed_vdof >= 0) ? values[vdof] : -values[vdof]; + hasher.AddBytes(&value, sizeof(value)); + } + } + return hasher.Value(); +} + +/** + * @brief Largest change of a grid function when restricted to true DOFs and prolongated back + * + * @param field Grid function to test + * @return max |field - P(R(field))| over this rank's local DOFs + * + * @details R (GridFunction::GetTrueDofs()) keeps one value per true DOF, the one held by the + * owning rank, and P (ParGridFunction::SetFromTrueDofs()) copies it back to every rank that + * shares the DOF. A field that was continuous in the writing run survives this unchanged if, + * and only if, every shared DOF is matched with the same true DOF on every rank. A + * mismatched shared DOF, for example from a wrong shared-face orientation, receives another + * DOF's value and shows up as a jump of the order of the element size. + * + * Collective: the prolongation exchanges shared values between ranks, so every rank must + * call this, including ranks with no DOFs (which return 0). + */ +double SharedDofInconsistency(const mfem::ParGridFunction& field) { + mfem::Vector true_values; + field.GetTrueDofs(true_values); + mfem::ParGridFunction prolongated(field.ParFESpace()); + prolongated.SetFromTrueDofs(true_values); + const double* original = field.HostRead(); + const double* round_trip = prolongated.HostRead(); + double max_difference = 0.0; + for (int i = 0; i < field.Size(); ++i) { + max_difference = std::max(max_difference, std::abs(original[i] - round_trip[i])); + } + return max_difference; +} + +/** + * @brief Create an external double array view + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param data Host data the view points at; must stay valid until the DataStore is saved + * @param count Number of entries + * + * @details No data is copied; Group::save() reads straight from data. Zero-length arrays are + * only described, since their data pointer may be null. Sidre's external-view API takes a + * non-const pointer, but saving only reads through it. + */ +void CreateExternalDoubleView(Group* group, + const std::string& path, + const double* data, + const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return; + } + group->createView(path, axom::sidre::FLOAT64_ID, count, const_cast(data)); +} + +/** + * @brief Create an owned double array view holding a copy of some data + * + * @param group Group to create the view in + * @param path Path of the new view relative to the group + * @param data Data to copy (may be null when count is 0) + * @param count Number of entries + * + * @details Zero-length arrays are only described, not allocated. + */ +void CreateOwnedDoubleCopy(Group* group, + const std::string& path, + const double* data, + const IndexType count) { + if (count == 0) { + group->createView(path, axom::sidre::FLOAT64_ID, 0); + return; + } + double* stored = + group->createViewAndAllocate(path, axom::sidre::FLOAT64_ID, count)->getData(); + std::copy(data, data + count, stored); +} + +/** + * @brief Name under which a quadrature space is registered in SimulationState + * + * @param spaces SimulationState's quadrature space registry + * @param space Space to look up + * @return The registry name, or an empty string if the space is not registered + * + * @details Quadrature functions only hold a pointer to their space, so the name is found by + * searching the registry. It is stored with each quadrature function so the reader can check + * that the function still lives on the same space. + */ +std::string FindQuadratureSpaceName( + const std::map>& spaces, + const mfem::expt::PartialQuadratureSpace* space) { + for (const auto& [name, candidate] : spaces) { + if (candidate.get() == space) { + return name; + } + } + return std::string(); +} + +/** + * @brief Quadrature functions that are recomputed before they are next read, and so are + * neither written nor restored + * + * @param sim_state Simulation state whose registry names to use + * @return Registry names of the global and every region's tangent stiffness + * + * @details The tangent stiffness is the largest quadrature function (36 values per point, + * both globally and per region) and it is always recomputed before any read: + * - Every NonlinearMechOperator::Setup() calls the material model's ModelSetup(), which + * recomputes the tangent from the converged state. Setup() runs at the start of every + * Newton residual evaluation (Mult()) and in SolveInit() after a boundary-condition change + * (GetUpdateBCsAction()). + * - The integrators read tangent values only while assembling the Jacobian + * (AssembleElementGrad(), AssembleGradPA(), AssembleEA(), AssembleGradDiagonalPA(), + * AddMult[Transpose]GradPA()), which always happens after such a Setup(). + * ICExaNLFIntegrator::AddMultPA() only uses the function's space (its integration rule), + * which the restarted run's constructor-built function provides. + * If a future change reads the tangent before the first Setup() of a step, it must be removed + * from this list so it is checkpointed again. + * + * The region names are generated for every region, including regions with no elements on + * this rank; looking them up in a set is harmless when they are absent from the registry. + */ +std::set RecomputedQuadratureFunctionNames(const SimulationState& sim_state) { + std::set names; + names.insert(sim_state.GetQuadratureFunctionMapName("tangent_stiffness", -1)); + for (std::size_t region = 0; region < sim_state.GetNumberOfRegions(); ++region) { + names.insert( + sim_state.GetQuadratureFunctionMapName("tangent_stiffness", static_cast(region))); + } + return names; +} + +/** + * @brief Collect the external views that have not been pointed at a buffer + * + * @param group Group to search, recursively + * @param out Receives (group, view name) for every external view with a null data pointer + * + * @details Group::loadExternalData() tries to read every external view described in the + * group, bound or not, so the unbound ones (zero-length arrays and quadrature functions + * present only in the checkpoint) have to be removed first; see RestoreState(). + */ +void CollectUnboundExternalViews(Group* group, std::vector>& out) { + for (View& view : group->views()) { + if (view.isExternal() && view.getVoidPtr() == nullptr) { + out.emplace_back(group, view.getName()); + } + } + for (Group& child : group->groups()) { + CollectUnboundExternalViews(&child, out); + } +} + +} // namespace + +// ========================================================================= +// CONSTRUCTION AND FAILURE REPORTING +// ========================================================================= + +RestartCheckpoint::RestartCheckpoint(const RestartOptions& options, MPI_Comm comm) + : m_options(options), m_comm(comm) { + MPI_Comm_rank(m_comm, &m_rank); + MPI_Comm_size(m_comm, &m_num_ranks); +} + +RestartCheckpoint::~RestartCheckpoint() = default; + +void RestartCheckpoint::RecordFailure(const std::string& message) { + m_local_failed = true; + m_local_failure_log += "\n - " + message; +} + +/** + * @details Implementation notes. Most checks in this file can fail on some ranks and not + * others, for example a malformed array in one rank's file. Aborting only on the failing rank + * would leave every other rank waiting forever in its next collective call, so each check + * records its failure (RecordFailure()) and every stage ends with this function, which every + * rank calls. The ranks agree on whether anyone failed. If so, each failing rank reports its + * own problems with MFEM_WARNING, since only it knows them, and every rank then aborts with a + * single shared MFEM_ABORT_0 message. + */ +void RestartCheckpoint::AbortIfAnyRankFailed(const char* stage) { + if (AllRanksTrue(!m_local_failed, m_comm)) { + m_local_failure_log.clear(); + return; + } + if (m_local_failed) { + MFEM_WARNING("Restart failed on rank " << m_rank << " while " << stage << ":" + << m_local_failure_log); + } + MFEM_ABORT_0("Restart failed while " << stage << " (see the per-rank warnings above)"); +} + +// ========================================================================= +// WRITING +// ========================================================================= + +/** + * @details Implementation notes. The decision uses only values every rank agrees on (the + * options, the time manager and m_last_written_cycles), so every rank makes the same decision + * and either all ranks or no ranks enter the collective WriteCheckpoint(). + * + * - IsCheckpointSafe(): the time manager is not in the middle of a retrial or sub-step + * sequence, whose counters are not checkpointed. With the current SystemDriver::Solve() this + * always holds after a converged step: retries loop inside Solve() until the step succeeds, + * AUTO stepping never sub-steps, and FIXED / CUSTOM stepping always report success. The + * guard keeps checkpoints correct if sub-stepping across driver iterations is ever added. + * - Counting cycles since the last checkpoint, rather than testing cycle % interval, keeps + * the interval correct after a restart from an arbitrary cycle and if a write is deferred. + * - `completed_cycles <= m_last_written_cycles` prevents writing the same cycle twice, for + * example a final checkpoint right after an interval checkpoint of the same cycle, and + * writing immediately after a restart. + */ +void RestartCheckpoint::WriteCheckpointIfDue(SimulationState& sim_state, + const std::size_t driver_step) { + if (!m_options.write) { + return; + } + const TimeManagement& time_manager = sim_state.m_time_manager; + if (!time_manager.IsCheckpointSafe()) { + return; + } + const TimeManagement::CheckpointState time_state = time_manager.GetCheckpointState(); + if (time_state.completed_cycles <= m_last_written_cycles) { + return; + } + const std::size_t cycles_since_last = time_state.completed_cycles - m_last_written_cycles; + const bool interval_due = m_options.interval > 0 && + cycles_since_last >= static_cast(m_options.interval); + const bool final_due = time_manager.IsFinished() && m_options.write_final; + if (!interval_due && !final_due) { + return; + } + WriteCheckpoint(sim_state, driver_step, time_state); +} + +/** + * @details Implementation notes. A checkpoint is written in three stages: + * + * 1. Rank 0 prepares the directory (PrepareCheckpointDirectory()). If that fails the + * checkpoint is skipped with a warning and the simulation continues; losing one checkpoint + * is better than losing the run. + * 2. Every rank fills a DataStore with its state and saves it to its rank file. The helpers + * each write one top-level group (see the background notes at the top of this file for + * the layout): + * WriteRestartInfo(), WriteMeshAndGrainIds(), WriteNodalFields(), + * WriteQuadratureSpaces(), WriteQuadratureFunctions() and WriteMaterials(). + * The large arrays are external views of SimulationState's host data, so nothing is copied + * in memory, and SimulationState must not change between building the store and saving it. + * The DataStore is destroyed right after saving, freeing the owned arrays (the mesh copy, + * mesh_ref and the small metadata). The external views never owned their memory. + * 3. Commit. The ranks agree that every save succeeded. Only then does rank 0 write + * commit.json, which is what makes the checkpoint usable, and apply the retention policy. + * The result is broadcast so every rank updates m_last_written_cycles consistently, and a + * checkpoint that was not committed is reported as a warning and does not count, so the + * next due cycle tries again. + * + * Collective; every rank, including ranks with no elements, takes part in every stage. + */ +void RestartCheckpoint::WriteCheckpoint(SimulationState& sim_state, + const std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::WriteCheckpoint"); + const double start_seconds = MPI_Wtime(); + const std::size_t completed_cycles = time_state.completed_cycles; + const double completed_time = time_state.completed_time; + const fs::path checkpoint_directory = m_options.directory / + CycleDirectoryName(completed_cycles); + + // --------------------------------------------------------------------- + // Stage 1: directory + // --------------------------------------------------------------------- + if (!PrepareCheckpointDirectory(checkpoint_directory)) { + MFEM_WARNING_0("Unable to prepare the checkpoint directory " + << checkpoint_directory << "; the checkpoint for cycle " << completed_cycles + << " was skipped and the simulation continues"); + return; + } + + // --------------------------------------------------------------------- + // Stage 2: every rank builds and saves its rank file + // --------------------------------------------------------------------- + bool saved = false; + { + DataStore store; + Group* root = store.getRoot(); + WriteRestartInfo(root, sim_state, driver_step, time_state); + WriteMeshAndGrainIds(root, sim_state); + WriteNodalFields(root, sim_state); + WriteQuadratureSpaces(root, sim_state); + WriteQuadratureFunctions(root, sim_state); + WriteMaterials(root, sim_state); + saved = root->save((checkpoint_directory / RankFileName(m_rank)).string(), + RANK_FILE_PROTOCOL); + } + + // --------------------------------------------------------------------- + // Stage 3: commit (only if every rank saved its file) + // --------------------------------------------------------------------- + const bool all_saved = AllRanksTrue(saved, m_comm); + int committed = 0; + if (all_saved && m_rank == 0) { + committed = WriteCommitFile( + checkpoint_directory, completed_cycles, driver_step, completed_time) + ? 1 + : 0; + if (committed != 0) { + ApplyRetentionPolicy(); + } + } + MPI_Bcast(&committed, 1, MPI_INT, 0, m_comm); + + if (committed == 0) { + MFEM_WARNING_0("The checkpoint " << checkpoint_directory + << " was not committed (a rank file could not be " + "saved or the commit file could not be written); " + "the simulation continues"); + return; + } + m_last_written_cycles = completed_cycles; + if (m_rank == 0) { + std::cout << "Wrote restart checkpoint " << checkpoint_directory.string() << " (cycle " + << completed_cycles << ", driver step " << driver_step + << ", t = " << completed_time << ") in " << (MPI_Wtime() - start_seconds) << " s" + << std::endl; + } +} + +/** + * @brief Create the directory for a checkpoint about to be written + * + * @param checkpoint_directory Directory of the checkpoint + * @return True on every rank if the directory is ready, false on every rank otherwise + * + * @details Collective. Only rank 0 touches the file system, and its result is broadcast so + * every rank makes the same decision. Rank 0: + * - supersedes checkpoints of another timeline on this run's first write + * (SupersedeCheckpointsAfter()); + * - creates the directory; + * - removes an existing commit file, so a directory being rewritten for the same cycle is + * marked incomplete until the new commit, rather than appearing complete with a mix of old + * and new rank files. + * The supersede step is attempted only once, even if this write fails, since it renames + * rather than deletes and retrying it gains nothing. + */ +bool RestartCheckpoint::PrepareCheckpointDirectory(const fs::path& checkpoint_directory) { + int prepared = 1; + if (m_rank == 0) { + try { + if (m_supersede_pending) { + SupersedeCheckpointsAfter(m_last_written_cycles); + } + fs::create_directories(checkpoint_directory); + fs::remove(checkpoint_directory / COMMIT_FILE_NAME); + } catch (const fs::filesystem_error& error) { + MFEM_WARNING_0("Unable to prepare " << checkpoint_directory << ": " << error.what()); + prepared = 0; + } + } + m_supersede_pending = false; + MPI_Bcast(&prepared, 1, MPI_INT, 0, m_comm); + return prepared != 0; +} + +/** + * @brief Write the `restart_info/` group + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * @param driver_step Driver loop counter of the step just completed + * @param time_state Resumable time state of the time manager + * + * @details The format version, rank count and rank let OpenForRestart() reject a checkpoint + * that this build or this run cannot read before anything else is touched. The time state is + * stored as unambiguous quantities (see TimeManagement::CheckpointState) rather than the time + * manager's raw members, whose meaning depends on its internal state. The previous and + * original time steps are only used in log messages; storing them lets the restarted run log + * exactly what the uninterrupted run would have. + * + * The driver step is stored separately from the cycle count even though the two are equal + * with the current SystemDriver::Solve(). The boundary-condition schedule and post-processing + * are keyed by the driver step, and storing it separately means a restart never relies on the + * two staying equal. + * + * The time-step type, mesh order and ExaConstit version are informational; the mesh order is + * checked separately through the mesh space (ValidateMeshSpace()). + */ +void RestartCheckpoint::WriteRestartInfo(Group* root, + const SimulationState& sim_state, + const std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) const { + Group* info = root->createGroup("restart_info"); + info->createViewScalar("format_version", RESTART_FORMAT_VERSION); + info->createViewScalar("num_ranks", m_num_ranks); + info->createViewScalar("rank", m_rank); + info->createViewScalar("completed_cycles", + static_cast(time_state.completed_cycles)); + info->createViewScalar("driver_step", static_cast(driver_step)); + info->createViewScalar("completed_time", time_state.completed_time); + info->createViewScalar("next_dt", time_state.next_dt); + info->createViewScalar("previous_dt", time_state.previous_dt); + info->createViewScalar("original_dt", time_state.original_dt); + info->createViewScalar("time_type", + static_cast(sim_state.m_time_manager.GetTimeStepType())); + info->createViewScalar("mesh_order", sim_state.m_options.mesh.order); + info->createViewString("exaconstit_version", sim_state.m_options.version); +} + +/** + * @brief Write the `mesh/` group: this rank's mesh piece and the element grain IDs + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details The mesh is written by WriteParMeshBlueprint(), which is collective (the ranks + * agree on the element geometries), so every rank must call this. + * + * The grain IDs are written as an extra element field, `fields/grain_id`. A restarted run + * needs them because the element attributes no longer hold grain IDs once SimulationState has + * been set up: its constructor replaces them with region IDs, and the grain-to-region map that + * links the two comes from the region-mapping file, which may no longer exist. SimulationState + * keeps the grain IDs as a piecewise-constant grid function with one value per local element, + * stored as doubles; they are whole numbers and are converted back to int here. On a rank + * with no elements the array is empty. + */ +void RestartCheckpoint::WriteMeshAndGrainIds(Group* root, const SimulationState& sim_state) const { + Group* mesh_group = root->createGroup("mesh"); + WriteParMeshBlueprint(*sim_state.m_mesh, mesh_group); + + const int num_elements = sim_state.m_mesh->GetNE(); + mesh_group->createViewString("fields/grain_id/association", "element"); + mesh_group->createViewString("fields/grain_id/topology", "mesh"); + if (num_elements == 0) { + mesh_group->createView("fields/grain_id/values", axom::sidre::INT_ID, 0); + return; + } + int* grain_ids = mesh_group + ->createViewAndAllocate( + "fields/grain_id/values", axom::sidre::INT_ID, num_elements) + ->getData(); + const double* grain_values = sim_state.m_grains->HostRead(); + for (int element_index = 0; element_index < num_elements; ++element_index) { + grain_ids[element_index] = static_cast(grain_values[element_index]); + } +} + +/** + * @brief Write the `fields/` group: the nodal state and a fingerprint of its DOF layout + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Only the state that cannot be recomputed is stored: + * - `mesh_current`: the current coordinates (all local DOFs, i.e. VSize values). + * - `primal_field`: the velocity true DOFs (TrueVSize values), which the solver works with. + * - `mesh_ref`: the reference coordinates. These are stored as an owned copy rather than an + * external view, because phase 1 (RestoreReferenceConfiguration()) needs them inside the + * SimulationState constructor, before the read side has buffers to bind external views to. + * Everything else is derived from these after the restart (ApplyPostRestartFixup()): + * `mesh_t_beg`, the previous velocity, the velocity grid function and the displacement. + * + * External views read HostRead() data. In MFEM's CUDA, HIP and Umpire device memory spaces + * HostRead() makes a synchronous device-to-host copy, so the host data is current when the + * file is saved. (A blocking device copy only orders against the default stream, which is the + * stream the solver uses.) + * + * `mesh_space/` records the sizes of the coordinate space (local, owned true and global true + * DOFs), the mesh order and ElementLocalFieldHash() of mesh_ref. The restarted run checks + * these before interpreting the stored coordinates (ValidateMeshSpace() and + * VerifyRestoredDofLayout()), since the vectors are only meaningful if the rebuilt space + * numbers its DOFs exactly as the writing run did. + * + * On a rank with no elements every vector here is empty; the views are only described and the + * hash is that of an empty sequence. + */ +void RestartCheckpoint::WriteNodalFields(Group* root, const SimulationState& sim_state) const { + Group* fields = root->createGroup("fields"); + const mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + const mfem::ParGridFunction& mesh_current = *sim_state.m_mesh_nodes.at("mesh_current"); + const mfem::Vector& primal_field = *sim_state.m_primal_field; + + CreateOwnedDoubleCopy(fields, + "mesh_ref/values", + mesh_ref.Size() > 0 ? mesh_ref.HostRead() : nullptr, + mesh_ref.Size()); + CreateExternalDoubleView(fields, + "mesh_current/values", + mesh_current.Size() > 0 ? mesh_current.HostRead() : nullptr, + mesh_current.Size()); + CreateExternalDoubleView(fields, + "primal_field/values", + primal_field.Size() > 0 ? primal_field.HostRead() : nullptr, + primal_field.Size()); + + const mfem::ParFiniteElementSpace& mesh_space = *sim_state.m_mesh_fes; + Group* space_info = fields->createGroup("mesh_space"); + space_info->createViewScalar("order", sim_state.m_options.mesh.order); + space_info->createViewScalar("vsize", static_cast(mesh_space.GetVSize())); + space_info->createViewScalar("true_vsize", static_cast(mesh_space.TrueVSize())); + space_info->createViewScalar("global_true_vsize", + static_cast(mesh_space.GlobalTrueVSize())); + space_info->createViewScalar("mesh_ref_element_hash", ElementLocalFieldHash(mesh_ref)); +} + +/** + * @brief Write the `qspaces/` group: validation data for every quadrature space + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Quadrature spaces are not restored; the restarted run rebuilds them in its + * constructor from the mesh and the regions. What is stored lets ValidateQuadratureSpaces() + * confirm that each rebuilt space matches the original one point for point, so a quadrature + * function's stored values can be copied straight into the rebuilt function: the integration + * order, the number of quadrature point values, the number of local elements, whether the + * space covers the whole local mesh, and a hash of its local-to-global element map (which + * elements, in which order). + * + * The global spaces exist on every rank, possibly empty. Region spaces only exist on ranks + * that own elements of the region, so they are only written there; the same rank of the + * restarted run will not have them either. + */ +void RestartCheckpoint::WriteQuadratureSpaces(Group* root, const SimulationState& sim_state) const { + Group* qspaces = root->createGroup("qspaces"); + for (const auto& [name, space] : sim_state.m_map_qs) { + if (!space) { + continue; + } + Group* entry = qspaces->createGroup(SidreSafeName(name)); + entry->createViewScalar("order", space->GetOrder()); + entry->createViewScalar("size", static_cast(space->GetSize())); + entry->createViewScalar("num_local_elements", + static_cast(space->GetNumLocalElements())); + entry->createViewScalar("is_full_space", space->IsFullSpace() ? 1 : 0); + entry->createViewScalar("local2global_hash", HashIntArray(space->GetLocal2Global())); + } +} + +/** + * @brief Write the `qfuncs/` group: every registered quadrature function + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details Every non-null function in SimulationState's registry is written, except the ones + * that are recomputed before they are next read (RecomputedQuadratureFunctionNames()). Writing + * everything rather than a hand-picked list means functions added by future models or + * post-processing are checkpointed without any change here. Each entry records the name of + * the function's space, its vdim and its number of values, which the reader checks before + * restoring, and then either the values (an external view) or `copy_of`. + * + * **Begin/end pairs and `copy_of`.** Material state is kept in pairs: beginning-of-step + * (`_beg`) and end-of-step (`_end`) functions, registered in m_model_update_qf_pairs. + * SystemDriver::UpdateModel() ends with SimulationState::SetupModelVariables(), which copies + * every `_beg` into its `_end` partner, and the checkpoint is written after UpdateModel(), so + * the two normally hold identical values. Instead of assuming that (post-processing runs in + * between and its code is not part of this contract), each pair is compared bit for bit on + * this rank: + * - identical: the `_end` entry only records `copy_of = <_beg name>`, and the reader restores + * it by copying the restored `_beg` (BindFieldsAndQuadratureFunctions()). For the state + * variables and stresses this nearly halves the size of the checkpoint; + * - different: the `_end` values are written like any other function. + * Either way the restart is bitwise exact. Since the comparison is per rank, one rank's file + * may use `copy_of` for a pair that another rank's file writes in full; each rank only reads + * its own file. + * + * Empty functions (a global function on a rank with no elements) are never compared, since + * their data pointers may be null, and are written as described zero-length views. + */ +void RestartCheckpoint::WriteQuadratureFunctions(Group* root, + const SimulationState& sim_state) const { + // End-of-step name -> beginning-of-step name, for the pairs that are bitwise identical. + std::map end_copy_of; + for (const auto& [beg_name, end_name] : sim_state.m_model_update_qf_pairs) { + const auto beg_it = sim_state.m_map_qfs.find(beg_name); + const auto end_it = sim_state.m_map_qfs.find(end_name); + if (beg_it == sim_state.m_map_qfs.end() || end_it == sim_state.m_map_qfs.end() || + !beg_it->second || !end_it->second) { + continue; + } + const mfem::expt::PartialQuadratureFunction& beg_function = *beg_it->second; + const mfem::expt::PartialQuadratureFunction& end_function = *end_it->second; + if (beg_function.Size() == 0 || beg_function.Size() != end_function.Size() || + beg_function.GetVDim() != end_function.GetVDim() || + beg_function.GetPartialSpaceShared() != end_function.GetPartialSpaceShared()) { + continue; + } + const std::size_t byte_count = static_cast(beg_function.Size()) * + sizeof(double); + if (std::memcmp(beg_function.HostRead(), end_function.HostRead(), byte_count) == 0) { + end_copy_of[end_name] = beg_name; + } + } + + const std::set recomputed = RecomputedQuadratureFunctionNames(sim_state); + Group* qfuncs = root->createGroup("qfuncs"); + for (const auto& [name, function] : sim_state.m_map_qfs) { + if (!function || recomputed.count(name) != 0) { + continue; + } + Group* entry = qfuncs->createGroup(SidreSafeName(name)); + entry->createViewString( + "qspace_name", + FindQuadratureSpaceName(sim_state.m_map_qs, function->GetPartialSpaceShared().get())); + entry->createViewScalar("vdim", function->GetVDim()); + entry->createViewScalar("value_count", static_cast(function->Size())); + const auto copy_it = end_copy_of.find(name); + if (copy_it != end_copy_of.end()) { + entry->createViewString("copy_of", copy_it->second); + } else { + CreateExternalDoubleView(entry, + "values", + function->Size() > 0 ? function->HostRead() : nullptr, + function->Size()); + } + } +} + +/** + * @brief Write the `materials/` group: the identity and properties of every region + * + * @param root Root group of the rank file being written + * @param sim_state Simulation state being written + * + * @details For each region: the material name, the model type and the effective number of + * state variables (including the orientation components SimulationState adds), and the + * material properties. ValidateMaterials() aborts the restart if the identity changed, since + * the stored state variables would then be meaningless, but only warns if the properties + * changed, since editing properties for a continuation is a legitimate use. + * + * Every rank holds the material list and properties for every region, whether or not it owns + * elements of that region, so every rank writes the same group. + */ +void RestartCheckpoint::WriteMaterials(Group* root, const SimulationState& sim_state) const { + Group* materials = root->createGroup("materials"); + const int num_regions = static_cast(sim_state.m_material_name_region.size()); + materials->createViewScalar("num_regions", num_regions); + for (int region = 0; region < num_regions; ++region) { + const auto region_index = static_cast(region); + Group* entry = materials->createGroup("region_" + std::to_string(region)); + entry->createViewString("name", sim_state.m_material_name_region[region_index].first); + entry->createViewScalar("mech_type", + static_cast(sim_state.m_region_material_type[region_index])); + entry->createViewScalar("num_state_vars", + sim_state.m_options.materials[region_index].state_vars.num_vars); + const std::vector& properties = sim_state.GetMaterialProperties(region); + CreateOwnedDoubleCopy( + entry, "properties", properties.data(), static_cast(properties.size())); + } +} + +/** + * @brief Write the commit file that marks a checkpoint as complete + * + * @param checkpoint_directory Directory of the checkpoint + * @param completed_cycles Number of completed cycles the checkpoint holds + * @param driver_step Driver loop counter of the checkpoint + * @param completed_time End time of the last completed step + * @return True if the commit file is in place + * + * @details Called on rank 0 only, after every rank saved its file. The file is written under + * a temporary name and then renamed. A rename within one file system is atomic on POSIX, so a + * reader either sees no commit file or a complete one, never a partial one. Besides marking + * the checkpoint complete, the commit file carries the format version and rank count that + * OpenForRestart() checks before any rank opens its own file. It uses the JSON protocol so it + * can be read by eye. + */ +bool RestartCheckpoint::WriteCommitFile(const fs::path& checkpoint_directory, + const std::size_t completed_cycles, + const std::size_t driver_step, + const double completed_time) const { + DataStore commit_store; + Group* commit = commit_store.getRoot(); + commit->createViewScalar("format_version", RESTART_FORMAT_VERSION); + commit->createViewScalar("num_ranks", m_num_ranks); + commit->createViewScalar("completed_cycles", static_cast(completed_cycles)); + commit->createViewScalar("driver_step", static_cast(driver_step)); + commit->createViewScalar("completed_time", completed_time); + commit->createViewString("rank_file_protocol", RANK_FILE_PROTOCOL); + + const fs::path temporary = checkpoint_directory / (std::string(COMMIT_FILE_NAME) + ".tmp"); + if (!commit->save(temporary.string(), COMMIT_PROTOCOL)) { + return false; + } + std::error_code error; + fs::rename(temporary, checkpoint_directory / COMMIT_FILE_NAME, error); + return !error; +} + +/** + * @brief Rename checkpoints newer than this run's starting point out of the way + * + * @param completed_cycles This run's starting point: the restart cycle, or 0 for a fresh run + * + * @details Called on rank 0 only, on this run's first write. Checkpoints in the write + * directory that are newer than the starting point belong to a different timeline: a fresh + * run reusing an old directory, or a restart from an older checkpoint. If they stayed, a + * later restart that picks "the newest committed checkpoint" could pick one of them instead + * of this run's. They are renamed to `.superseded` (or `.superseded.N`), never deleted, + * so nothing is lost if this was a mistake; the renamed directories no longer match + * ParseCycleDirectoryName() and are ignored. A rename failure is reported and otherwise + * ignored, since it only affects which checkpoint a later automatic selection would pick. + */ +void RestartCheckpoint::SupersedeCheckpointsAfter(const std::size_t completed_cycles) const { + for (const CheckpointDirectoryEntry& entry : ListCheckpointDirectories(m_options.directory)) { + if (entry.completed_cycles <= completed_cycles) { + continue; + } + fs::path target = entry.path; + target += ".superseded"; + for (int suffix = 1; fs::exists(target); ++suffix) { + target = entry.path; + target += ".superseded." + std::to_string(suffix); + } + std::error_code error; + fs::rename(entry.path, target, error); + if (error) { + MFEM_WARNING_0("Unable to rename the superseded checkpoint " << entry.path << ": " + << error.message()); + } else { + std::cout << "Renamed superseded restart checkpoint " << entry.path.string() << " to " + << target.string() << std::endl; + } + } +} + +/** + * @brief Remove checkpoints beyond the configured retention + * + * @details Called on rank 0 only, right after a successful commit, so the newest checkpoint + * is always complete before anything older is removed. Two kinds of directories are removed + * from the write directory: + * - the oldest committed checkpoints beyond the newest `keep_last` (keep_last = 0 keeps all); + * - incomplete (uncommitted) checkpoints older than the newest committed one, which are left + * over from failed or interrupted writes and can never be restarted from. + * An incomplete checkpoint newer than the newest committed one is left alone, since it may be + * a write still in progress by another run. A removal failure is reported and otherwise + * ignored; it only costs disk space. + */ +void RestartCheckpoint::ApplyRetentionPolicy() const { + const std::vector entries = ListCheckpointDirectories( + m_options.directory); + std::size_t newest_committed = 0; + bool any_committed = false; + std::vector committed; + for (const CheckpointDirectoryEntry& entry : entries) { + if (entry.committed) { + committed.push_back(&entry); + newest_committed = entry.completed_cycles; + any_committed = true; + } + } + + std::vector to_remove; + if (m_options.keep_last > 0 && + committed.size() > static_cast(m_options.keep_last)) { + // entries is sorted oldest first, so the first remove_count committed ones go. + const std::size_t remove_count = committed.size() - + static_cast(m_options.keep_last); + for (std::size_t i = 0; i < remove_count; ++i) { + to_remove.push_back(committed[i]->path); + } + } + for (const CheckpointDirectoryEntry& entry : entries) { + if (!entry.committed && any_committed && entry.completed_cycles < newest_committed) { + to_remove.push_back(entry.path); + } + } + + for (const fs::path& path : to_remove) { + std::error_code error; + fs::remove_all(path, error); + if (error) { + MFEM_WARNING_0("Unable to remove the old checkpoint " << path << ": " + << error.message()); + } + } +} + +// ========================================================================= +// RESTARTING, PHASE 1 (inside the SimulationState constructor) +// ========================================================================= + +/** + * @brief Pick the checkpoint directory to restart from and check its commit file + * + * @return Path of the selected checkpoint directory, the same on every rank + * + * @details Collective. Only rank 0 looks at the file system, so every rank is guaranteed to + * open the same checkpoint even if directories appear or disappear while the run starts. + * Rank 0 selects `[Restart] read_cycle` in the read directory if it is set, otherwise the + * newest committed checkpoint there, and reads its commit file. Any problem (requested + * checkpoint missing or incomplete, nothing committed, unreadable commit file) is broadcast as + * a message so every rank aborts with it. + * + * The format version and rank count from the commit file are then checked on every rank. + * Checking them from the commit file, before any rank opens its own file, gives a clear + * message for the common mistake of restarting with a different number of ranks, instead of + * some ranks finding no file at all. The checkpoint layout is partition dependent (each rank + * holds its own piece of the mesh and state), so the rank count has to match exactly. + */ +std::string RestartCheckpoint::SelectCheckpointDirectory() { + std::string selected_directory; + std::string selection_error; + int commit_format_version = -1; + int commit_num_ranks = -1; + if (m_rank == 0) { + const fs::path& read_directory = m_options.GetReadDirectory(); + fs::path selected; + if (m_options.read_cycle.has_value()) { + selected = read_directory / CycleDirectoryName(*m_options.read_cycle); + if (!fs::exists(selected / COMMIT_FILE_NAME)) { + selection_error = "the requested checkpoint " + selected.string() + + " does not exist or is incomplete (no " + COMMIT_FILE_NAME + ")"; + } + } else { + const auto entries = ListCheckpointDirectories(read_directory); + for (auto it = entries.rbegin(); it != entries.rend(); ++it) { + if (it->committed) { + selected = it->path; + break; + } + } + if (selected.empty()) { + selection_error = "no complete checkpoint was found in " + read_directory.string(); + } + } + if (selection_error.empty()) { + DataStore commit_store; + Group* commit = commit_store.getRoot(); + if (commit->load((selected / COMMIT_FILE_NAME).string(), COMMIT_PROTOCOL) && + commit->hasView("format_version") && commit->hasView("num_ranks")) { + commit_format_version = ReadSidreScalar(commit, "format_version"); + commit_num_ranks = ReadSidreScalar(commit, "num_ranks"); + selected_directory = selected.string(); + } else { + selection_error = "unable to read " + (selected / COMMIT_FILE_NAME).string(); + } + } + } + + BroadcastString(selection_error, m_comm); + MFEM_VERIFY_0(selection_error.empty(), "Unable to restart: " << selection_error); + BroadcastString(selected_directory, m_comm); + MPI_Bcast(&commit_format_version, 1, MPI_INT, 0, m_comm); + MPI_Bcast(&commit_num_ranks, 1, MPI_INT, 0, m_comm); + MFEM_VERIFY_0(commit_format_version == RESTART_FORMAT_VERSION, + "Unable to restart: the checkpoint " + << selected_directory << " has format version " << commit_format_version + << ", but this build reads version " << RESTART_FORMAT_VERSION); + MFEM_VERIFY_0(commit_num_ranks == m_num_ranks, + "Unable to restart: the checkpoint " + << selected_directory << " was written with " << commit_num_ranks + << " MPI ranks, but this run has " << m_num_ranks + << "; a restart must use the same number of ranks"); + return selected_directory; +} + +/** + * @details Implementation notes. + * 1. SelectCheckpointDirectory() picks the checkpoint on rank 0 and checks its commit file on + * every rank. + * 2. Every rank loads its own rank file into m_read_store, which stays alive until the end of + * RestoreState(). Group::load() materializes the owned arrays (the mesh, mesh_ref, grain + * IDs and metadata) and restores the external views (the bulk state) as descriptions only; + * their data is read later, straight into SimulationState's buffers. + * 3. Each rank checks that its file's restart_info matches: format version, rank count and, + * importantly, its own rank, which catches files renamed or copied between ranks. The + * stored time state and driver step are kept for phase 2. + * 4. m_last_written_cycles is set to the restart point, so the interval for this run's first + * checkpoint is counted from the restart, and its first write supersedes any checkpoints + * newer than the restart point (SupersedeCheckpointsAfter()). + * + * A problem with one rank's file is recorded and every rank aborts together at the end of the + * stage (AbortIfAnyRankFailed()). + */ +void RestartCheckpoint::OpenForRestart() { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::OpenForRestart"); + + const std::string selected_directory = SelectCheckpointDirectory(); + + m_read_rank_file = fs::path(selected_directory) / RankFileName(m_rank); + m_read_store = std::make_unique(); + Group* root = m_read_store->getRoot(); + if (!fs::exists(m_read_rank_file) || + !root->load(m_read_rank_file.string(), RANK_FILE_PROTOCOL)) { + RecordFailure("unable to load " + m_read_rank_file.string()); + } else if (!root->hasGroup("restart_info") || !root->hasGroup("mesh")) { + RecordFailure(m_read_rank_file.string() + " has no restart_info/ or mesh/ group"); + } else { + Group* info = root->getGroup("restart_info"); + const int stored_version = ReadSidreScalar(info, "format_version"); + const int stored_rank = ReadSidreScalar(info, "rank"); + const int stored_num_ranks = ReadSidreScalar(info, "num_ranks"); + if (stored_version != RESTART_FORMAT_VERSION || stored_rank != m_rank || + stored_num_ranks != m_num_ranks) { + RecordFailure(m_read_rank_file.string() + " was written by rank " + + std::to_string(stored_rank) + " of " + std::to_string(stored_num_ranks) + + " with format version " + std::to_string(stored_version) + + ", which does not match this rank"); + } else { + m_restart_driver_step = static_cast( + ReadSidreScalar(info, "driver_step")); + m_restart_time_state.completed_cycles = static_cast( + ReadSidreScalar(info, "completed_cycles")); + m_restart_time_state.completed_time = ReadSidreScalar(info, "completed_time"); + m_restart_time_state.next_dt = ReadSidreScalar(info, "next_dt"); + m_restart_time_state.previous_dt = ReadSidreScalar(info, "previous_dt"); + m_restart_time_state.original_dt = ReadSidreScalar(info, "original_dt"); + } + } + AbortIfAnyRankFailed("opening the checkpoint"); + + m_last_written_cycles = m_restart_time_state.completed_cycles; + + if (m_rank == 0) { + std::cout << "Restarting from checkpoint " << selected_directory << " (cycle " + << m_restart_time_state.completed_cycles << ", driver step " + << m_restart_driver_step << ", t = " << m_restart_time_state.completed_time << ")" + << std::endl; + } +} + +/** + * @details Implementation notes. ExaSidreParMesh rebuilds the mesh and checks the stored + * arrays itself (collectively). This function adds a final check that the edges, faces and + * communication groups MFEM regenerated from the connectivity match the counts recorded at + * write time. The mesh is returned as a plain mfem::ParMesh, so SimulationState owns it like + * any other mesh. + */ +std::shared_ptr RestartCheckpoint::ReconstructMesh() { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::ReconstructMesh"); + MFEM_VERIFY_0(m_read_store, "ReconstructMesh() requires OpenForRestart() first"); + Group* mesh_group = m_read_store->getRoot()->getGroup("mesh"); + + std::shared_ptr mesh = ExaSidreParMesh::Create(mesh_group, m_comm); + + const MeshTopologyCounts stored = ReadStoredMeshTopologyCounts(mesh_group); + const MeshTopologyCounts rebuilt = CountMeshTopology(*mesh); + if (!stored.Matches(rebuilt)) { + std::ostringstream message; + message << "the rebuilt mesh topology differs from the checkpoint (vertices, elements, " + "boundary elements, edges, faces, communication groups: stored " + << stored.num_vertices << ", " << stored.num_elements << ", " + << stored.num_boundary_elements << ", " << stored.num_edges << ", " + << stored.num_faces << ", " << stored.num_communication_groups << "; rebuilt " + << rebuilt.num_vertices << ", " << rebuilt.num_elements << ", " + << rebuilt.num_boundary_elements << ", " << rebuilt.num_edges << ", " + << rebuilt.num_faces << ", " << rebuilt.num_communication_groups << ")"; + RecordFailure(message.str()); + } + AbortIfAnyRankFailed("rebuilding the mesh"); + return mesh; +} + +/** + * @details Implementation notes. The grain IDs were written by WriteMeshAndGrainIds() as one + * int per local element, in local element order. The rebuilt mesh has the same local element + * order, so they are copied straight across. The count is checked against the local element + * count, which also covers ranks with no elements (both zero, nothing copied). + */ +void RestartCheckpoint::ReadElementGrainIds(mfem::Array& grain_ids) { + MFEM_VERIFY_0(m_read_store, "ReadElementGrainIds() requires OpenForRestart() first"); + Group* mesh_group = m_read_store->getRoot()->getGroup("mesh"); + const std::string path = "fields/grain_id/values"; + if (!mesh_group->hasView(path)) { + RecordFailure("the checkpoint has no mesh/" + path); + } else { + View* view = mesh_group->getView(path); + if (view->getNumElements() != grain_ids.Size()) { + RecordFailure("the checkpoint has " + std::to_string(view->getNumElements()) + + " grain IDs for " + std::to_string(grain_ids.Size()) + " local elements"); + } else if (grain_ids.Size() > 0) { + const int* stored = view->getData(); + std::copy(stored, stored + grain_ids.Size(), grain_ids.HostWrite()); + } + } + AbortIfAnyRankFailed("reading the grain IDs"); +} + +/** + * @details Implementation notes. + * + * **Why the coordinates are restored during construction.** "Build first, overlay last" + * would normally leave every field for phase 2. The coordinates are the exception, because + * some constructors compute and keep values derived from the starting geometry: + * - SystemDriver's constructor, with mono_def_bcs, selects essential DOFs from the mesh nodes + * using tight min/max coordinate comparisons (1e-12). + * - AbaqusUmatModel::InitLocSFGrads() stores reference shape-function gradients that every + * later step reuses. + * In a fresh run these see the reference coordinates projected from the original mesh file's + * Nodes. The rebuilt mesh has no Nodes, so the SimulationState constructor's + * Mesh::GetNodes(GridFunction&) projects from the vertex positions instead, which agrees with + * the original only to rounding (a few entries differ by about 1e-16). Small as that is, it + * changes those stored values and so the restarted run would no longer be bitwise identical. + * Overwriting the coordinate fields with the stored mesh_ref bits right after they are created + * gives every constructor exactly the geometry it saw in the original run. This is safe: in + * the original run mesh_ref, mesh_t_beg and mesh_current all held exactly these values until + * the first step (the constructor sets them equal and SwapNodes() makes mesh_current the + * mesh's Nodes; the driver only zeroes the velocity and displacement). + * + * **Steps.** + * 1. ValidateMeshSpace(): the coordinate space must have the same order and sizes as in the + * writing run, or the stored vector cannot be interpreted at all. + * 2. Check that mesh_ref was stored as an owned double array of the space's local size. It + * has to be owned: an external view would have no data until loadExternalData(), which + * phase 2 does. + * 3. Copy it into mesh_ref, mesh_t_beg and mesh_current, then tell the mesh its Nodes changed + * (mesh_current is the mesh's Nodes) so cached geometric factors are dropped. + * 4. VerifyRestoredDofLayout(): confirm, on the overlaid coordinates, that the rebuilt space + * numbers its DOFs exactly as the writing run did, before anything is built on it. + * 5. Drop the stored mesh_ref; it has been copied. + * + * On a rank with no elements the space and every vector are empty, nothing is copied, and the + * rank still takes part in the collective checks. + */ +void RestartCheckpoint::RestoreReferenceConfiguration(SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::RestoreReferenceConfiguration"); + MFEM_VERIFY_0(m_read_store, "RestoreReferenceConfiguration() requires OpenForRestart() first"); + Group* root = m_read_store->getRoot(); + + // Step 1: the coordinate space must match before the stored vector means anything. + ValidateMeshSpace(root, sim_state); + + // Step 2: locate the owned copy of the reference coordinates. + mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + const std::string path = "fields/mesh_ref/values"; + const double* stored_reference = nullptr; + if (!root->hasView(path)) { + RecordFailure("the checkpoint has no " + path); + } else { + View* view = root->getView(path); + if (view->isExternal() || view->getNumElements() != mesh_ref.Size() || + (mesh_ref.Size() > 0 && view->getTypeID() != axom::sidre::FLOAT64_ID)) { + RecordFailure(path + " is not an owned double array with one entry per local DOF " + "of the coordinate space"); + } else if (mesh_ref.Size() > 0) { + stored_reference = view->getData(); + } + } + AbortIfAnyRankFailed("restoring the reference configuration"); + + // Step 3: overwrite all three coordinate fields with the original reference bits. + for (const char* name : {"mesh_ref", "mesh_t_beg", "mesh_current"}) { + mfem::ParGridFunction& field = *sim_state.m_mesh_nodes.at(name); + if (field.Size() > 0) { + std::copy(stored_reference, stored_reference + field.Size(), field.HostWrite()); + } + } + sim_state.m_mesh->NodesUpdated(); + + // Step 4: verify the DOF layout before anything is built on these coordinates. + VerifyRestoredDofLayout(sim_state); + + // Step 5: the stored copy is no longer needed. + root->destroyGroupAndData("fields/mesh_ref"); +} + +/** + * @details Implementation notes. Called at the end of the SimulationState constructor, once + * the mesh has been rebuilt and the grain IDs copied out. The mesh arrays are often the + * largest owned data in the file, so freeing them early keeps the peak memory of the restart + * down; the rest of the store (the bulk state descriptions) is needed until phase 2. + */ +void RestartCheckpoint::ReleaseMeshArrays() { + if (m_read_store && m_read_store->getRoot()->hasGroup("mesh")) { + m_read_store->getRoot()->destroyGroupAndData("mesh"); + } +} + +/** + * @brief Check that the coordinate space matches the one the checkpoint was written with + * + * @param root Root group of the loaded rank file + * @param sim_state Simulation state under construction + * + * @details Compares the mesh order and the coordinate space's local, true and global true DOF + * counts with the stored values, and keeps the stored DOF-layout hash for + * VerifyRestoredDofLayout(). A different order (or, equivalently, different sizes) means the + * stored coordinate and velocity vectors cannot be mapped onto this space at all, so this is + * recorded as a failure; the caller aborts at the end of its stage. + */ +void RestartCheckpoint::ValidateMeshSpace(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("fields/mesh_space")) { + RecordFailure("the checkpoint has no fields/mesh_space group"); + return; + } + Group* space_info = root->getGroup("fields/mesh_space"); + const mfem::ParFiniteElementSpace& mesh_space = *sim_state.m_mesh_fes; + const int stored_order = ReadSidreScalar(space_info, "order"); + const auto stored_vsize = ReadSidreScalar(space_info, "vsize"); + const auto stored_true_vsize = ReadSidreScalar(space_info, "true_vsize"); + const auto stored_global_true_vsize = ReadSidreScalar(space_info, + "global_true_vsize"); + m_stored_mesh_ref_element_hash = ReadSidreScalar(space_info, + "mesh_ref_element_hash"); + if (stored_order != sim_state.m_options.mesh.order) { + RecordFailure("the mesh order changed (checkpoint " + std::to_string(stored_order) + + ", this run " + std::to_string(sim_state.m_options.mesh.order) + + "); the mesh order cannot change on a restart"); + } + if (stored_vsize != mesh_space.GetVSize() || stored_true_vsize != mesh_space.TrueVSize() || + stored_global_true_vsize != static_cast(mesh_space.GlobalTrueVSize())) { + RecordFailure( + "the coordinate space sizes differ (local, true, global true DOFs: checkpoint " + + std::to_string(stored_vsize) + ", " + std::to_string(stored_true_vsize) + ", " + + std::to_string(stored_global_true_vsize) + "; this run " + + std::to_string(mesh_space.GetVSize()) + ", " + std::to_string(mesh_space.TrueVSize()) + + ", " + std::to_string(static_cast(mesh_space.GlobalTrueVSize())) + ")"); + } +} + +/** + * @brief Verify that the rebuilt coordinate space numbers its DOFs as in the writing run + * + * @param sim_state Simulation state whose mesh_ref has just been overwritten + * + * @details Collective. Matching sizes are not enough: the stored vectors are only correct if + * every value lands on the same DOF it came from. Two independent checks, each run on the + * overlaid reference coordinates (a continuous field with distinct values at distinct nodes): + * + * (a) Local DOF numbering. ElementLocalFieldHash() of mesh_ref must equal the hash recorded + * at write time. It visits the values element by element in each element's DOF order, so + * any difference in how this rank numbers its local DOFs changes it. + * + * (b) Shared DOF matching across ranks. SharedDofInconsistency() restricts mesh_ref to true + * DOFs and prolongates it back. If every shared DOF is matched with the same true DOF on + * every rank, the field comes back unchanged up to rounding; a mismatch, such as a wrong + * shared-face orientation at H1 order >= 3, moves another node's coordinate onto the DOF, + * a jump of the order of the element size. The tolerance is 1e-10 times the largest + * coordinate magnitude in the mesh (agreed on with an MPI_Allreduce so every rank uses the + * same value), far above rounding and far below any real mismatch. A mesh whose + * coordinates are all zero (only possible on a degenerate mesh) skips the check. + * + * Every rank computes both checks, including ranks with no DOFs, since (b) and the scale + * reduction are collective. Failures abort every rank. + */ +void RestartCheckpoint::VerifyRestoredDofLayout(const SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::VerifyRestoredDofLayout"); + const mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + + // (a) Local DOF numbering. + if (ElementLocalFieldHash(mesh_ref) != m_stored_mesh_ref_element_hash) { + RecordFailure("the local DOF numbering of the rebuilt mesh differs from the checkpoint " + "(the element-local hash of mesh_ref does not match)"); + } + + // (b) Shared DOF matching across ranks. + double local_scale = 0.0; + if (mesh_ref.Size() > 0) { + const double* reference = mesh_ref.HostRead(); + for (int i = 0; i < mesh_ref.Size(); ++i) { + local_scale = std::max(local_scale, std::abs(reference[i])); + } + } + double global_scale = 0.0; + MPI_Allreduce(&local_scale, &global_scale, 1, MPI_DOUBLE, MPI_MAX, m_comm); + const double inconsistency = SharedDofInconsistency(mesh_ref); + const double tolerance = 1.0e-10 * global_scale; + if (global_scale > 0.0 && inconsistency > tolerance) { + std::ostringstream message; + message << "shared DOFs are matched inconsistently across ranks (max |x - P R x| = " + << inconsistency << ", tolerance " << tolerance << ")"; + RecordFailure(message.str()); + } + AbortIfAnyRankFailed("verifying the restored DOF layout"); +} + +// ========================================================================= +// RESTARTING, PHASE 2 (from the driver) +// ========================================================================= + +/** + * @details Implementation notes. By the time this runs, SimulationState, SystemDriver and + * PostProcessingDriver have all been constructed exactly as in a fresh run, so every buffer + * the checkpoint can fill exists. The steps are ordered so that nothing is overwritten until + * everything has been checked: + * + * 1. Validate (ValidateMaterials(), ValidateQuadratureSpaces()). A failure here aborts before + * any buffer is touched, so SimulationState is never left half restored. The coordinate + * space and DOF layout were already validated in phase 1. + * 2. Bind (BindFieldsAndQuadratureFunctions()). Every external view in the checkpoint that + * has a matching SimulationState buffer is pointed at that buffer's host memory. Functions + * stored as `copy_of` are queued in m_pending_copies instead. + * 3. Drop unbound external views. Group::loadExternalData() reads every external view + * described in the group whether or not it was bound, and an unbound view has nowhere to + * put its data. The unbound ones are zero-length arrays and functions that are only in the + * checkpoint (with a warning from step 2); they are destroyed first. + * 4. Load. A single loadExternalData() call per rank reads all bound arrays straight into + * SimulationState's buffers. HostReadWrite() during binding made the host copy the valid + * one, so on a GPU run the next device access copies the restored values over. The queued + * `copy_of` functions are then filled from their restored partners. + * 5. Resume time (TimeManagement::ResumeFromCheckpoint()) and recompute the fields derived + * from the restored state (ApplyPostRestartFixup()). + * 6. Release the checkpoint. From here on nothing restored is owned by Sidre. + * + * Collective, including on ranks with no elements: they bind nothing (every array of theirs + * is empty) but still take part in every agreement step. + */ +void RestartCheckpoint::RestoreState(SimulationState& sim_state) { + CALI_CXX_MARK_SCOPE("exaconstit::RestartCheckpoint::RestoreState"); + MFEM_VERIFY_0(m_read_store, "RestoreState() requires OpenForRestart() first"); + const double start_seconds = MPI_Wtime(); + Group* root = m_read_store->getRoot(); + + // Step 1: validate everything before anything is overwritten. + ValidateMaterials(root, sim_state); + ValidateQuadratureSpaces(root, sim_state); + AbortIfAnyRankFailed("checking the checkpoint against this run"); + + // Step 2: point each stored array at the SimulationState buffer it restores. + BindFieldsAndQuadratureFunctions(root, sim_state); + AbortIfAnyRankFailed("matching the checkpoint arrays to this run"); + + // Step 3: loadExternalData() would try to read unbound views too, so drop them. + std::vector> unbound; + CollectUnboundExternalViews(root, unbound); + for (auto& [group, view_name] : unbound) { + group->destroyView(view_name); + } + + // Step 4: one bulk read into the bound buffers, then fill the copy_of functions. + if (!root->loadExternalData(m_read_rank_file.string())) { + RecordFailure("unable to read the state arrays from " + m_read_rank_file.string()); + } + AbortIfAnyRankFailed("reading the checkpoint arrays"); + for (auto& [destination, source] : m_pending_copies) { + *destination = static_cast(*source); + } + m_pending_copies.clear(); + + // Step 5: time, then the fields derived from the restored state. + sim_state.m_time_manager.ResumeFromCheckpoint(m_restart_time_state); + ApplyPostRestartFixup(sim_state); + + // Step 6: nothing restored is owned by Sidre from here on. + m_read_store.reset(); + + const TimeManagement& time_manager = sim_state.m_time_manager; + if (m_rank == 0) { + std::cout << "Restored checkpoint state in " << (MPI_Wtime() - start_seconds) + << " s; next step: cycle " << (m_restart_time_state.completed_cycles + 1) + << ", driver step " << (m_restart_driver_step + 1) + << ", t_end = " << time_manager.GetTime() + << ", dt = " << time_manager.GetDeltaTime() + << ", t_final = " << time_manager.GetFinalTime() << std::endl; + } + if (time_manager.IsFinished()) { + MFEM_WARNING_0("The checkpoint is already at the configured final time, so there is " + "nothing left to run; increase t_final (or extend the custom time steps) " + "to continue the simulation"); + } +} + +/** + * @brief Check that the material regions match the ones the checkpoint was written with + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details The stored state variables are only meaningful for the same material model with + * the same state-variable layout, so a different number of regions, or a different name, + * model type or effective state-variable count for any region, is recorded as a failure. + * Changed material properties only produce a warning: editing properties for a continuation + * is legitimate, and this run's values are the ones used (the models read them from + * SimulationState when they are constructed). The warning is issued on every rank through + * MFEM_WARNING_0, which prints it once; every rank holds the same material data, so every + * rank reaches the same conclusion. + */ +void RestartCheckpoint::ValidateMaterials(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("materials")) { + RecordFailure("the checkpoint has no materials/ group"); + return; + } + Group* materials = root->getGroup("materials"); + const int stored_regions = ReadSidreScalar(materials, "num_regions"); + const int runtime_regions = static_cast(sim_state.m_material_name_region.size()); + if (stored_regions != runtime_regions) { + RecordFailure("the number of material regions changed (checkpoint " + + std::to_string(stored_regions) + ", this run " + + std::to_string(runtime_regions) + ")"); + return; + } + for (int region = 0; region < runtime_regions; ++region) { + const auto region_index = static_cast(region); + Group* entry = materials->getGroup("region_" + std::to_string(region)); + const std::string stored_name = entry->getView("name")->getString(); + const int stored_type = ReadSidreScalar(entry, "mech_type"); + const int stored_state_vars = ReadSidreScalar(entry, "num_state_vars"); + const std::string& runtime_name = sim_state.m_material_name_region[region_index].first; + const int runtime_type = static_cast(sim_state.m_region_material_type[region_index]); + const int runtime_state_vars = + sim_state.m_options.materials[region_index].state_vars.num_vars; + if (stored_name != runtime_name || stored_type != runtime_type || + stored_state_vars != runtime_state_vars) { + RecordFailure("region " + std::to_string(region + 1) + + " changed its material name, model type or number of state variables " + "(checkpoint '" + + stored_name + "', type " + std::to_string(stored_type) + ", " + + std::to_string(stored_state_vars) + " state variables; this run '" + + runtime_name + "', type " + std::to_string(runtime_type) + ", " + + std::to_string(runtime_state_vars) + " state variables)"); + continue; + } + + const std::vector& runtime_properties = sim_state.GetMaterialProperties(region); + View* stored_view = entry->getView("properties"); + const IndexType stored_count = stored_view->getNumElements(); + bool same = stored_count == static_cast(runtime_properties.size()); + if (same && stored_count > 0) { + const double* stored = stored_view->getData(); + same = std::equal(runtime_properties.begin(), runtime_properties.end(), stored); + } + if (!same) { + MFEM_WARNING_0("The material properties of region " + << (region + 1) << " ('" << runtime_name + << "') differ from the checkpoint; the restart continues with this " + "run's properties"); + } + } +} + +/** + * @brief Check that the rebuilt quadrature spaces match the ones the checkpoint was written + * with + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details Every quadrature space present both in this run and in the checkpoint must have the + * same integration order, number of values, number of local elements, full/partial status and + * local-to-global element map (by hash). Only then can a quadrature function's stored values + * be copied straight into the rebuilt function, point for point. A space present on only one + * side is not an error here: the functions on it are handled (and reported) during binding. + * + * On a rank with no elements only the (empty) global spaces exist and they match trivially; + * region spaces exist on neither side. + */ +void RestartCheckpoint::ValidateQuadratureSpaces(Group* root, const SimulationState& sim_state) { + if (!root->hasGroup("qspaces")) { + RecordFailure("the checkpoint has no qspaces/ group"); + return; + } + Group* qspaces = root->getGroup("qspaces"); + for (const auto& [name, space] : sim_state.m_map_qs) { + if (!space || !qspaces->hasGroup(SidreSafeName(name))) { + continue; + } + Group* entry = qspaces->getGroup(SidreSafeName(name)); + const bool same = ReadSidreScalar(entry, "order") == space->GetOrder() && + ReadSidreScalar(entry, "size") == space->GetSize() && + ReadSidreScalar(entry, "num_local_elements") == + space->GetNumLocalElements() && + ReadSidreScalar(entry, "is_full_space") == + (space->IsFullSpace() ? 1 : 0) && + ReadSidreScalar(entry, "local2global_hash") == + HashIntArray(space->GetLocal2Global()); + if (!same) { + RecordFailure("the quadrature space '" + name + + "' differs from the checkpoint (integration order, size, element " + "subset or element order)"); + } + } +} + +/** + * @brief Point every stored state array at the SimulationState buffer it restores + * + * @param root Root group of the loaded rank file + * @param sim_state Fully constructed simulation state + * + * @details Nothing is read yet; this only sets data pointers (View::setExternalDataPtr()) so + * that RestoreState()'s single loadExternalData() call reads every array straight into place. + * + * **Nodal fields.** `mesh_current` and `primal_field` must be external double arrays with + * exactly the size of the target vector. mesh_ref was already restored in phase 1, and the + * other nodal fields are recomputed (ApplyPostRestartFixup()). + * + * **Quadrature functions.** Every function registered in this run, except the recomputed ones + * (RecomputedQuadratureFunctionNames()), is matched with the checkpoint entry of the same + * name: + * - A different space name, vdim or value count is a failure; the values would not line up. + * - A `copy_of` entry is queued in m_pending_copies and filled after the load from its + * partner, which must exist in this run with the same size and have stored values in the + * checkpoint. + * - Otherwise the stored values view must be an external double array of the right size, and + * is pointed at the function's host data. + * - Empty functions (on ranks with no elements of their space) are skipped; their stored view + * is zero length and is dropped before the load. + * + * **Mismatched sets are warnings, not failures.** A function registered in this run but not in + * the checkpoint keeps the value its constructor gave it, which is expected when a + * continuation enables extra post-processing. A function only in the checkpoint is skipped, + * which is expected when a continuation disables some. Both are reported on rank 0 through + * MFEM_WARNING_0. Because region functions only exist where the region has elements, the + * lists reflect rank 0's view; in practice the sets are decided by the options and are the + * same everywhere a region exists. + * + * HostReadWrite() is used for the target pointers so that the host copy becomes the valid one + * and, on a GPU run, the next device access copies the restored values to the device. + */ +void RestartCheckpoint::BindFieldsAndQuadratureFunctions(Group* root, SimulationState& sim_state) { + // --------------------------------------------------------------------- + // Part 1: nodal fields + // + // Only two nodal vectors are restored here. mesh_ref was restored in phase 1 (and its + // stored copy already dropped), and every other nodal field is recomputed from these two + // afterwards (ApplyPostRestartFixup()): + // - mesh_current: all local DOFs of the coordinate space (VSize entries). This is also the + // mesh's Nodes, so restoring it moves the mesh to its checkpointed configuration. + // - primal_field: the velocity true DOFs (TrueVSize entries), the solver's unknown. + // + // Each stored view must be an external double array with exactly the target's size. It + // was written as an external view, so a stored owned view or a different type means the + // file does not have the expected layout. The sizes were already implied by the mesh-space + // check in phase 1, so a size mismatch here means the file is damaged. + // + // Binding only records the target pointer; the data is read later by RestoreState()'s + // loadExternalData() call. HostReadWrite() marks the host copy as the valid one, so on a + // GPU run the next device access copies the restored values to the device instead of + // using stale device data. On a rank with no elements both vectors are empty. Their views + // were written without a data pointer, which Sidre stores as an EMPTY view rather than an + // external one, so only the size is checked (0 == 0) and nothing is bound or loaded. + // --------------------------------------------------------------------- + const auto bind_vector = [&](const std::string& path, mfem::Vector& target) { + if (!root->hasView(path)) { + RecordFailure("the checkpoint has no " + path); + return; + } + View* view = root->getView(path); + if (target.Size() == 0) { + if (view->getNumElements() != 0) { + RecordFailure(path + " has " + std::to_string(view->getNumElements()) + + " entries, but this rank owns no elements"); + } + return; + } + if (!view->isExternal() || view->getTypeID() != axom::sidre::FLOAT64_ID || + view->getNumElements() != target.Size()) { + RecordFailure(path + " has " + std::to_string(view->getNumElements()) + + " entries or the wrong type, but this run needs " + + std::to_string(target.Size()) + " doubles"); + return; + } + view->setExternalDataPtr(target.HostReadWrite()); + }; + bind_vector("fields/mesh_current/values", *sim_state.m_mesh_nodes.at("mesh_current")); + bind_vector("fields/primal_field/values", *sim_state.m_primal_field); + + // --------------------------------------------------------------------- + // Part 2: quadrature functions + // + // The loop runs over this run's registry rather than over the checkpoint, because the + // question being answered is "where does each of this run's functions get its value + // from?". Functions that are only in the checkpoint are reported in part 3. + // + // For each registered function, in order: + // a. Skip it if it is null (an unused registry slot) or recomputed before it is next + // read (RecomputedQuadratureFunctionNames()); it keeps its constructor value until + // then, exactly as it would after a normal step. + // b. Look up the checkpoint entry of the same (encoded) name. If there is none, remember + // the name for the warning in part 3 and leave the function as constructed. + // c. Check that the stored function has the same space name, vdim and number of values. + // ValidateQuadratureSpaces() has already checked that a space of that name has the + // same points in the same order, so matching names plus matching shapes means value i + // of the stored array belongs at value i of this function. Anything else would put + // values on the wrong quadrature points, so it is a failure, not a warning. + // d. If the entry is a `copy_of`, its values were not stored because they were bitwise + // equal to its beginning-of-step partner (see WriteQuadratureFunctions()). Queue the + // copy for after the load. The partner must exist in this run with the same size, and + // must have stored values in the checkpoint; otherwise nothing could be copied. + // e. Otherwise bind the stored values to the function's host data, after checking that + // the view is an external double array of the right size. An empty function (a + // global function on a rank with no elements) is skipped before that check: there is + // nothing to read, and its zero-length stored view is dropped before the load. + // + // Every problem is recorded rather than aborting immediately, so all of them are reported + // together and every rank reaches the agreement in RestoreState(). + // --------------------------------------------------------------------- + if (!root->hasGroup("qfuncs")) { + RecordFailure("the checkpoint has no qfuncs/ group"); + return; + } + Group* qfuncs = root->getGroup("qfuncs"); + const std::set recomputed = RecomputedQuadratureFunctionNames(sim_state); + std::vector runtime_only; + for (const auto& [name, function] : sim_state.m_map_qfs) { + // (a) Unused or recomputed. + if (!function || recomputed.count(name) != 0) { + continue; + } + + // (b) Not in the checkpoint. + const std::string encoded = SidreSafeName(name); + if (!qfuncs->hasGroup(encoded)) { + runtime_only.push_back(name); + continue; + } + + // (c) Same space, vdim and number of values. + Group* entry = qfuncs->getGroup(encoded); + const std::string stored_space = entry->getView("qspace_name")->getString(); + const std::string runtime_space = FindQuadratureSpaceName( + sim_state.m_map_qs, function->GetPartialSpaceShared().get()); + const int stored_vdim = ReadSidreScalar(entry, "vdim"); + const auto stored_count = ReadSidreScalar(entry, "value_count"); + if (stored_space != runtime_space || stored_vdim != function->GetVDim() || + stored_count != function->Size()) { + RecordFailure("the quadrature function '" + name + + "' changed shape (checkpoint: space '" + stored_space + "', vdim " + + std::to_string(stored_vdim) + ", " + std::to_string(stored_count) + + " values; this run: space '" + runtime_space + "', vdim " + + std::to_string(function->GetVDim()) + ", " + + std::to_string(function->Size()) + " values)"); + continue; + } + + // (d) Stored as a copy of its partner: copy after the load. The source pointer is the + // partner's SimulationState function, which the load fills from its own stored values + // before m_pending_copies is processed. + if (entry->hasView("copy_of")) { + const std::string source_name = entry->getView("copy_of")->getString(); + const auto source_it = sim_state.m_map_qfs.find(source_name); + const bool source_restorable = + source_it != sim_state.m_map_qfs.end() && source_it->second && + source_it->second->Size() == function->Size() && + qfuncs->hasGroup(SidreSafeName(source_name)) && + qfuncs->getGroup(SidreSafeName(source_name))->hasView("values"); + if (!source_restorable) { + RecordFailure("the quadrature function '" + name + "' is stored as a copy of '" + + source_name + "', which cannot be restored in this run"); + continue; + } + m_pending_copies.emplace_back(function, source_it->second); + continue; + } + + // (e) Stored values: point the stored view at the function's host data. + if (function->Size() == 0) { + continue; + } + if (!entry->hasView("values")) { + RecordFailure("the quadrature function '" + name + "' has no stored values"); + continue; + } + View* values = entry->getView("values"); + if (!values->isExternal() || values->getTypeID() != axom::sidre::FLOAT64_ID || + values->getNumElements() != function->Size()) { + RecordFailure("the stored values of the quadrature function '" + name + + "' have the wrong type or size"); + continue; + } + values->setExternalDataPtr(function->HostReadWrite()); + } + + // --------------------------------------------------------------------- + // Part 3: report the functions present on only one side + // + // Neither case stops the restart (see the function documentation for why both are + // expected when a continuation changes its post-processing options). The checkpoint-only + // functions are left unbound here, so RestoreState() drops their views before the load. + // + // MFEM_WARNING_0 is used so each message appears once, from rank 0. It is not a + // collective call: each rank evaluates its own lists, and only rank 0's output is shown. + // Region functions only exist on ranks that own elements of the region, so if rank 0 owns + // none of a region, a mismatch in that region's functions is not reported. In practice + // which functions exist is decided by the options and is the same on every rank where a + // region exists. + // --------------------------------------------------------------------- + for (const std::string& name : runtime_only) { + MFEM_WARNING_0("The quadrature function '" + << name + << "' is not in the checkpoint; it keeps the value it was initialized " + "with"); + } + for (Group& entry : qfuncs->groups()) { + bool registered = false; + for (const auto& [name, function] : sim_state.m_map_qfs) { + if (function && SidreSafeName(name) == entry.getName()) { + registered = true; + break; + } + } + if (!registered) { + MFEM_WARNING_0("The checkpoint quadrature function '" + << entry.getName() << "' is not used by this run and is skipped"); + } + } +} + +/** + * @brief Recompute the nodal fields derived from the restored state + * + * @param sim_state Simulation state whose stored fields were just restored + * + * @details This repeats exactly what SimulationState::FinishCycle() did at the end of the step + * the checkpoint was written after, using the restored coordinates and velocity: + * - mesh_t_beg = mesh_current (beginning-of-step coordinates); + * - previous velocity = velocity (true DOFs); + * - velocity grid function = the velocity true DOFs distributed to local DOFs (collective); + * - displacement = mesh_current - mesh_ref. + * mesh_current is the mesh's Nodes (SimulationState swaps it in during construction), and its + * values just changed underneath the mesh, so the mesh is told to drop its cached geometric + * factors. If FinishCycle() ever derives another field, it must be added here as well. + */ +void RestartCheckpoint::ApplyPostRestartFixup(SimulationState& sim_state) const { + mfem::ParGridFunction& mesh_current = *sim_state.m_mesh_nodes.at("mesh_current"); + mfem::ParGridFunction& mesh_ref = *sim_state.m_mesh_nodes.at("mesh_ref"); + (*sim_state.m_mesh_nodes.at("mesh_t_beg")) = mesh_current; + (*sim_state.m_primal_field_prev) = *sim_state.m_primal_field; + sim_state.m_mesh_qoi_nodes.at("velocity")->Distribute(*sim_state.m_primal_field); + mfem::ParGridFunction& displacement = *sim_state.m_mesh_qoi_nodes.at("displacement"); + displacement = mesh_current; + displacement -= mesh_ref; + sim_state.m_mesh->NodesUpdated(); +} + +} // namespace exaconstit + +#endif // EXACONSTIT_USE_AXOM diff --git a/src/sim_state/restart_checkpoint.hpp b/src/sim_state/restart_checkpoint.hpp new file mode 100644 index 00000000..c476cfed --- /dev/null +++ b/src/sim_state/restart_checkpoint.hpp @@ -0,0 +1,306 @@ +#pragma once + +#include "mfem_expt/partial_qfunc.hpp" +#include "options/option_parser_v2.hpp" +#include "sim_state/simulation_state.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace axom { +namespace sidre { +class DataStore; +class Group; +} // namespace sidre +} // namespace axom + +namespace exaconstit { + +/** + * @brief Version of the checkpoint rank-file layout written by RestartCheckpoint + * + * Stored in `restart_info/format_version` of every rank file and in `commit.json`. Bump it + * whenever the layout changes, so older checkpoints are rejected with a clear message instead + * of being misread. The mesh subtree has its own version (MESH_CHECKPOINT_FORMAT_VERSION). + */ +inline constexpr int RESTART_FORMAT_VERSION = 1; + +/** + * @brief Binary checkpoint writing and restarting for a SimulationState + * + * @details Writes the evolving simulation state to disk with Axom Sidre (one HDF5 file per + * rank) and restores it in a later run. A restart works from the checkpoint alone: the mesh, + * grain, region-mapping, orientation and state-variable files of the original run are never + * opened again. See doc/restart.md for the user-facing description and the design record. + * + * **Checkpoint layout on disk:** + * @code + * /cycle_/ + * rank_.hdf5 written by every rank + * commit.json written last, by rank 0 + * @endcode + * A checkpoint without `commit.json` is incomplete ("torn") and is never read. + * + * **Restarting is split into two phases**, because the state a restart has to restore is + * created in two places: + * - Phase 1 runs inside the SimulationState constructor, in place of reading the original + * input files: OpenForRestart(), ReconstructMesh(), RestoreReferenceConfiguration(), + * ReadElementGrainIds() and ReleaseMeshArrays(), in that order. + * - Phase 2, RestoreState(), runs from the driver after SystemDriver and + * PostProcessingDriver have been constructed, so every quadrature function any component + * registers already exists. It overwrites the evolving state with the checkpointed values, + * replacing whatever the constructors initialized. + * + * **Ownership.** SimulationState owns the RestartCheckpoint (std::unique_ptr) and makes it a + * friend so it can reach the field and quadrature-function registries. RestartCheckpoint + * never stores a pointer back to its owner; the SimulationState& arguments are borrowed for + * the duration of each call. Everything restored ends up owned by SimulationState, and + * nothing is owned by Sidre once RestoreState() returns. + * + * **Collective calls.** Every public member function except GetRestartDriverStep() is + * collective over the communicator given to the constructor and must be called on every + * rank, including ranks that own no elements. A problem found on any rank aborts every rank. + * + * **What is checkpointed.** + * - Quadrature data: every function registered in SimulationState's quadrature-function + * registry (SimulationState::AddQuadratureFunction()), whatever component registered it, + * except `tangent_stiffness`. Quadrature data kept outside the registry is not saved. + * - Nodal data: a fixed list, not a registry: the reference and current coordinates and the + * velocity true DOFs. The other SimulationState grid functions are derived from these. + * Any new grid function or solution vector that carries history, e.g. an additional + * primal variable, must be added to WriteNodalFields() and to the binding in + * BindFieldsAndQuadratureFunctions(), or it is silently not restored. + * - Mesh: the volume ParMesh only. Anything built on top of it (submeshes, surface meshes, + * other topologies) is not saved and must either be rebuilt deterministically from the + * restored mesh or have its own checkpoint data. + * - Time state, the element grain IDs and the material properties (the properties are only + * compared against the restarted run's; that run uses its own). + * + * **Restart requirements.** The restarted run must use the same number of MPI ranks, the + * same mesh order, and the same material list (names, model types and state-variable + * counts). These are checked and violations abort with a message saying what changed. + */ +class RestartCheckpoint { +public: + /** + * @brief Construct the checkpoint handler + * + * @param options [Restart] options (copied) + * @param comm Communicator of the simulation (SimulationState uses MPI_COMM_WORLD) + */ + RestartCheckpoint(const RestartOptions& options, MPI_Comm comm); + ~RestartCheckpoint(); + + RestartCheckpoint(const RestartCheckpoint&) = delete; + RestartCheckpoint& operator=(const RestartCheckpoint&) = delete; + + // ========================================================================= + // WRITING + // ========================================================================= + + /** + * @brief Write a checkpoint if one is due + * + * @param sim_state Simulation state to write + * @param driver_step The driver's loop counter `ti` for the step just completed + * + * @details Call once at the end of every driver-loop iteration, after FinishCycle(), + * UpdateModel() and post-processing. A checkpoint is due when `[Restart] write` is set, + * the time manager is not in the middle of a retrial or sub-step sequence, at least one + * cycle has completed since the last checkpoint (or the restart point), and either + * `interval` cycles have passed or the simulation just finished and `write_final` is set. + * + * A failed write is reported as a warning and the simulation continues; the incomplete + * checkpoint is never read. + */ + void WriteCheckpointIfDue(SimulationState& sim_state, std::size_t driver_step); + + // ========================================================================= + // RESTARTING, PHASE 1 (inside the SimulationState constructor) + // ========================================================================= + + /** + * @brief Select the checkpoint to restart from and load this rank's file + * + * @details Uses `[Restart] read_cycle` if set, otherwise the newest complete checkpoint + * in the read directory. Aborts if no usable checkpoint exists, or if it was written with + * a different format version or number of ranks. Must be called before any other phase 1 + * or phase 2 function. + */ + void OpenForRestart(); + + /** + * @brief Rebuild the parallel mesh from the loaded checkpoint + * + * @return The rebuilt mesh, identical to the one that was written (see ExaSidreParMesh) + * + * @details Replaces reading the mesh file. Aborts if the rebuilt topology does not match + * the counts recorded in the checkpoint. + */ + std::shared_ptr ReconstructMesh(); + + /** + * @brief Fill in the element grain IDs from the checkpoint + * + * @param grain_ids Array sized to the local element count; receives one grain ID per + * local element + * + * @details Replaces reading the grain IDs from the mesh file's element attributes, which + * hold region IDs in a checkpointed mesh. + */ + void ReadElementGrainIds(mfem::Array& grain_ids); + + /** + * @brief Overwrite the coordinate fields with the checkpointed reference configuration + * + * @param sim_state Simulation state under construction + * + * @details Call from the SimulationState constructor right after the coordinate fields + * exist and before anything else reads them. Sets `mesh_ref`, `mesh_t_beg` and + * `mesh_current` (the mesh's Nodes) to the checkpointed reference coordinates, which is + * exactly what they held in the original run until its first step, and checks that the + * mesh's finite element space numbers its DOFs as in the original run. Aborts on a + * mismatch. + */ + void RestoreReferenceConfiguration(SimulationState& sim_state); + + /** + * @brief Free the mesh arrays held by the loaded checkpoint + * + * @details Call at the end of the SimulationState constructor; the mesh and grain IDs + * have been copied out by then. The rest of the checkpoint stays loaded for phase 2. + */ + void ReleaseMeshArrays(); + + // ========================================================================= + // RESTARTING, PHASE 2 (from the driver) + // ========================================================================= + + /** + * @brief Overwrite the evolving state with the checkpointed values + * + * @param sim_state Fully constructed simulation state + * + * @details Call once, after SystemDriver and PostProcessingDriver are constructed and + * before the first time step. Checks the checkpoint against this run (materials and + * quadrature spaces), restores the nodal fields and every quadrature function present in + * both, resumes the time state, recomputes the fields derived from the restored state, + * and releases the checkpoint. Aborts on a mismatch before anything is overwritten. + */ + void RestoreState(SimulationState& sim_state); + + /** + * @brief Driver loop counter recorded in the checkpoint + * + * @return The `ti` of the last completed step; the first restarted step is this + 1 + */ + std::size_t GetRestartDriverStep() const { + return m_restart_driver_step; + } + +private: + // --------------------------------------------------------------------- + // Writing (each documented in restart_checkpoint.cpp) + // --------------------------------------------------------------------- + void WriteCheckpoint(SimulationState& sim_state, + std::size_t driver_step, + const TimeManagement::CheckpointState& time_state); + bool PrepareCheckpointDirectory(const std::filesystem::path& checkpoint_directory); + void WriteRestartInfo(axom::sidre::Group* root, + const SimulationState& sim_state, + std::size_t driver_step, + const TimeManagement::CheckpointState& time_state) const; + void WriteMeshAndGrainIds(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteNodalFields(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteQuadratureSpaces(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteQuadratureFunctions(axom::sidre::Group* root, const SimulationState& sim_state) const; + void WriteMaterials(axom::sidre::Group* root, const SimulationState& sim_state) const; + bool WriteCommitFile(const std::filesystem::path& checkpoint_directory, + std::size_t completed_cycles, + std::size_t driver_step, + double completed_time) const; + void SupersedeCheckpointsAfter(std::size_t completed_cycles) const; + void ApplyRetentionPolicy() const; + + // --------------------------------------------------------------------- + // Restarting (each documented in restart_checkpoint.cpp) + // --------------------------------------------------------------------- + std::string SelectCheckpointDirectory(); + void ValidateMeshSpace(axom::sidre::Group* root, const SimulationState& sim_state); + void VerifyRestoredDofLayout(const SimulationState& sim_state); + void ValidateMaterials(axom::sidre::Group* root, const SimulationState& sim_state); + void ValidateQuadratureSpaces(axom::sidre::Group* root, const SimulationState& sim_state); + void BindFieldsAndQuadratureFunctions(axom::sidre::Group* root, SimulationState& sim_state); + void ApplyPostRestartFixup(SimulationState& sim_state) const; + + // --------------------------------------------------------------------- + // Failure reporting + // --------------------------------------------------------------------- + /** + * @brief Record a problem found on this rank + * + * @param message Description of the problem + * + * @details Problems are collected rather than aborting immediately, so that every rank + * reaches AbortIfAnyRankFailed() and the abort happens on every rank together. + */ + void RecordFailure(const std::string& message); + + /** + * @brief Abort on every rank if any rank recorded a failure + * + * @param stage Short description of what was being done, used in the abort message + * + * @details Collective. Each failing rank reports its own recorded problems as warnings, + * then every rank aborts with one shared message. + */ + void AbortIfAnyRankFailed(const char* stage); + + /** @brief [Restart] options this handler was created with */ + RestartOptions m_options; + /** @brief Communicator of the simulation */ + MPI_Comm m_comm; + /** @brief This rank in m_comm */ + int m_rank = 0; + /** @brief Number of ranks in m_comm */ + int m_num_ranks = 1; + + /** @brief Completed cycles at the last committed checkpoint, or at the restart point */ + std::size_t m_last_written_cycles = 0; + /** @brief Whether the next write still has to supersede checkpoints of another timeline */ + bool m_supersede_pending = true; + + /** @brief Checkpoint loaded by OpenForRestart(); released at the end of RestoreState() */ + std::unique_ptr m_read_store; + /** @brief Path of this rank's file in the checkpoint being restarted from */ + std::filesystem::path m_read_rank_file; + /** @brief Driver loop counter recorded in the checkpoint being restarted from */ + std::size_t m_restart_driver_step = 0; + /** @brief Time state recorded in the checkpoint being restarted from */ + TimeManagement::CheckpointState m_restart_time_state; + /** @brief Element-local hash of mesh_ref recorded in the checkpoint */ + std::uint64_t m_stored_mesh_ref_element_hash = 0; + + /** + * @brief End-of-step quadrature functions to restore by copying their beginning-of-step + * partner once the bulk load is done (destination, source) + */ + std::vector, + std::shared_ptr>> + m_pending_copies; + + /** @brief Whether this rank recorded a failure since the last AbortIfAnyRankFailed() */ + bool m_local_failed = false; + /** @brief Problems recorded on this rank since the last AbortIfAnyRankFailed() */ + std::string m_local_failure_log; +}; + +} // namespace exaconstit diff --git a/src/sim_state/restart_sidre_utils.hpp b/src/sim_state/restart_sidre_utils.hpp new file mode 100644 index 00000000..7fe77b9a --- /dev/null +++ b/src/sim_state/restart_sidre_utils.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include "utilities/unified_logger.hpp" + +#include "mfem.hpp" + +#include +#include +#include + +#include "axom/sidre.hpp" + +/** + * @brief Small helpers shared by the checkpoint/restart implementation files + * + * @details Only included from exa_sidre_parmesh.cpp and restart_checkpoint.cpp, which are + * compiled when ExaConstit is built with Axom (EXACONSTIT_USE_AXOM). Not part of the public + * restart interface. + */ +namespace exaconstit::restart_detail { + +/** + * @brief Read a scalar view, converting from whatever numeric type it was stored as + * + * @tparam ScalarType Type to return (integer, unsigned integer or floating point) + * @param group Group that holds the view + * @param path Path of the scalar view relative to the group + * @return The stored value converted to ScalarType + * + * @details View::getScalar() does not convert between integer widths, and the JSON protocol + * does not preserve them: Conduit's JSON reader brings every integer back as int64, so an int + * written to `commit.json` as 1 reads back as 0 through getScalar() into an int. Going through + * conduit::Node::to_int64(), to_uint64() and to_float64() makes every read independent of the + * protocol and of the exact type the writer used. + */ +template +ScalarType ReadSidreScalar(axom::sidre::Group* group, const std::string& path) { + const auto& node = group->getView(path)->getNode(); + if constexpr (std::is_floating_point::value) { + return static_cast(node.to_float64()); + } else if constexpr (std::is_unsigned::value) { + return static_cast(node.to_uint64()); + } else { + return static_cast(node.to_int64()); + } +} + +/** + * @brief Check whether a condition holds on every rank + * + * @param local_value This rank's result + * @param comm Communicator to reduce over + * @return True on every rank if local_value is true on every rank, false on every rank + * otherwise + * + * @details Collective over comm. Used to turn rank-local checks into a decision every rank + * agrees on, so that either all ranks continue or all ranks abort together. Aborting on only + * the ranks that failed would leave the others waiting forever in their next collective call. + */ +inline bool AllRanksTrue(const bool local_value, MPI_Comm comm) { + int local_flag = local_value ? 1 : 0; + int global_flag = 0; + MPI_Allreduce(&local_flag, &global_flag, 1, MPI_INT, MPI_MIN, comm); + return global_flag == 1; +} + +/** + * @brief Abort on every rank if a rank-local check failed on any rank + * + * @param local_ok This rank's result + * @param comm Communicator to reduce over + * @param local_message Description of the problem, reported by each rank where it occurred + * @param summary Message rank 0 aborts with, shared by every rank + * + * @details Collective over comm. Each failing rank prints its own warning, since only that + * rank knows the details, and then every rank aborts with the same summary through + * MFEM_ABORT_0, so the summary appears once. + */ +inline void VerifyOnAllRanks(const bool local_ok, + MPI_Comm comm, + const std::string& local_message, + const std::string& summary) { + if (AllRanksTrue(local_ok, comm)) { + return; + } + if (!local_ok) { + int rank = 0; + MPI_Comm_rank(comm, &rank); + MFEM_WARNING("Rank " << rank << ": " << local_message); + } + MFEM_ABORT_0(summary << " (see the per-rank warnings above)"); +} + +} // namespace exaconstit::restart_detail diff --git a/src/sim_state/simulation_state.cpp b/src/sim_state/simulation_state.cpp index abf8813e..edcf9dae 100644 --- a/src/sim_state/simulation_state.cpp +++ b/src/sim_state/simulation_state.cpp @@ -1,6 +1,9 @@ #include "sim_state/simulation_state.hpp" #include "utilities/unified_logger.hpp" +#if defined(EXACONSTIT_USE_AXOM) +#include "sim_state/restart_checkpoint.hpp" +#endif namespace { @@ -410,11 +413,112 @@ bool TimeManagement::BCTime(const double desired_bc_time) { return false; } +/** + * @details Implementation notes. After a successful step, UpdateDeltaTime() leaves the time + * manager describing the *next* step: `old_time` is the end of the completed step, `dt` and + * `time` are the next step's size and end time, `simulation_cycle` counts completed steps and + * the tracker says whether the next step is a NORMAL or the FINAL one. This function rebuilds + * exactly that state from the checkpoint, using this run's time-step settings so a + * continuation can change them: + * + * 1. Completed time and cycle count come from the checkpoint. The retrial and sub-step + * counters are reset: checkpoints are only written when IsCheckpointSafe(), i.e. when no + * retrial or sub-step sequence is in flight, so they were zero (or irrelevant) then too. + * 2. prev_dt and dt_orig come from the checkpoint. Only log messages use them, and taking them + * from the checkpoint makes the restarted run print the same messages as the uninterrupted + * run. + * - prev_dt is the size of the last completed step. The driver prints it as "dt old was" at + * the start of the next cycle. If this run's settings change dt, the message shows the + * last step taken before the restart followed by the new dt. + * - dt_orig is reset at the first failure of a retrial sequence, before it is ever printed. + * It is restored only so both runs' time managers hold the same values. + * 3. The next dt is chosen the way the success branch of UpdateDeltaTime() would choose it: + * CUSTOM takes custom_dt[completed cycles] (or the stored dt if this run's list is shorter), + * FIXED takes dt_fixed, and AUTO takes the stored next dt, which the writing run had + * already computed from its last Newton iteration count, clamped to this run's + * [dt_min, dt_max]. With unchanged settings this reproduces the writing run's dt exactly. + * 4. The final-time logic of UpdateDeltaTime() is applied against this run's time_final (same + * 1e-3 * dt tolerance): the next step becomes FINAL if it reaches t_final, and its dt is + * shortened if it would overshoot. One case UpdateDeltaTime() never meets is added: a + * checkpoint written at t_final (by write_final) resumes as FINISHED, so the driver loop + * does not run at all unless t_final has been increased. + */ +void TimeManagement::ResumeFromCheckpoint(const CheckpointState& state) { + // Step 1: completed time and cycles; no retrial or sub-step sequence is in flight. + old_time = state.completed_time; + simulation_cycle = state.completed_cycles; + num_failures = 0; + num_sub_steps = 0; + required_num_sub_steps = 0; + + // Step 2: the time steps used only in log messages. + prev_dt = state.previous_dt; + dt_orig = state.original_dt; + + // Step 3: next dt from this run's scheme. + if (time_type == TimeStepType::CUSTOM) { + dt = (simulation_cycle < custom_dt.size()) ? custom_dt[simulation_cycle] : state.next_dt; + } else if (time_type == TimeStepType::FIXED) { + dt = dt_fixed; + } else { + dt = std::clamp(state.next_dt, dt_min, dt_max); + } + + // Step 4: final-time logic against this run's time_final. + const double tolerance = std::abs(1e-3 * dt); + if (time_final - old_time <= tolerance) { + internal_tracker = TimeStep::FINISHED; + time = old_time; + return; + } + const double tnew = old_time + dt; + if (std::abs(tnew - time_final) <= tolerance) { + internal_tracker = TimeStep::FINAL; + time = tnew; + } else if (tnew > time_final) { + internal_tracker = TimeStep::FINAL; + dt = time_final - old_time; + time = time_final; + } else { + internal_tracker = TimeStep::NORMAL; + time = tnew; + } +} + SimulationState::SimulationState(ExaOptions& options) : m_time_manager(options), m_options(options), class_device(options.solvers.rtmodel) { MPI_Comm_rank(MPI_COMM_WORLD, &my_id); m_time_manager = TimeManagement(options); + + // Restart runs ([Restart] read = true) build this object exactly as a fresh run does, + // except that the few things a fresh run reads from input files are taken from the + // checkpoint instead, since those files may no longer exist ("phase 1" of a restart; see + // the background notes in restart_checkpoint.cpp). Each restart-specific step below is + // marked. The evolving state itself is restored later, from the driver, by + // RestoreFromCheckpoint() ("phase 2"), once every component has registered its + // quadrature functions. + // + // Restart step: open the checkpoint and rebuild the mesh from it instead of reading the + // mesh file. Both are collective, so they run on every rank, including ranks that end up + // owning no elements. m_restart is also created for runs that only write checkpoints. + const bool restart_run = options.restart.read; +#if defined(EXACONSTIT_USE_AXOM) + if (options.restart.read || options.restart.write) { + m_restart = std::make_unique(options.restart, + MPI_COMM_WORLD); + } + if (restart_run) { + m_restart->OpenForRestart(); + m_mesh = m_restart->ReconstructMesh(); + } else { + m_mesh = ::makeMesh(options, my_id); + } +#else + // RestartOptions::validate() already rejects this; kept as a safeguard for callers that + // construct SimulationState without validating their options. + MFEM_VERIFY_0(!restart_run, "Restarting from a checkpoint requires ExaConstit built with Axom"); m_mesh = ::makeMesh(options, my_id); +#endif ::setupBoundaryConditions(options); // m_bc_manager = BCManager::GetInstance(); @@ -467,6 +571,20 @@ SimulationState::SimulationState(ExaOptions& options) m_primal_field_prev->UseDevice(true); (*m_primal_field) = 0.0; (*m_primal_field_prev) = 0.0; + +#if defined(EXACONSTIT_USE_AXOM) + // Restart step: overwrite the coordinate fields just created with the checkpointed + // reference coordinates, and verify the DOF layout, before anything reads them. The + // rebuilt mesh has no Nodes, so GetNodes() above projected from the vertex positions, + // which differs from the original run's coordinates in the last bits. Components built + // before RestoreFromCheckpoint() keep values derived from the starting geometry + // (SystemDriver's mono_def_bcs DOF search, the UMAT reference shape gradients), so they + // must see exactly the original coordinates for the restart to be bitwise exact. See + // RestartCheckpoint::RestoreReferenceConfiguration(). Collective. + if (restart_run) { + m_restart->RestoreReferenceConfiguration(*this); + } +#endif } { @@ -519,23 +637,67 @@ SimulationState::SimulationState(ExaOptions& options) region_map = false; mfem::Array grains(loc_nelems); + // Grain IDs. A fresh run reads them from the mesh's element attributes. In a + // checkpointed mesh the element attributes already hold region IDs (they are + // overwritten below on a fresh run), so the restart step reads the grain IDs the + // checkpoint stored separately. Collective on restart runs. + if (restart_run) { +#if defined(EXACONSTIT_USE_AXOM) + m_restart->ReadElementGrainIds(grains); +#endif + } else { + for (int i = 0; i < loc_nelems; i++) { + grains.operator[](i) = m_mesh->GetAttribute(i); + } + } + { auto pfes = GetParFiniteElementSpace(1); m_grains = std::make_shared(pfes.get()); m_grains->HostWrite(); for (int i = 0; i < loc_nelems; i++) { - grains.operator[](i) = m_mesh->GetAttribute(i); - m_grains->operator[](i) = m_mesh->GetAttribute(i); + m_grains->operator[](i) = grains.operator[](i); } } - const auto grains2region = ::create_grains_to_map(options, grains); + std::map grains2region; + if (!restart_run) { + grains2region = ::create_grains_to_map(options, grains); - for (int i = 0; i < loc_nelems; i++) { - const int grain_id = grains.operator[](i); - const int region_id = grains2region.at(grain_id); - m_mesh->SetAttribute(i, region_id); - region_map(region_id - 1, i) = true; + for (int i = 0; i < loc_nelems; i++) { + const int grain_id = grains.operator[](i); + const int region_id = grains2region.at(grain_id); + m_mesh->SetAttribute(i, region_id); + region_map(region_id - 1, i) = true; + } + } else { + // Restart step: a fresh run maps grains to regions with the region-mapping file and + // then stores the region IDs in the element attributes. The checkpointed mesh + // already carries those 1-based region IDs, so the region map is built from them + // directly and the region-mapping file is never read. + // + // A region ID outside [1, number of materials] means the material list shrank + // since the checkpoint was written. That check depends on each rank's own + // elements, so the ranks agree on it with an MPI_Allreduce and all abort together; + // a rank with no elements contributes "no problem". + const int num_regions = static_cast(options.materials.size()); + int local_out_of_range = 0; + for (int i = 0; i < loc_nelems; i++) { + const int region_id = m_mesh->GetAttribute(i); + if (region_id < 1 || region_id > num_regions) { + local_out_of_range = 1; + continue; + } + region_map(region_id - 1, i) = true; + } + int any_out_of_range = 0; + MPI_Allreduce( + &local_out_of_range, &any_out_of_range, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + MFEM_VERIFY_0(any_out_of_range == 0, + "The checkpoint has elements in regions beyond the " + << num_regions + << " material(s) configured for this run; the material list " + "cannot change on a restart"); } // update all of our attributes @@ -633,8 +795,20 @@ SimulationState::SimulationState(ExaOptions& options) } CreateRegionCommunicators(); - InitializeStateVariables(grains2region); + // Restart step: skip the state-variable and orientation initialization. Its input + // files may no longer exist, and RestoreFromCheckpoint() overwrites all state + // variables (orientations included) with the checkpointed values anyway. + if (!restart_run) { + InitializeStateVariables(grains2region); + } } +#if defined(EXACONSTIT_USE_AXOM) + // Restart step: the mesh and grain IDs have been copied out of the checkpoint, so free + // those arrays now; the rest of the checkpoint stays loaded for RestoreFromCheckpoint(). + if (restart_run) { + m_restart->ReleaseMeshArrays(); + } +#endif } SimulationState::~SimulationState() { @@ -645,18 +819,52 @@ SimulationState::~SimulationState() { } } +void SimulationState::RestoreFromCheckpoint() { +#if defined(EXACONSTIT_USE_AXOM) + MFEM_VERIFY_0(m_restart && m_options.restart.read, + "RestoreFromCheckpoint() was called on a run that is not restarting"); + m_restart->RestoreState(*this); +#else + MFEM_ABORT_0("Restarting from a checkpoint requires ExaConstit built with Axom"); +#endif +} + +void SimulationState::WriteCheckpointIfDue(const size_t driver_step) { +#if defined(EXACONSTIT_USE_AXOM) + if (m_restart) { + m_restart->WriteCheckpointIfDue(*this, driver_step); + } +#else + static_cast(driver_step); +#endif +} + +size_t SimulationState::GetRestartDriverStep() const { +#if defined(EXACONSTIT_USE_AXOM) + return m_restart ? m_restart->GetRestartDriverStep() : 0; +#else + return 0; +#endif +} + bool SimulationState::AddQuadratureFunction(const std::string_view& qf_name, const int vdim, const int region) { std::string qf_name_mat = GetQuadratureFunctionMapName(qf_name, region); - if (m_map_qfs.find(qf_name_mat) == m_map_qfs.end()) { - std::string qspace_name = GetRegionName(region); - m_map_qfs.emplace(qf_name_mat, - std::make_shared( - m_map_qs[qspace_name], vdim, 0.0)); - return true; + if (m_map_qfs.find(qf_name_mat) != m_map_qfs.end()) { + return false; } - return false; + // find(), not operator[]: a region with no elements on this rank has no quadrature space + // here, and operator[] would insert a null entry into m_map_qs, which every loop over the + // spaces (including the checkpoint writer) would then dereference. + const auto space = m_map_qs.find(GetRegionName(region)); + if (space == m_map_qs.end() || !space->second) { + return false; + } + m_map_qfs.emplace( + qf_name_mat, + std::make_shared(space->second, vdim, 0.0)); + return true; } std::pair diff --git a/src/sim_state/simulation_state.hpp b/src/sim_state/simulation_state.hpp index 18ba4102..dbbcf066 100644 --- a/src/sim_state/simulation_state.hpp +++ b/src/sim_state/simulation_state.hpp @@ -14,6 +14,10 @@ #include #include +namespace exaconstit { +class RestartCheckpoint; +} // namespace exaconstit + /** * @brief Enumeration for time step status and control * @@ -253,20 +257,76 @@ class TimeManagement { } /** - * @brief Restart simulation from a specific time and cycle - * - * @param time_restart Time to restart from - * @param dt_restart Time step size to use - * @param cycle Cycle number to restart from + * @brief Time state needed to resume a run exactly after a completed cycle. + * + * @details Stored as unambiguous quantities because the raw members change meaning + * with the tracker: after a successful UpdateDeltaTime() that returns NORMAL/FINAL, + * `time` is the END time of the NEXT step and `simulation_cycle` counts completed + * steps; after FINAL->FINISHED, `time` is the end of the step just completed and + * `simulation_cycle` was NOT incremented. `old_time` is the end time of the step just + * completed in both cases. + * + * `previous_dt` and `original_dt` are carried along so that a restarted run's time-step + * logging (PrintTimeStats(), PrintRetrialStats(), PrintSubStepStats()) behaves exactly as + * the uninterrupted run's would. + */ + struct CheckpointState { + /** @brief End time of the last completed step (old_time) */ + double completed_time = 0.0; + /** @brief Time step the writing run would have taken next */ + double next_dt = 0.0; + /** @brief Time step of the last completed step (prev_dt) */ + double previous_dt = 0.0; + /** @brief Time step saved at the start of the last retrial sequence (dt_orig) */ + double original_dt = 0.0; + /** @brief Number of completed cycles (the index into custom_dt of the next step) */ + size_t completed_cycles = 0; + }; + + /** + * @brief Snapshot the resumable time state (see CheckpointState). + * @pre IsCheckpointSafe() + */ + CheckpointState GetCheckpointState() const { + CheckpointState state; + state.completed_time = old_time; + state.next_dt = dt; + state.previous_dt = prev_dt; + state.original_dt = dt_orig; + state.completed_cycles = simulation_cycle + + ((internal_tracker == TimeStep::FINISHED) ? 1 : 0); + return state; + } + + /** + * @brief True when no retrial or sub-step sequence is in flight, i.e. the retrial / + * sub-step counters carry no information a checkpoint would lose. + */ + bool IsCheckpointSafe() const { + return internal_tracker == TimeStep::NORMAL || internal_tracker == TimeStep::FINAL || + internal_tracker == TimeStep::FINISHED; + } + + /** + * @brief Resume from a CheckpointState using THIS run's time-step configuration. * - * @details Used for simulation restarts from checkpoint data. - * Sets all time-related state to the specified restart values. - * Does not modify time step type or other configuration parameters. + * @details The next dt comes from the current scheme (CUSTOM: custom_dt[completed + * cycles] when available; FIXED: dt_fixed; AUTO: the stored dt clamped to + * [dt_min, dt_max]), then the same final-time logic as UpdateDeltaTime() is applied + * against the current time_final, so extending t_final (or the custom dt list) on a + * restart continues the run and a checkpoint already at t_final reports FINISHED. + * The logging-only previous and original time steps are restored as stored. */ - void RestartTimeState(const double time_restart, const double dt_restart, const size_t cycle) { - simulation_cycle = cycle; - time = time_restart; - dt = dt_restart; + void ResumeFromCheckpoint(const CheckpointState& state); + + /** @brief Active time stepping mode */ + TimeStepType GetTimeStepType() const { + return time_type; + } + + /** @brief Configured final simulation time */ + double GetFinalTime() const { + return time_final; } /** @@ -459,14 +519,17 @@ class SimulationState { /** @brief Reference to simulation options */ ExaOptions& m_options; -#if defined(EXACONSTIT_USE_AXOM) - // We want this to be something akin to a axom::sidre::MFEMSidreDataCollection - // However, we need it flexible enough to handle multiple different mesh topologies in it that - // we might due to different mfem::SubMesh objects that correspond to each - // PartialQuadraturePoint - /** @brief Simulation restart data store (optional Axom/Sidre support) */ - std::unique_ptr m_simulation_restart; -#endif + /** + * @brief Binary Sidre checkpoint/restart driver; null unless [Restart] read or write. + * + * @details Owned here. It borrows `*this` for the duration of each call and never + * stores a pointer back to its owner (a stored shared_ptr would form a cycle, and + * shared_from_this() is unavailable inside the constructor where the mesh is rebuilt). + * Friend access lets it enumerate the QSpace/QF registries and resume TimeManagement + * without widening the public API. See doc/restart.md. + */ + std::unique_ptr m_restart; + friend class exaconstit::RestartCheckpoint; /** @brief MPI rank identifier */ int my_id; @@ -566,7 +629,8 @@ class SimulationState { * @param qf_name Quadrature function name * @param region Region index (-1 for global) * @return Shared pointer to the quadrature function - * @throws std::runtime_error if quadrature function doesn't exist + * @return The quadrature function, or nullptr if no function is registered under the + * resolved name (the lookup never inserts into the registry). * * @details Primary interface for accessing simulation data. Automatically resolves * region-specific names and returns the appropriate quadrature function. @@ -585,7 +649,10 @@ class SimulationState { */ std::shared_ptr GetQuadratureFunction(const std::string_view& qf_name, const int region = -1) { - return m_map_qfs[GetQuadratureFunctionMapName(qf_name, region)]; + // find(), not operator[]: a lookup of an unregistered name (e.g. a region inactive on + // this rank) must not insert a nullptr entry into m_map_qfs. + const auto it = m_map_qfs.find(GetQuadratureFunctionMapName(qf_name, region)); + return (it != m_map_qfs.end()) ? it->second : nullptr; } /** @@ -594,7 +661,9 @@ class SimulationState { * @param qf_name Base quadrature function name * @param vdim Vector dimension of the quadrature function * @param region Region index (-1 for global) - * @return True if successfully added, false if already exists + * @return True if added; false if a function of that name is already registered, or if + * the region has no quadrature space on this rank (it owns none of the region's + * elements) * * @details Creates a new quadrature function with the specified vector dimension * and associates it with the given region. The function is initialized with zeros @@ -605,6 +674,27 @@ class SimulationState { * - Stress/strain tensors: vdim = 6 (Voigt notation) * - Deformation gradients: vdim = 9 (3x3 matrix) * - State variables: vdim = model-dependent + * + * **Checkpoint/restart contract.** Every registered quadrature function is written to + * restart checkpoints and restored on restart automatically, with no restart code + * needed for it (the one exception is `tangent_stiffness`, which is recomputed before + * use). Quadrature point data held anywhere else, such as a model's own mfem::Vector or + * PartialQuadratureFunction, is NOT checkpointed, and nothing warns about it. So: + * - Any per-point quantity that carries history from one step to the next must be + * registered here. Scratch data that is fully recomputed every step does not need to + * be. + * - Register it while the simulation objects are being constructed (SimulationState, + * SystemDriver, the models, PostProcessingDriver), i.e. before the driver restores a + * checkpoint. A function registered later keeps its initial value on restart. + * - The writing and restarted runs must register the same functions with the same vector + * dimension. A function only one of them registers is reported as a warning and starts + * from its initial value. + * - For a beginning/end-of-step pair, also register the pair with + * AddUpdateVariablePairNames(). This is optional for restart (an unregistered pair is + * simply stored in full), but lets the checkpoint store an identical end-of-step + * function as a reference to its partner. + * - Call this on every rank. Ranks that own none of the region's elements get false back + * and have no function of that name, which the checkpoint handles. */ bool AddQuadratureFunction(const std::string_view& qf_name, const int vdim = 1, @@ -680,7 +770,7 @@ class SimulationState { */ void UpdateModel() { for (auto [name_prev, name_cur] : m_model_update_qf_pairs) { - m_map_qfs[name_prev]->Swap(*m_map_qfs[name_cur]); + m_map_qfs.at(name_prev)->Swap(*m_map_qfs.at(name_cur)); } } @@ -693,7 +783,8 @@ class SimulationState { */ void SetupModelVariables() { for (auto [name_prev, name_cur] : m_model_update_qf_pairs) { - m_map_qfs[name_cur]->operator=(*m_map_qfs[name_prev]); + // Value copy (PartialQuadratureFunction's explicit copy assignment). + *m_map_qfs.at(name_cur) = *m_map_qfs.at(name_prev); } } @@ -1093,14 +1184,14 @@ class SimulationState { * @brief Update time step based on solver performance * * @param nr_steps Number of Newton-Raphson iterations required - * @param failure Whether the time step failed to converge + * @param success Whether the time step converged (true) or failed (false) * @return Updated time step status * * @details Delegates to TimeManagement for comprehensive adaptive time step control. * See TimeManagement::UpdateDeltaTime() for detailed algorithm description. */ - TimeStep UpdateDeltaTime(const int nr_steps, const bool failure = false) { - return m_time_manager.UpdateDeltaTime(nr_steps, failure); + TimeStep UpdateDeltaTime(const int nr_steps, const bool success) { + return m_time_manager.UpdateDeltaTime(nr_steps, success); } /** @@ -1150,6 +1241,55 @@ class SimulationState { m_time_manager.PrintRetrialStats(); } + // ========================================================================= + // CHECKPOINT / RESTART + // ========================================================================= + + /** + * @brief Check whether this run restarts from a checkpoint + * + * @return True if [Restart] read is set + * + * @details On a restart run the constructor rebuilt the mesh and took the grain and + * region IDs from the checkpoint, and skipped the orientation and state-variable + * initialization. The driver must call RestoreFromCheckpoint() before the first time step. + */ + bool IsRestartRun() const { + return m_options.restart.read; + } + + /** + * @brief Overwrite the evolving state with the checkpointed values (restart phase 2) + * + * @details Call exactly once on a restart run, after SystemDriver and PostProcessingDriver + * have been constructed (so every quadrature function they register exists) and before the + * first time step. Restores the nodal fields and every quadrature function present both in + * this run and in the checkpoint, resumes the time state and recomputes the derived fields. + * Aborts if the checkpoint does not match this run. Collective over MPI_COMM_WORLD. + * See exaconstit::RestartCheckpoint::RestoreState(). + */ + void RestoreFromCheckpoint(); + + /** + * @brief Write a checkpoint if one is due + * + * @param driver_step The driver's loop counter `ti` for the step just completed + * + * @details Call at the end of every driver-loop iteration, after FinishCycle(), the model + * update and post-processing. Does nothing unless [Restart] write is set; when it is, the + * [Restart] interval and write_final settings decide whether this step is written. + * Collective over MPI_COMM_WORLD, so every rank must call it every iteration. + */ + void WriteCheckpointIfDue(size_t driver_step); + + /** + * @brief Driver loop counter recorded in the checkpoint + * + * @return The `ti` of the last step completed before the checkpoint, or 0 if this run is + * not a restart; the first restarted iteration is this value + 1 + */ + size_t GetRestartDriverStep() const; + private: /** @brief Create MPI communicators for each region containing only ranks with that region * @details This prevents deadlocks in collective operations when some ranks have no diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c521415d..a897a44f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,6 +47,172 @@ blt_add_executable(NAME test_grad_oper blt_add_test(NAME test_gradient_operation COMMAND test_grad_oper) +#################################### +# Checkpoint/restart tests (Axom Sidre) +#################################### +if(ENABLE_AXOM) + # Mesh round trip through the checkpoint mesh I/O, including a case with empty ranks + blt_add_executable(NAME test_restart_mesh_roundtrip + SOURCES test_restart_mesh_roundtrip.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${EXACONSTIT_TEST_DEPENDS} gtest) + + # SimulationState checkpoint, delete the input files, restart, compare bitwise + blt_add_executable(NAME test_restart_e2e + SOURCES test_restart_e2e.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${EXACONSTIT_TEST_DEPENDS} gtest) + + # MPI tests never use more ranks than the machine has cores, so the suite does not + # oversubscribe small machines such as the 4-core CI runners (Open MPI refuses to start more + # ranks than slots by default). Open MPI counts physical cores as slots, not hardware + # threads, so the default is the number of physical cores CMake detects (falling back to + # logical cores if that is unknown); set EXACONSTIT_TEST_MAX_MPI_TASKS to override it. Rank + # counts above the cap are skipped. + cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_PHYSICAL_CORES) + if(NOT _detected_cores OR _detected_cores LESS 1) + cmake_host_system_information(RESULT _detected_cores QUERY NUMBER_OF_LOGICAL_CORES) + endif() + if(NOT _detected_cores OR _detected_cores LESS 1) + set(_detected_cores 1) + endif() + set(EXACONSTIT_TEST_MAX_MPI_TASKS ${_detected_cores} CACHE STRING + "Largest number of MPI ranks any test may use (default: detected physical cores)") + + foreach(_np 1 2 3 4) + if(NOT _np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + blt_add_test(NAME test_restart_mesh_roundtrip_np${_np} + COMMAND test_restart_mesh_roundtrip + NUM_MPI_TASKS ${_np}) + endif() + endforeach() + + foreach(_np 2 3) + if(NOT _np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + blt_add_test(NAME test_restart_e2e_np${_np} + COMMAND test_restart_e2e + NUM_MPI_TASKS ${_np}) + endif() + endforeach() + + # The two tools below only use Axom, but an Axom built with GPU support requires every + # file using axom::Array to be compiled as device code, so they need the GPU dependency + # that makes BLT compile them with the CUDA/HIP compiler. + set(_restart_tool_depends axom) + if(ENABLE_CUDA) + if (${BLT_VERSION} VERSION_GREATER_EQUAL 0.6.0) + list(APPEND _restart_tool_depends blt::cuda_runtime blt::cuda) + else() + list(APPEND _restart_tool_depends cuda cuda_runtime) + endif() + endif() + if(ENABLE_HIP) + list(APPEND _restart_tool_depends blt::hip blt::hip_runtime) + endif() + + # Diagnostic: lists how each quadrature function is stored in a checkpoint rank file + blt_add_executable(NAME inspect_restart_rank_file + SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/inspect_restart_rank_file.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${_restart_tool_depends}) + + # Compares two checkpoint directories view by view (bitwise by default) + blt_add_executable(NAME compare_restart_checkpoints + SOURCES ${CMAKE_SOURCE_DIR}/scripts/restart/compare_restart_checkpoints.cpp + OUTPUT_DIR ${TEST_OUTPUT_DIR} + DEPENDS_ON ${_restart_tool_depends}) + + # Restart outputs (a short case of test_restart_runs.py, part of the default suite): + # checkpoints go inside the run's output folder, and restarts keep the visualization files + # written before them (ParaView and VisIt, also restarting from an earlier checkpoint) + set(_restart_outputs_np 2) + if(_restart_outputs_np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + set(_restart_outputs_np ${EXACONSTIT_TEST_MAX_MPI_TASKS}) + endif() + set(_restart_outputs_mpiexec "${MPIEXEC_EXECUTABLE}") + if(NOT _restart_outputs_mpiexec) + set(_restart_outputs_mpiexec "mpirun") + endif() + add_test(NAME test_restart_outputs + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test_restart_runs.py + --cases outputs + --nxyz 4 + --np ${_restart_outputs_np} + --always-mpiexec + --mpiexec ${_restart_outputs_mpiexec} + --exe $ + --compare-tool $ + --workdir ${CMAKE_BINARY_DIR}/test/restart_outputs + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_outputs PROPERTIES + LABELS "restart" + PROCESSORS ${_restart_outputs_np}) + + # Restart tests on real runs (test_restart_runs.py): cyclic loading chained every 1 and 2 + # cycles, long monotonic tension chained halfway, a run with empty ranks, restarts from past + # reference checkpoints, and a negative control. They take 10-15 minutes on 8 ranks, so they + # are not part of the default (CI) test suite. Developers working on the restart code must + # enable them locally with -DEXACONSTIT_ENABLE_LONG_TESTS=ON and make sure they pass: + # ctest -L long --output-on-failure + option(EXACONSTIT_ENABLE_LONG_TESTS + "Register the long-running tests (run-level restart tests); not for CI" OFF) + set(EXACONSTIT_RESTART_RUN_TEST_NP 8 CACHE STRING + "MPI ranks for test_restart_runs (capped at EXACONSTIT_TEST_MAX_MPI_TASKS)") + if(EXACONSTIT_ENABLE_LONG_TESTS) + set(_restart_runs_np ${EXACONSTIT_RESTART_RUN_TEST_NP}) + if(_restart_runs_np GREATER EXACONSTIT_TEST_MAX_MPI_TASKS) + message(STATUS "test_restart_runs: using ${EXACONSTIT_TEST_MAX_MPI_TASKS} MPI ranks " + "instead of ${EXACONSTIT_RESTART_RUN_TEST_NP} " + "(EXACONSTIT_TEST_MAX_MPI_TASKS)") + set(_restart_runs_np ${EXACONSTIT_TEST_MAX_MPI_TASKS}) + endif() + set(_restart_runs_mpiexec "${MPIEXEC_EXECUTABLE}") + if(NOT _restart_runs_mpiexec) + set(_restart_runs_mpiexec "mpirun") + endif() + add_custom_command(TARGET compare_restart_checkpoints POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/test/test_restart_runs.py + $/../test/test_restart_runs.py) + add_test(NAME test_restart_runs + COMMAND ${PYTHON_EXECUTABLE} test_restart_runs.py + --np ${_restart_runs_np} + --mpiexec ${_restart_runs_mpiexec} + --exe $ + --compare-tool $ + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_runs PROPERTIES + LABELS "restart;long" + PROCESSORS ${_restart_runs_np} + RESOURCE_LOCK restart_runs + TIMEOUT 3600) + + # GPU builds also run the same tests on the device. Restored data is written on the + # host, so this checks that it reaches the device, and that GPU runs are bitwise + # reproducible across restarts. EA is the default since it is the fastest GPU assembly + # for these runs. + if(ENABLE_CUDA OR ENABLE_HIP) + set(EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY "EA" CACHE STRING + "Assembly used by test_restart_runs_gpu (FULL, PA or EA)") + add_test(NAME test_restart_runs_gpu + COMMAND ${PYTHON_EXECUTABLE} test_restart_runs.py + --np ${_restart_runs_np} + --rtmodel GPU + --assembly ${EXACONSTIT_RESTART_RUN_TEST_GPU_ASSEMBLY} + --mpiexec ${_restart_runs_mpiexec} + --exe $ + --compare-tool $ + --workdir ${CMAKE_BINARY_DIR}/test/restart_runs_gpu + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) + set_tests_properties(test_restart_runs_gpu PROPERTIES + LABELS "restart;long;gpu" + PROCESSORS ${_restart_runs_np} + RESOURCE_LOCK restart_runs + TIMEOUT 7200) + endif() + endif() +endif() + if (NOT DEFINED PYTHON_EXECUTABLE) message(FATAL_ERROR "cmake: PYTHON_EXECUTABLE must be defined if running test suite current value: ${PYTHON_EXECUTABLE}") endif() diff --git a/test/test_restart_e2e.cpp b/test/test_restart_e2e.cpp new file mode 100644 index 00000000..ece9a7ec --- /dev/null +++ b/test/test_restart_e2e.cpp @@ -0,0 +1,519 @@ +#include "sim_state/simulation_state.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// End-to-end restart test at the SimulationState level (no SystemDriver), for the checkpoint +// and restart code in src/sim_state/restart_checkpoint.* and the restart branches of the +// SimulationState constructor. +// +// Run A builds a real SimulationState from input files: a perturbed order-3 hex mesh whose +// element attributes are grain IDs, two UMAT regions assigned through a region-mapping file, +// and orientations for the first region. It is then given a synthetic evolved state (deformed +// coordinates, a velocity, distinct values in every quadrature function, three completed time +// steps) and writes a checkpoint. The mesh, region-mapping and orientation files are then +// deleted, and run B restarts from the checkpoint alone. The test checks that: +// - phase 1 reproduces the original reference coordinates bit for bit inside the constructor +// (the vertex-geometry projection a rebuilt mesh would otherwise use differs in the last +// bits; the summary line reports by how much); +// - after phase 2 the mesh attributes, grain IDs, region flags, every nodal field, every +// checkpointed quadrature function and the full time state are bitwise identical, and the +// time-step line the driver logs reads the same as in run A; +// - the tangent stiffness is not restored (it keeps its constructor value until the first +// model setup recomputes it); +// - the restarted run can write its next checkpoint. +// One begin/end state-variable pair is deliberately made unequal, so both the `copy_of` and the +// stored-values paths of the quadrature-function checkpoint are exercised. +// +// The first region also covers orientation placement: its grain table has no +// ori_state_var_loc, so the quaternions go at the end of each state vector. +// +// Run with 2 or 3 ranks, for example `mpirun -np 2 ./test_restart_e2e`. Scratch files go to +// restart_e2e_np/ in the working directory. + +namespace fs = std::filesystem; + +namespace { + +/** + * @brief Deterministic unit quaternion for a grain + * + * @param grain Grain ID + * @return Normalized quaternion (w, x, y, z), different for every grain + */ +std::array GrainQuaternion(const int grain) { + std::array quaternion = {1.0, 0.1 * grain, 0.2 * grain, 0.05 * grain}; + const double norm = std::sqrt(quaternion[0] * quaternion[0] + quaternion[1] * quaternion[1] + + quaternion[2] * quaternion[2] + quaternion[3] * quaternion[3]); + for (double& component : quaternion) { + component /= norm; + } + return quaternion; +} + +/** + * @brief Check two vectors for bitwise equality + * + * @param a First vector + * @param b Second vector + * @return True if both have the same size and every entry compares equal + */ +bool BitwiseEqual(const mfem::Vector& a, const mfem::Vector& b) { + if (a.Size() != b.Size()) { + return false; + } + if (a.Size() == 0) { + return true; + } + const double* x = a.HostRead(); + const double* y = b.HostRead(); + for (int i = 0; i < a.Size(); ++i) { + if (x[i] != y[i]) { + return false; + } + } + return true; +} + +/** + * @brief Capture the time-step line the driver logs at the start of a cycle + * + * @param sim_state Simulation state whose time manager is printed + * @return The output of SimulationState::PrintTimeStats(), printed with 17 significant digits + * so that any difference in the time steps shows up in the text + */ +std::string CaptureTimeStats(const SimulationState& sim_state) { + std::ostringstream captured; + std::streambuf* original = std::cout.rdbuf(captured.rdbuf()); + const std::streamsize original_precision = std::cout.precision(17); + sim_state.PrintTimeStats(); + std::cout.precision(original_precision); + std::cout.rdbuf(original); + return captured.str(); +} + +/** + * @brief Options for the test runs + * + * @param directory Scratch directory holding the input files and the checkpoints + * @param read Whether this run restarts from a checkpoint + * @return Options with two UMAT regions, AUTO time stepping and checkpoint writing enabled + * + * @details The first region ("soft") has 3 state variables plus quaternion orientations with + * no ori_state_var_loc, so the orientations go at the end of its state vector (7 values per + * point). The second region ("hard") has 5 state variables and no orientations. + */ +ExaOptions MakeOptions(const fs::path& directory, const bool read) { + ExaOptions options; + options.basename = "e2e"; + options.mesh.mesh_type = MeshType::FILE; + options.mesh.mesh_file = directory / "mesh.mesh"; + options.mesh.order = 3; + // AUTO stepping with dt_max well above the steps taken, so the Newton iteration counts + // FillEvolvedState() reports make every step a different size and the logged "dt old was" + // value differs from the next dt. + options.time.time_type = TimeStepType::AUTO; + options.time.auto_time = TimeOptions::AutoTimeOptions{}; + options.time.auto_time->dt_start = 0.1; + options.time.auto_time->dt_min = 1e-3; + options.time.auto_time->dt_max = 10.0; + options.time.auto_time->t_final = 100.0; + options.solvers.rtmodel = RTModel::CPU; + options.region_mapping_file = directory / "regions.txt"; + for (int region = 0; region < 2; ++region) { + MaterialOptions material; + material.material_name = (region == 0) ? "soft" : "hard"; + material.region_id = region + 1; + material.mech_type = MechType::UMAT; + material.properties.num_props = 2; + material.properties.properties = {10.0 + region, 20.0 + region}; + material.state_vars.num_vars = 3 + 2 * region; + for (int k = 0; k < material.state_vars.num_vars; ++k) { + material.state_vars.initial_values.push_back(1.0 + k + 10 * region); + } + if (region == 0) { + GrainInfo grains; + grains.orientation_file = directory / "orientations.txt"; + grains.ori_type = OriType::QUAT; + grains.ori_stride = 4; + grains.num_grains = 8; + grains.ori_state_var_loc = -1; + material.grain_info = grains; + } + options.materials.push_back(material); + } + options.restart.write = true; + options.restart.interval = 1; + options.restart.keep_last = 2; + options.restart.read = read; + options.restart.directory = directory / "restart"; + return options; +} + +/** + * @brief A quadrature function registry entry: base name and region (-1 for global) + */ +struct QuadratureFunctionName { + /** @brief Base name, as passed to SimulationState::GetQuadratureFunction() */ + std::string base; + /** @brief Region index, or -1 for a global function */ + int region; +}; + +/** + * @brief The quadrature functions the test fills and compares + * + * @return Global and per-region names; functions not created on a rank are skipped there + */ +std::vector QuadratureFunctionNames() { + std::vector names = {{"cauchy_stress_beg", -1}, + {"cauchy_stress_end", -1}, + {"cauchy_stress_avg", -1}, + {"kinetic_grads", -1}, + {"tangent_stiffness", -1}}; + for (int region = 0; region < 2; ++region) { + for (const char* base : {"state_var_beg", + "state_var_end", + "state_var_avg", + "cauchy_stress_beg", + "cauchy_stress_end", + "cauchy_stress_avg", + "tangent_stiffness", + "def_grad_beg"}) { + names.push_back({base, region}); + } + } + return names; +} + +/** + * @brief Give a freshly constructed SimulationState a synthetic evolved state + * + * @param sim_state Simulation state to modify + * + * @details Mirrors what the driver leaves behind after three converged steps, in the same + * order (solve, FinishCycle(), model update, post-processing, checkpoint): + * - a velocity with distinct values per true DOF, and current coordinates displaced by it + * (built from true DOFs, so the field is continuous across ranks); + * - distinct deterministic values in every quadrature function, different per name and rank; + * - three completed AUTO time steps with different Newton iteration counts, so each step has + * a different size; + * - SetupModelVariables(), which makes every end-of-step function equal to its + * beginning-of-step partner, as SystemDriver::UpdateModel() does. + * Finally one state-variable pair is made unequal on purpose, so its end-of-step values are + * written in full rather than as `copy_of`. + */ +void FillEvolvedState(SimulationState& sim_state) { + auto space = sim_state.GetMeshParFiniteElementSpace(); + mfem::Vector& primal = *sim_state.GetPrimalField(); + const HYPRE_BigInt offset = space->GetMyTDofOffset(); + for (int i = 0; i < primal.Size(); ++i) { + primal[i] = std::sin(0.37 * static_cast(offset + i)); + } + mfem::Vector reference_true; + sim_state.GetRefCoords()->GetTrueDofs(reference_true); + mfem::Vector current_true = reference_true; + current_true.Add(0.01, primal); + sim_state.GetCurrentCoords()->SetFromTrueDofs(current_true); + + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + int salt = 1; + for (const auto& name : QuadratureFunctionNames()) { + auto function = sim_state.GetQuadratureFunction(name.base, name.region); + ++salt; + if (!function || function->Size() == 0) { + continue; + } + double* data = function->HostReadWrite(); + for (int i = 0; i < function->Size(); ++i) { + data[i] = std::cos(0.013 * i + 0.7 * salt + 0.11 * rank); + } + } + + for (const int newton_iterations : {3, 7, 12}) { + sim_state.UpdateDeltaTime(newton_iterations, true); + } + sim_state.FinishCycle(); + sim_state.SetupModelVariables(); + + auto end_state = sim_state.GetQuadratureFunction("state_var_end", 0); + if (end_state && end_state->Size() > 0) { + end_state->HostReadWrite()[0] += 1.0; + } +} + +/** + * @brief Write the input files of run A (rank 0 only) + * + * @param directory Scratch directory + * + * @details A perturbed 4x4x4 hex mesh whose element attributes are grain IDs 1..8, an + * orientation file with one quaternion per grain, and a region-mapping file sending grains + * 1-4 to region 1 and grains 5-8 to region 2. The mesh is written with 17 significant digits so + * the vertex positions survive the text round trip exactly. + */ +void WriteInputFiles(const fs::path& directory) { + fs::remove_all(directory); + fs::create_directories(directory); + + mfem::Mesh mesh = mfem::Mesh::MakeCartesian3D( + 4, 4, 4, mfem::Element::HEXAHEDRON, 1.0, 1.0, 1.0, false); + for (int vertex = 0; vertex < mesh.GetNV(); ++vertex) { + double* x = mesh.GetVertex(vertex); + const double x0 = x[0]; + const double x1 = x[1]; + const double x2 = x[2]; + x[0] += 0.01 * std::sin(3.1 * x1 + 1.7 * x2 + 0.3); + x[1] += 0.01 * std::sin(2.3 * x2 + 0.9 * x0 + 1.1); + x[2] += 0.01 * std::sin(1.9 * x0 + 2.9 * x1 + 0.7); + } + for (int element = 0; element < mesh.GetNE(); ++element) { + mesh.SetAttribute(element, 1 + (element * 5) % 8); + } + mesh.SetAttributes(); + std::ofstream mesh_file(directory / "mesh.mesh"); + mesh_file.precision(17); + mesh.Print(mesh_file); + + std::ofstream orientations(directory / "orientations.txt"); + orientations.precision(17); + for (int grain = 1; grain <= 8; ++grain) { + const auto quaternion = GrainQuaternion(grain); + orientations << quaternion[0] << " " << quaternion[1] << " " << quaternion[2] << " " + << quaternion[3] << "\n"; + } + + std::ofstream regions(directory / "regions.txt"); + for (int grain = 1; grain <= 8; ++grain) { + regions << grain << " " << (grain <= 4 ? 1 : 2) << "\n"; + } +} + +/** + * @brief Count the quadrature points of region 0 whose state does not match the inputs + * + * @param sim_state Freshly constructed simulation state of run A + * @return Number of points whose state vector is not {1, 2, 3, quaternion of the grain} + */ +int CountMisplacedOrientations(SimulationState& sim_state) { + auto state = sim_state.GetQuadratureFunction("state_var_beg", 0); + if (!state || state->GetVDim() != 7) { + return state ? 1 : 0; + } + auto space = state->GetPartialSpaceShared(); + const auto& local_to_global = space->GetLocal2Global(); + const int num_elements = space->GetNumLocalElements(); + if (num_elements == 0) { + return 0; + } + const int points_per_element = state->Size() / 7 / num_elements; + const double* data = state->HostRead(); + const double* grains = sim_state.GetGrains()->HostRead(); + int wrong = 0; + for (int element = 0; element < num_elements; ++element) { + const auto quaternion = GrainQuaternion(static_cast(grains[local_to_global[element]])); + for (int point = 0; point < points_per_element; ++point) { + const double* s = data + + static_cast(element * points_per_element + point) * 7; + const bool ok = s[0] == 1.0 && s[1] == 2.0 && s[2] == 3.0 && s[3] == quaternion[0] && + s[4] == quaternion[1] && s[5] == quaternion[2] && s[6] == quaternion[3]; + wrong += ok ? 0 : 1; + } + } + return wrong; +} + +/** + * @brief Compare the original reference coordinates with a vertex-geometry projection + * + * @param directory Scratch directory holding the mesh file + * @param reference Reference coordinates of run A + * @return Number of differing entries on this rank and the largest difference + * + * @details A mesh rebuilt from a checkpoint has no Nodes, so GetNodes() projects from the + * vertex positions. This measures how far that is from the original reference coordinates, + * which is the gap the phase-1 coordinate overlay closes. + */ +std::pair VertexProjectionDifference(const fs::path& directory, + const mfem::Vector& reference) { + mfem::Mesh serial((directory / "mesh.mesh").string().c_str(), 1, 1, true); + mfem::ParMesh vertex_mesh(MPI_COMM_WORLD, serial); + mfem::H1_FECollection fec(3, 3); + mfem::ParFiniteElementSpace space(&vertex_mesh, &fec, 3); + mfem::ParGridFunction projected(&space); + vertex_mesh.GetNodes(projected); + int mismatches = 0; + double max_difference = 0.0; + if (projected.Size() == reference.Size()) { + for (int i = 0; i < projected.Size(); ++i) { + const double difference = std::abs(projected(i) - reference(i)); + mismatches += difference != 0.0 ? 1 : 0; + max_difference = std::max(max_difference, difference); + } + } + return {mismatches, max_difference}; +} + +} // namespace + +TEST(restart_simulation_state, checkpoint_and_restart) { + int rank = 0; + int num_ranks = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + const fs::path directory = fs::current_path() / ("restart_e2e_np" + std::to_string(num_ranks)); + if (rank == 0) { + WriteInputFiles(directory); + } + MPI_Barrier(MPI_COMM_WORLD); + + // --------------------------------------------------------------------- + // Run A: build from the input files, evolve, checkpoint + // --------------------------------------------------------------------- + ExaOptions options_a = MakeOptions(directory, false); + SimulationState run_a(options_a); + EXPECT_EQ(CountMisplacedOrientations(run_a), 0) + << "region 0 state vectors should be {1, 2, 3} followed by the grain's quaternion"; + const auto [projection_mismatches, projection_max_difference] = VertexProjectionDifference( + directory, *run_a.GetRefCoords()); + + FillEvolvedState(run_a); + run_a.WriteCheckpointIfDue(3); + EXPECT_TRUE(fs::exists(directory / "restart" / "cycle_0000000003" / "commit.json")) + << "run A should have committed a checkpoint for cycle 3"; + + // The restart must not need the original input files. + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0) { + fs::remove(directory / "mesh.mesh"); + fs::remove(directory / "regions.txt"); + fs::remove(directory / "orientations.txt"); + } + MPI_Barrier(MPI_COMM_WORLD); + + // --------------------------------------------------------------------- + // Run B: phase 1 (constructor) must reproduce the original construction-time geometry + // --------------------------------------------------------------------- + ExaOptions options_b = MakeOptions(directory, true); + SimulationState run_b(options_b); + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetRefCoords())) << "phase 1: mesh_ref"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetCurrentCoords())) + << "phase 1: mesh_current should equal the reference coordinates"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetTimeStartCoords())) + << "phase 1: mesh_t_beg should equal the reference coordinates"; + + // --------------------------------------------------------------------- + // Run B: phase 2 must restore everything bitwise + // --------------------------------------------------------------------- + run_b.RestoreFromCheckpoint(); + + EXPECT_EQ(run_a.GetMesh()->GetNE(), run_b.GetMesh()->GetNE()) << "local element count"; + for (int element = 0; element < std::min(run_a.GetMesh()->GetNE(), run_b.GetMesh()->GetNE()); + ++element) { + if (run_a.GetMesh()->GetAttribute(element) != run_b.GetMesh()->GetAttribute(element)) { + ADD_FAILURE() << "region attribute of element " << element; + break; + } + } + EXPECT_TRUE(BitwiseEqual(*run_a.GetGrains(), *run_b.GetGrains())) << "grain IDs"; + for (int region = 0; region < 2; ++region) { + EXPECT_EQ(run_a.IsRegionActive(region), run_b.IsRegionActive(region)) + << "region " << region << " active flag"; + } + + EXPECT_TRUE(BitwiseEqual(*run_a.GetRefCoords(), *run_b.GetRefCoords())) << "mesh_ref"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetCurrentCoords(), *run_b.GetCurrentCoords())) + << "mesh_current"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetTimeStartCoords(), *run_b.GetTimeStartCoords())) + << "mesh_t_beg"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetPrimalField(), *run_b.GetPrimalField())) << "velocity"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetPrimalFieldPrev(), *run_b.GetPrimalFieldPrev())) + << "previous velocity"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetVelocity(), *run_b.GetVelocity())) + << "velocity grid function"; + EXPECT_TRUE(BitwiseEqual(*run_a.GetDisplacement(), *run_b.GetDisplacement())) << "displacement"; + + int compared = 0; + for (const auto& name : QuadratureFunctionNames()) { + auto function_a = run_a.GetQuadratureFunction(name.base, name.region); + auto function_b = run_b.GetQuadratureFunction(name.base, name.region); + const std::string label = name.base + " (region " + std::to_string(name.region) + ")"; + if (!function_a || !function_b) { + EXPECT_TRUE(!function_a && !function_b) << "present in only one run: " << label; + continue; + } + if (name.base == "tangent_stiffness") { + // Not checkpointed: it keeps the constructor's zeros until the first model setup. + bool zero = true; + if (function_b->Size() > 0) { + const double* data = function_b->HostRead(); + for (int i = 0; i < function_b->Size(); ++i) { + zero = zero && data[i] == 0.0; + } + } + EXPECT_TRUE(zero) << "the tangent should not be restored: " << label; + } else { + EXPECT_TRUE(BitwiseEqual(*function_a, *function_b)) << "restored values: " << label; + ++compared; + } + } + + EXPECT_EQ(run_a.GetTime(), run_b.GetTime()) << "time"; + EXPECT_EQ(run_a.GetDeltaTime(), run_b.GetDeltaTime()) << "dt"; + EXPECT_EQ(run_a.GetSimulationCycle(), run_b.GetSimulationCycle()) << "cycle"; + EXPECT_EQ(run_a.GetTrueCycleTime(), run_b.GetTrueCycleTime()) << "completed time"; + EXPECT_EQ(run_a.IsLastStep(), run_b.IsLastStep()) << "time step tracker (last step)"; + EXPECT_EQ(run_a.IsFinished(), run_b.IsFinished()) << "time step tracker (finished)"; + EXPECT_EQ(run_b.GetRestartDriverStep(), 3u) << "driver step"; + // The line the driver logs at the start of the next cycle, including the logging-only + // previous time step, must read exactly as in the uninterrupted run. + EXPECT_EQ(CaptureTimeStats(run_a), CaptureTimeStats(run_b)) << "logged time-step change"; + + // --------------------------------------------------------------------- + // Both runs take one more step; run B writes its next checkpoint + // --------------------------------------------------------------------- + run_a.UpdateDeltaTime(5, true); + run_a.FinishCycle(); + run_b.UpdateDeltaTime(5, true); + run_b.FinishCycle(); + EXPECT_EQ(run_a.GetTime(), run_b.GetTime()) << "time after one more step"; + EXPECT_EQ(run_a.GetDeltaTime(), run_b.GetDeltaTime()) << "dt after one more step"; + EXPECT_EQ(CaptureTimeStats(run_a), CaptureTimeStats(run_b)) + << "logged time-step change after one more step"; + run_b.WriteCheckpointIfDue(4); + EXPECT_TRUE(fs::exists(directory / "restart" / "cycle_0000000004" / "commit.json")) + << "the restarted run should have committed a checkpoint for cycle 4"; + + int global_mismatches = 0; + double global_max_difference = 0.0; + MPI_Allreduce(&projection_mismatches, &global_mismatches, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce( + &projection_max_difference, &global_max_difference, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (rank == 0) { + std::cout << "Restart end-to-end: " << num_ranks << " ranks, " << compared + << " quadrature functions compared on rank 0; a vertex-geometry projection " + "would have differed from the original reference coordinates in " + << global_mismatches << " entries (max " << global_max_difference << ")" + << std::endl; + } +} + +int main(int argc, char* argv[]) { + MPI_Init(&argc, &argv); + mfem::Hypre::Init(); + ::testing::InitGoogleTest(&argc, argv); + const int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} diff --git a/test/test_restart_mesh_roundtrip.cpp b/test/test_restart_mesh_roundtrip.cpp new file mode 100644 index 00000000..fa7a0c3f --- /dev/null +++ b/test/test_restart_mesh_roundtrip.cpp @@ -0,0 +1,536 @@ +#include "sim_state/exa_sidre_parmesh.hpp" + +#include "mfem.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axom/sidre.hpp" +#include + +// Mesh round-trip test for the restart checkpoint mesh I/O (WriteParMeshBlueprint and +// ExaSidreParMesh, src/sim_state/exa_sidre_parmesh.*). +// +// Each case partitions a small perturbed serial mesh, writes every rank's piece with +// WriteParMeshBlueprint(), saves it to HDF5, loads it back into a fresh DataStore, rebuilds the +// parallel mesh with ExaSidreParMesh, and then checks that the rebuilt mesh is identical to the +// original in every way the finite element spaces depend on: +// - local topology counts and the global element count; +// - every communication group: its master and, entity by entity in group order, its shared +// vertices, edges and faces, including each shared edge's and face's orientation; +// - for H1 spaces of order 1 to 4: identical local, true and global true sizes, bitwise +// identical local and true-DOF vectors of a projected nonlinear field, identical +// element-local DOF hashes, and a field that survives restriction to true DOFs and +// prolongation back unchanged (shared DOFs matched consistently across ranks). +// +// Order 4 matters: a shared face then carries several interior DOFs, so a wrong shared-face +// orientation changes which DOFs are matched across ranks (see the background notes in +// exa_sidre_parmesh.cpp). The `empty_ranks` case partitions the mesh so that half of the ranks +// own no elements, covering zero-length arrays in the checkpoint and empty rebuilt meshes. +// +// Every case also runs a negative control that rewrites the shared edges and faces in each +// rank's local vertex order before saving (what axom::sidre::MFEMSidreDataCollection writes) +// and reports how far the shared-DOF round trip is then off. It is printed for information +// rather than checked, since whether a given partition happens to have mismatched +// orientations is not guaranteed; on 2 or more ranks it is typically 0.1 or larger for hexes at +// order >= 3, against ~1e-15 for the real writer. +// +// Run with any number of ranks, for example `mpirun -np 3 ./test_restart_mesh_roundtrip`. +// Scratch files go to restart_roundtrip_np_/ in the working directory. + +using axom::sidre::DataStore; +using axom::sidre::Group; + +namespace { + +/** + * @brief 64-bit FNV-1a hash, the same as the one used by restart_checkpoint.cpp + */ +class Fnv1a64 { +public: + /** + * @brief Add raw bytes to the hash + * + * @param data Bytes to add + * @param byte_count Number of bytes + */ + void AddBytes(const void* data, const std::size_t byte_count) { + const auto* bytes = static_cast(data); + for (std::size_t i = 0; i < byte_count; ++i) { + m_state ^= bytes[i]; + m_state *= 1099511628211ULL; + } + } + + /** + * @brief Current hash value + * + * @return The hash of every byte added so far + */ + std::uint64_t Value() const { + return m_state; + } + +private: + /** @brief Running hash state, starting from the FNV-1a 64-bit offset basis */ + std::uint64_t m_state = 1469598103934665603ULL; +}; + +/** + * @brief Hash of a grid function visited element by element in local DOF order + * + * @param field Grid function to hash + * @return FNV-1a hash of the values in GetElementVDofs() order (sign-flipped for reversed DOFs) + * + * @details Same definition as ElementLocalFieldHash() in restart_checkpoint.cpp. Equal hashes + * on the original and rebuilt meshes mean both number their local DOFs the same way. + */ +std::uint64_t ElementLocalFieldHash(const mfem::ParGridFunction& field) { + const mfem::ParFiniteElementSpace* space = field.ParFESpace(); + const double* values = field.HostRead(); + mfem::Array element_vdofs; + Fnv1a64 hasher; + for (int element_index = 0; element_index < space->GetNE(); ++element_index) { + space->GetElementVDofs(element_index, element_vdofs); + for (const int signed_vdof : element_vdofs) { + const int vdof = signed_vdof >= 0 ? signed_vdof : -1 - signed_vdof; + const double value = signed_vdof >= 0 ? values[vdof] : -values[vdof]; + hasher.AddBytes(&value, sizeof(value)); + } + } + return hasher.Value(); +} + +/** + * @brief Largest change of a grid function under restriction to true DOFs and prolongation + * + * @param field Grid function to test + * @return max |field - P(R(field))| over this rank's local DOFs + * + * @details Collective. Zero (up to rounding) exactly when every shared DOF is matched with + * the same true DOF on every rank; see SharedDofInconsistency() in restart_checkpoint.cpp. + */ +double SharedDofInconsistency(const mfem::ParGridFunction& field) { + mfem::Vector true_values; + field.GetTrueDofs(true_values); + mfem::ParGridFunction round_trip(field.ParFESpace()); + round_trip.SetFromTrueDofs(true_values); + double max_difference = 0.0; + for (int i = 0; i < field.Size(); ++i) { + max_difference = std::max(max_difference, std::abs(field(i) - round_trip(i))); + } + return max_difference; +} + +/** + * @brief Smooth nonlinear vector field projected onto the H1 spaces + * + * @param x Point + * @param y Field value at x + * + * @details Nonlinear in every coordinate, so every DOF of a high-order space gets a distinct + * value and any DOF permutation changes the projected vector. + */ +void TestField(const mfem::Vector& x, mfem::Vector& y) { + y(0) = std::sin(2.0 * x(0) + x(1)) + 0.3 * x(2) * x(2); + y(1) = std::cos(x(0) - 3.0 * x(2)) + x(1) * x(1) * x(0); + y(2) = x(0) * x(1) * x(2) + std::sin(1.3 * x(2) + 0.7 * x(1)); +} + +/** + * @brief Move every vertex by a small smooth offset + * + * @param mesh Serial mesh to perturb + * @param amplitude Size of the offset + * + * @details Makes the coordinates distinct and non-axis-aligned, so a mismatched vertex or DOF + * always changes a value instead of possibly landing on an equal one. + */ +void PerturbVertices(mfem::Mesh& mesh, const double amplitude) { + for (int vertex = 0; vertex < mesh.GetNV(); ++vertex) { + double* x = mesh.GetVertex(vertex); + const double x0 = x[0]; + const double x1 = x[1]; + const double x2 = x[2]; + x[0] += amplitude * std::sin(3.1 * x1 + 1.7 * x2 + 0.3); + x[1] += amplitude * std::sin(2.3 * x2 + 0.9 * x0 + 1.1); + x[2] += amplitude * std::sin(1.9 * x0 + 2.9 * x1 + 0.7); + } +} + +/** + * @brief Largest value of a double over all ranks + * + * @param value This rank's value + * @return The maximum over MPI_COMM_WORLD + */ +double GlobalMax(const double value) { + double result = 0.0; + MPI_Allreduce(&value, &result, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return result; +} + +/** + * @brief Sorted MPI ranks of a communication group + * + * @param mesh Parallel mesh + * @param group Local group index + * @return The member ranks in increasing order + * + * @details Groups are matched between the original and rebuilt meshes by their member ranks, + * so the comparison does not assume anything about local group indices. + */ +std::vector GroupRanks(const mfem::ParMesh& mesh, const int group) { + std::vector ranks; + for (int member = 0; member < mesh.gtopo.GetGroupSize(group); ++member) { + ranks.push_back(mesh.gtopo.GetNeighborRank(mesh.gtopo.GetGroup(group)[member])); + } + std::sort(ranks.begin(), ranks.end()); + return ranks; +} + +/** + * @brief Check that two meshes have identical communication groups and shared entities + * + * @param original Mesh that was written + * @param rebuilt Mesh rebuilt from the checkpoint + * + * @details For every group of the original mesh, finds the rebuilt group with the same member + * ranks and checks its master and, entity by entity in group order, its shared vertices, the + * local vertices and orientation of its shared edges, and the local vertices and orientation + * of its shared triangles and quadrilaterals. Both the order and the orientation matter (see + * the background notes in exa_sidre_parmesh.cpp). + */ +void ExpectSameSharedStructures(const mfem::ParMesh& original, const mfem::ParMesh& rebuilt) { + EXPECT_EQ(original.GetNGroups(), rebuilt.GetNGroups()) << "number of communication groups"; + std::map, int> rebuilt_group_index; + for (int group = 1; group < rebuilt.GetNGroups(); ++group) { + rebuilt_group_index[GroupRanks(rebuilt, group)] = group; + } + + mfem::Array original_vertices; + mfem::Array rebuilt_vertices; + for (int ga = 1; ga < original.GetNGroups(); ++ga) { + const auto it = rebuilt_group_index.find(GroupRanks(original, ga)); + if (it == rebuilt_group_index.end()) { + ADD_FAILURE() << "original group " << ga << " has no rebuilt group with the same ranks"; + continue; + } + const int gb = it->second; + EXPECT_EQ(original.gtopo.GetGroupMasterRank(ga), rebuilt.gtopo.GetGroupMasterRank(gb)) + << "group master of group " << ga; + + // Shared vertices, in group order. + EXPECT_EQ(original.GroupNVertices(ga), rebuilt.GroupNVertices(gb)); + for (int i = 0; i < std::min(original.GroupNVertices(ga), rebuilt.GroupNVertices(gb)); + ++i) { + EXPECT_EQ(original.GroupVertex(ga, i), rebuilt.GroupVertex(gb, i)) + << "shared vertex " << i << " of group " << ga; + } + + // Shared edges: same local edge (by its vertices) and same orientation. + EXPECT_EQ(original.GroupNEdges(ga), rebuilt.GroupNEdges(gb)); + for (int i = 0; i < std::min(original.GroupNEdges(ga), rebuilt.GroupNEdges(gb)); ++i) { + int edge_a = -1; + int orientation_a = 0; + int edge_b = -1; + int orientation_b = 0; + original.GroupEdge(ga, i, edge_a, orientation_a); + rebuilt.GroupEdge(gb, i, edge_b, orientation_b); + original.GetEdgeVertices(edge_a, original_vertices); + rebuilt.GetEdgeVertices(edge_b, rebuilt_vertices); + EXPECT_TRUE(original_vertices[0] == rebuilt_vertices[0] && + original_vertices[1] == rebuilt_vertices[1] && + orientation_a == orientation_b) + << "shared edge " << i << " of group " << ga; + } + + // Shared faces: same local face (by its vertices) and same orientation. + const auto expect_same_faces = + [&](const int count_a, const int count_b, const bool triangles, const char* label) { + EXPECT_EQ(count_a, count_b) << "number of shared " << label; + for (int i = 0; i < std::min(count_a, count_b); ++i) { + int face_a = -1; + int orientation_a = 0; + int face_b = -1; + int orientation_b = 0; + if (triangles) { + original.GroupTriangle(ga, i, face_a, orientation_a); + rebuilt.GroupTriangle(gb, i, face_b, orientation_b); + } else { + original.GroupQuadrilateral(ga, i, face_a, orientation_a); + rebuilt.GroupQuadrilateral(gb, i, face_b, orientation_b); + } + original.GetFaceVertices(face_a, original_vertices); + rebuilt.GetFaceVertices(face_b, rebuilt_vertices); + bool same = original_vertices.Size() == rebuilt_vertices.Size() && + orientation_a == orientation_b; + for (int k = 0; same && k < original_vertices.Size(); ++k) { + same = original_vertices[k] == rebuilt_vertices[k]; + } + EXPECT_TRUE(same) << "shared " << label << " " << i << " of group " << ga; + } + }; + expect_same_faces( + original.GroupNTriangles(ga), rebuilt.GroupNTriangles(gb), true, "triangle"); + expect_same_faces(original.GroupNQuadrilaterals(ga), + rebuilt.GroupNQuadrilaterals(gb), + false, + "quadrilateral"); + } +} + +/** + * @brief Overwrite the stored shared edges and faces with each rank's local vertex order + * + * @param mesh Mesh that was written + * @param mesh_group Mesh group written by WriteParMeshBlueprint() + * + * @details Reproduces what axom::sidre::MFEMSidreDataCollection writes (local order, no + * orientation), for the negative control. The arrays are edited in place, so everything else + * the writer stored, including each group's local_group_index, is unchanged. + */ +void RewriteSharedEntitiesInLocalOrder(const mfem::ParMesh& mesh, Group* mesh_group) { + Group* groups = mesh_group->getGroup("adjsets/mesh/groups"); + mfem::Array vertices; + for (int group = 1; group < mesh.GetNGroups(); ++group) { + const std::string name = "g" + std::to_string(mesh.gtopo.GetGroupMasterRank(group)) + "_" + + std::to_string(mesh.gtopo.GetGroupMasterGroup(group)); + Group* shared_group = groups->getGroup(name); + if (mesh.GroupNEdges(group) > 0) { + int* edges = shared_group->getView("edges")->getData(); + for (int i = 0; i < mesh.GroupNEdges(group); ++i) { + int edge = -1; + int orientation = 0; + mesh.GroupEdge(group, i, edge, orientation); + mesh.GetEdgeVertices(edge, vertices); + edges[2 * i] = vertices[0]; + edges[2 * i + 1] = vertices[1]; + } + } + if (mesh.GroupNTriangles(group) > 0) { + int* triangles = shared_group->getView("triangles")->getData(); + for (int i = 0; i < mesh.GroupNTriangles(group); ++i) { + int face = -1; + int orientation = 0; + mesh.GroupTriangle(group, i, face, orientation); + mesh.GetFaceVertices(face, vertices); + for (int k = 0; k < 3; ++k) { + triangles[3 * i + k] = vertices[k]; + } + } + } + if (mesh.GroupNQuadrilaterals(group) > 0) { + int* quads = shared_group->getView("quadrilaterals")->getData(); + for (int i = 0; i < mesh.GroupNQuadrilaterals(group); ++i) { + int face = -1; + int orientation = 0; + mesh.GroupQuadrilateral(group, i, face, orientation); + mesh.GetFaceVertices(face, vertices); + for (int k = 0; k < 4; ++k) { + quads[4 * i + k] = vertices[k]; + } + } + } + } +} + +/** + * @brief Write a mesh through HDF5 and rebuild it, exactly as a checkpoint and restart do + * + * @param mesh Mesh to write + * @param directory Directory for the rank files (created by rank 0) + * @param local_order_control Whether to rewrite the shared entities in local order first + * @return The rebuilt mesh + * + * @details The DataStore used for loading goes out of scope before the rebuilt mesh is used, + * which also checks that the rebuilt mesh does not point into Sidre memory. + */ +std::shared_ptr RoundTrip(const mfem::ParMesh& mesh, + const std::filesystem::path& directory, + const bool local_order_control) { + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) { + std::filesystem::create_directories(directory); + } + MPI_Barrier(MPI_COMM_WORLD); + + const std::string file = (directory / ("rank_" + std::to_string(rank) + ".hdf5")).string(); + { + DataStore store; + Group* mesh_group = store.getRoot()->createGroup("mesh"); + exaconstit::WriteParMeshBlueprint(mesh, mesh_group); + if (local_order_control) { + RewriteSharedEntitiesInLocalOrder(mesh, mesh_group); + } + MFEM_VERIFY(store.getRoot()->save(file, "sidre_hdf5"), "Unable to save " << file); + } + MPI_Barrier(MPI_COMM_WORLD); + + DataStore store; + MFEM_VERIFY(store.getRoot()->load(file, "sidre_hdf5"), "Unable to load " << file); + return exaconstit::ExaSidreParMesh::Create(store.getRoot()->getGroup("mesh"), MPI_COMM_WORLD); +} + +/** + * @brief Run one round-trip case + * + * @param label Case name, used for the scratch directory and the summary line + * @param type Element type of the serial mesh + * @param n Elements per cube edge of the serial mesh + * @param ref_par Number of uniform parallel refinements after partitioning + * @param leave_ranks_empty Whether to partition onto only half of the ranks (at least one), so + * the remaining ranks own no elements + * + * @details Collective. See the comment at the top of this file for what is checked. + */ +void RunRoundTripCase(const std::string& label, + const mfem::Element::Type type, + const int n, + const int ref_par, + const bool leave_ranks_empty) { + int rank = 0; + int num_ranks = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + + mfem::Mesh serial = mfem::Mesh::MakeCartesian3D(n, n, n, type, 1.0, 1.0, 1.0, false); + PerturbVertices(serial, 0.04 / n); + + // With leave_ranks_empty, elements are dealt out round-robin over the first half of the + // ranks only, so the other half own nothing. MFEM accepts a partitioning that leaves ranks + // empty; this is the situation the recent zero-element fixes in the solver guard against. + std::vector partitioning; + if (leave_ranks_empty) { + const int active_ranks = std::max(1, num_ranks / 2); + partitioning.resize(static_cast(serial.GetNE())); + for (int element = 0; element < serial.GetNE(); ++element) { + partitioning[static_cast(element)] = element % active_ranks; + } + } + mfem::ParMesh original( + MPI_COMM_WORLD, serial, leave_ranks_empty ? partitioning.data() : nullptr); + for (int refinement = 0; refinement < ref_par; ++refinement) { + original.UniformRefinement(); + } + + // The rank count is part of the directory name so tests with different rank counts can + // run at the same time. + const std::filesystem::path directory = std::filesystem::current_path() / + ("restart_roundtrip_np" + std::to_string(num_ranks) + + "_" + label); + const auto rebuilt = RoundTrip(original, directory, false); + + // Topology and parallel structure. + EXPECT_TRUE( + exaconstit::CountMeshTopology(original).Matches(exaconstit::CountMeshTopology(*rebuilt))) + << "local topology counts"; + EXPECT_EQ(original.GetGlobalNE(), rebuilt->GetGlobalNE()) << "global element count"; + EXPECT_EQ(original.bdr_attributes.Size(), rebuilt->bdr_attributes.Size()) + << "boundary attribute list"; + ExpectSameSharedStructures(original, *rebuilt); + + // H1 spaces of order 1 to 4 on both meshes must be indistinguishable. + mfem::VectorFunctionCoefficient coefficient(3, TestField); + double worst_inconsistency = 0.0; + for (int order = 1; order <= 4; ++order) { + const std::string context = label + ", order " + std::to_string(order); + mfem::H1_FECollection fec(order, 3); + mfem::ParFiniteElementSpace space_a(&original, &fec, 3); + mfem::ParFiniteElementSpace space_b(rebuilt.get(), &fec, 3); + EXPECT_EQ(space_a.GetVSize(), space_b.GetVSize()) << context; + EXPECT_EQ(space_a.TrueVSize(), space_b.TrueVSize()) << context; + EXPECT_EQ(space_a.GlobalTrueVSize(), space_b.GlobalTrueVSize()) << context; + + mfem::ParGridFunction field_a(&space_a); + mfem::ParGridFunction field_b(&space_b); + field_a.ProjectCoefficient(coefficient); + field_b.ProjectCoefficient(coefficient); + double local_difference = 0.0; + for (int i = 0; i < std::min(field_a.Size(), field_b.Size()); ++i) { + local_difference = std::max(local_difference, std::abs(field_a(i) - field_b(i))); + } + EXPECT_EQ(local_difference, 0.0) << "local DOF vectors differ, " << context; + EXPECT_EQ(ElementLocalFieldHash(field_a), ElementLocalFieldHash(field_b)) + << "element-local DOF hash differs, " << context; + + mfem::Vector true_a; + mfem::Vector true_b; + field_a.GetTrueDofs(true_a); + field_b.GetTrueDofs(true_b); + double true_difference = 0.0; + for (int i = 0; i < std::min(true_a.Size(), true_b.Size()); ++i) { + true_difference = std::max(true_difference, std::abs(true_a(i) - true_b(i))); + } + EXPECT_EQ(true_difference, 0.0) << "true DOF vectors differ, " << context; + + const double inconsistency = GlobalMax(SharedDofInconsistency(field_b)); + worst_inconsistency = std::max(worst_inconsistency, inconsistency); + EXPECT_LT(inconsistency, 1e-12) << "shared DOFs matched inconsistently, " << context; + } + + // Negative control (reported, not checked; see the comment at the top of this file). + const auto control = RoundTrip(original, directory.string() + "_control", true); + std::array control_inconsistency{}; + for (int order = 1; order <= 4; ++order) { + mfem::H1_FECollection fec(order, 3); + mfem::ParFiniteElementSpace space_c(control.get(), &fec, 3); + mfem::ParGridFunction field_c(&space_c); + field_c.ProjectCoefficient(coefficient); + control_inconsistency[static_cast(order)] = GlobalMax( + SharedDofInconsistency(field_c)); + } + + // GetGlobalNE() is collective, so it is called on every rank before printing on rank 0. + const long long global_elements = original.GetGlobalNE(); + int local_empty = original.GetNE() == 0 ? 1 : 0; + int empty_ranks = 0; + MPI_Allreduce(&local_empty, &empty_ranks, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + if (rank == 0) { + std::cout << label << ": " << num_ranks << " ranks (" << empty_ranks << " empty), " + << global_elements << " elements, max |x - P R x| = " << worst_inconsistency + << "; negative control at orders 1-4: " << control_inconsistency[1] << ", " + << control_inconsistency[2] << ", " << control_inconsistency[3] << ", " + << control_inconsistency[4] << std::endl; + } +} + +} // namespace + +TEST(restart_mesh_roundtrip, hex) { + RunRoundTripCase("hex", mfem::Element::HEXAHEDRON, 4, 0, false); +} + +TEST(restart_mesh_roundtrip, hex_parallel_refinement) { + RunRoundTripCase("hex_refpar", mfem::Element::HEXAHEDRON, 3, 1, false); +} + +TEST(restart_mesh_roundtrip, tet) { + RunRoundTripCase("tet", mfem::Element::TETRAHEDRON, 3, 0, false); +} + +TEST(restart_mesh_roundtrip, tet_parallel_refinement) { + RunRoundTripCase("tet_refpar", mfem::Element::TETRAHEDRON, 2, 1, false); +} + +TEST(restart_mesh_roundtrip, empty_ranks) { + RunRoundTripCase("empty_ranks", mfem::Element::HEXAHEDRON, 3, 0, true); +} + +int main(int argc, char* argv[]) { + MPI_Init(&argc, &argv); + mfem::Hypre::Init(); + ::testing::InitGoogleTest(&argc, argv); + const int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} diff --git a/test/test_restart_runs.py b/test/test_restart_runs.py new file mode 100644 index 00000000..523d8f8f --- /dev/null +++ b/test/test_restart_runs.py @@ -0,0 +1,1017 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +ExaConstit Restart Testing Framework +Runs real ExaConstit jobs and checks that restarted runs match uninterrupted ones exactly + +Each test case runs a reference job once, uninterrupted, writing checkpoints along the way. +It then checks two kinds of restart against that reference: + +* Chained runs: the job is stopped and restarted from its own checkpoints, one segment at a + time (each segment extends t_final). +* Restarts from the reference's checkpoints: a continuation reads a past checkpoint of the + reference with `[Restart] read_directory` and runs to the end, writing its own checkpoints + elsewhere. Any number of past checkpoints can be checked this way without rerunning the + reference. + +Every checkpoint a restarted job writes is compared with the reference checkpoint of the same +cycle (scripts/restart/compare_restart_checkpoints.cpp), and the volume-average outputs are +compared row by row. Both comparisons are bitwise unless --rtol is given. + +A bitwise match alone cannot tell a real restart from a run that ignored the checkpoint and +recomputed from step 0 (the runs are deterministic), so two more checks guard against that: + +* Each restarted job's log must report restarting from the expected checkpoint, and the job + must run exactly the remaining steps, starting at the step after the checkpoint. +* Negative control (needs h5py; skipped with a warning otherwise): the state variables in a + copy of a reference checkpoint are perturbed, a continuation restarts from it, and its final + checkpoint and outputs must DIFFER from the reference. This proves the restored state is what + the continuation uses, and that the comparisons detect a difference. + +Cases: + cyclic Fully reversed strain cycles along z (0 -> +amp -> -amp -> 0) at a fixed dt. + Chains stop every cycle and every 2 cycles. Cycle ends fall inside a boundary + condition segment, so the boundary-condition replay on restart is exercised. + monotonic Long tension along z at a fixed dt. A chain stops halfway; the reference is also + restarted from its halfway checkpoint; the negative control runs here. + empty_ranks Short tension run on a 2-element mesh, so with more than 2 ranks every rank but + two owns no elements. Covers checkpointing and restarting with empty ranks. + outputs Short tension run with ParaView and VisIt output (not run by default; a short + ctest of its own): checkpoints are written inside the run's output folder, and + a restart (also from an earlier checkpoint) keeps the visualization files written + before it, ending with exactly the files of an uninterrupted run. + +Defaults: 16^3 = 4096 hex elements, 8 MPI ranks, 4 cycles of +/-1% strain (160 steps), 10% +monotonic strain (100 steps), and a 20-step empty-rank run. Run with --help for the options +(mesh size, rank count, checkpoint interval and retention, which reference checkpoints to +restart from). +""" + +import argparse +import random +import re +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, List, Optional, Tuple + +import numpy as np + +SCRIPT_DIR = Path(__file__).resolve().parent + +# The top face is pulled in z at this velocity and every run uses this fixed time step, giving +# 1e-3 strain per step on the unit cube. +TOP_VELOCITY = 0.01 +DT = 0.1 + + +class Failure(Exception): + """Raised when a run fails outright, which stops the whole test""" + + +@dataclass +class MeshSpec: + """Auto-generated mesh used by a test case""" + nxyz: Tuple[int, int, int] + grain_file: str + + @property + def num_elements(self) -> int: + return self.nxyz[0] * self.nxyz[1] * self.nxyz[2] + + +# ============================================================================ +# INPUT FILES +# ============================================================================ + +def find_data_dir(explicit: Optional[str]) -> Path: + """ + Find the directory holding the Voce material input files + + Args: + explicit: Directory given on the command line, or None to search next to this script + + Returns: + Directory containing props_cp_voce.txt + """ + candidates = [Path(explicit)] if explicit else [SCRIPT_DIR, SCRIPT_DIR / "data"] + for candidate in candidates: + if (candidate / "props_cp_voce.txt").exists(): + return candidate + raise Failure(f"cannot find props_cp_voce.txt in {[str(c) for c in candidates]}") + + +def prepare_inputs(workdir: Path, data_dir: Path, meshes: List[MeshSpec], num_grains: int, + seed: int) -> None: + """ + Copy the material inputs into the work directory and write one grain file per mesh + + Args: + workdir: Work directory (created if needed) + data_dir: Directory with the Voce material input files + meshes: Meshes whose grain files to write + num_grains: Number of grains (orientations) to draw from + seed: Seed for the element-to-grain assignment, so every run gets the same grains + """ + workdir.mkdir(parents=True, exist_ok=True) + for name in ("props_cp_voce.txt", "state_cp_voce.txt", "voce_quats.ori"): + shutil.copy(data_dir / name, workdir / name) + for mesh in meshes: + rng = random.Random(seed) + with open(workdir / mesh.grain_file, "w") as f: + for _ in range(mesh.num_elements): + f.write(f"{rng.randint(1, num_grains)}\n") + + +def bc_block(update_steps: List[int], top_velocities: List[float]) -> str: + """ + Build the [BCs] table for a run whose top-face velocity changes at given steps + + The bottom face is fixed in z, the x = 0 and y = 0 faces are symmetry planes, and the top + face moves in z with the given velocity. The legacy changing-BC format is used because it + expresses a per-step velocity schedule directly. + + Args: + update_steps: Steps at which a new velocity takes effect (must start with 1) + top_velocities: Top-face z velocity for each update step + + Returns: + TOML text of the [BCs] table + """ + ids = ", ".join(["[1, 2, 3, 4]"] * len(update_steps)) + comps = ", ".join(["[3, 1, 2, 3]"] * len(update_steps)) + vals = ",\n ".join( + "[" + ", ".join(["0.0"] * 11 + [repr(v)]) + "]" for v in top_velocities) + return f"""[BCs] + changing_ess_bcs = true + update_steps = {list(update_steps)} + essential_ids = [{ids}] + essential_comps = [{comps}] + essential_vals = [{vals}] +""" + + +def restart_block(restart: Dict[str, object]) -> str: + """ + Build the [Restart] table + + Args: + restart: [Restart] keys and values (bool, int, float or string) + + Returns: + TOML text of the [Restart] table + """ + lines = ["[Restart]"] + for key, value in restart.items(): + if isinstance(value, bool): + lines.append(f" {key} = {'true' if value else 'false'}") + elif isinstance(value, (int, float)): + lines.append(f" {key} = {value}") + else: + lines.append(f' {key} = "{value}"') + return "\n".join(lines) + "\n" + + +def write_toml(path: Path, basename: str, mesh: MeshSpec, num_grains: int, bcs: str, + t_final: float, restart: Dict[str, object], rtmodel: str = "CPU", + assembly: str = "FULL", visualize: bool = False) -> None: + """ + Write the options file for one run + + Every run uses the same material (Voce FCC crystal plasticity through ExaCMech), solver + settings and volume-average outputs; only the mesh, boundary conditions, final time and + restart settings differ between runs. + + Args: + path: Options file to write + basename: Output basename; runs sharing it append to the same output files + mesh: Mesh of the test case + num_grains: Number of grains in the orientation file + bcs: [BCs] table from bc_block() + t_final: Final simulation time of this run + restart: [Restart] keys and values + rtmodel: [Solvers] rtmodel (CPU, OPENMP or GPU) + assembly: [Solvers] assembly (FULL, PA or EA) + visualize: Write ParaView and VisIt files at every step + """ + viz = "true" if visualize else "false" + path.write_text(f"""Version = "0.10.0" +basename = "{basename}" +grain_file = "{mesh.grain_file}" + +[[Materials]] + material_name = "voce_fcc" + mech_type = "exacmech" + temperature = 298.0 + [Materials.Properties] + floc = "props_cp_voce.txt" + num_props = 17 + [Materials.State_Vars] + floc = "state_cp_voce.txt" + num_vars = 24 + [Materials.Grain] + ori_state_var_loc = 9 + ori_stride = 4 + ori_type = "quat" + num_grains = {num_grains} + orientation_file = "voce_quats.ori" + [Materials.Model] + crystal_plasticity = true + [Materials.Model.ExaCMech] + shortcut = "evptn_FCC_A" + +{bcs} +[Time] + [Time.Fixed] + dt = {DT} + t_final = {t_final!r} + +[Visualizations] + steps = 1 + visit = {viz} + conduit = false + paraview = {viz} + +[PostProcessing] + [PostProcessing.volume_averages] + enabled = true + stress = true + def_grad = true + euler_strain = true + plastic_work = true + output_frequency = 1 + output_directory = "./results" + +[Solvers] + assembly = "{assembly}" + rtmodel = "{rtmodel}" + [Solvers.NR] + iter = 25 + rel_tol = 5e-5 + abs_tol = 5e-10 + [Solvers.Krylov] + iter = 1000 + rel_tol = 1e-7 + abs_tol = 1e-27 + solver = "CG" + +[Mesh] + type = "auto" + ref_ser = 0 + ref_par = 0 + p_refinement = 1 + [Mesh.Auto] + mxyz = [1.0, 1.0, 1.0] + nxyz = [{mesh.nxyz[0]}, {mesh.nxyz[1]}, {mesh.nxyz[2]}] + +{restart_block(restart)}""") + + +def time_of(steps: int) -> float: + """ + Simulation time after a number of fixed steps + + Args: + steps: Number of completed steps + + Returns: + steps * DT, rounded so it prints and compares cleanly + """ + return round(steps * DT, 10) + + +def checkpoint_root(name: str) -> str: + """ + Where a run writes its checkpoints, relative to the work directory + + Every run here has basename name and [Restart] directory = "ckpt_", which ExaConstit + places inside the run's output folder, results/. + + Args: + name: Run name (its basename) + + Returns: + results//ckpt_ + """ + return f"results/{name}/ckpt_{name}" + + +# ============================================================================ +# RUNNING AND COMPARING +# ============================================================================ + +class Harness: + """Runs ExaConstit jobs and records the outcome of every check""" + + def __init__(self, args: argparse.Namespace, workdir: Path): + """ + Args: + args: Parsed command-line options + workdir: Work directory holding the inputs, checkpoints, outputs and logs + """ + self.args = args + self.workdir = workdir + self.logdir = workdir / "logs" + self.logdir.mkdir(exist_ok=True) + self.failures: List[str] = [] + self.checks = 0 + + def check(self, ok: bool, message: str) -> None: + """ + Record and print the outcome of one check + + Args: + ok: Whether the check passed + message: Description of what was checked and what was found + """ + self.checks += 1 + print(f" [{'PASS' if ok else 'FAIL'}] {message}", flush=True) + if not ok: + self.failures.append(message) + + def run(self, name: str, toml_path: Path) -> None: + """ + Run ExaConstit on an options file, logging its output + + With --np 1 the binary is run directly (no MPI launcher) unless --always-mpiexec is + given, which allows single-rank runs where the launcher cannot be used. + + Args: + name: Run name, used for the log file + toml_path: Options file in the work directory + + Raises: + Failure: If the run exits with a non-zero status + """ + if self.args.np == 1 and not self.args.always_mpiexec: + cmd = [str(self.args.exe), "-opt", toml_path.name] + else: + cmd = self.args.mpiexec.split() + ["-np", str(self.args.np), str(self.args.exe), + "-opt", toml_path.name] + log = self.logdir / f"{name}.log" + start = time.time() + with open(log, "w") as f: + result = subprocess.run(cmd, cwd=self.workdir, stdout=f, stderr=subprocess.STDOUT) + elapsed = time.time() - start + print(f" ran {name} in {elapsed:.1f} s (log: {log.relative_to(self.workdir.parent)})", + flush=True) + if result.returncode != 0: + tail = log.read_text().splitlines()[-30:] + raise Failure(f"run {name} failed with exit code {result.returncode}:\n " + + "\n ".join(tail)) + + def verify_restart_log(self, name: str, read_dir: str, cycle: int, end_step: int) -> None: + """ + Check from a run's log that it really restarted, from the expected checkpoint + + The run must report restarting from read_dir/cycle_ and restoring the state, and + it must run exactly steps cycle+1 .. end_step. A run that ignored the checkpoint and + started from step 0 would otherwise go unnoticed, since it produces the same results. + + Args: + name: Run name (its log is checked) + read_dir: Checkpoint directory the run should have read, relative to the work + directory (a full path in the log ending in it also matches) + cycle: Cycle of the checkpoint it should have restarted from + end_step: Last step it should have run + """ + text = (self.logdir / f"{name}.log").read_text() + # The log prints the directory as ExaConstit resolved it: as given for read_directory, + # the full path for a directory inside the output folder + expected = re.compile(r"Restarting from checkpoint (\S*/)?" + re.escape(read_dir) + + rf"/cycle_{cycle:010d} \(cycle {cycle},") + restarted = bool(expected.search(text)) and "Restored checkpoint state" in text + steps = [int(line.split(":")[1]) for line in text.splitlines() + if line.startswith("Simulation cycle:")] + ran_remaining = steps == list(range(cycle + 1, end_step + 1)) + detail = (f"steps {steps[0]}..{steps[-1]}" if steps else "no steps") + \ + f" (expected {cycle + 1}..{end_step})" + self.check(restarted and ran_remaining, + f"{name}: restarted from {read_dir}/cycle_{cycle:010d} and ran only the " + f"remaining steps: {'yes' if restarted else 'NO restart message'}, {detail}") + + def checkpoint(self, ckpt_root: str, cycle: int) -> Path: + """ + Path of a checkpoint directory + + Args: + ckpt_root: Checkpoint root directory, relative to the work directory + cycle: Completed cycles of the checkpoint + + Returns: + Path of the cycle_ directory + """ + return self.workdir / ckpt_root / f"cycle_{cycle:010d}" + + def compare_checkpoints(self, label: str, ckpt_a: Path, ckpt_b: Path, + expect_differ: bool = False) -> None: + """ + Compare two checkpoints with compare_restart_checkpoints + + Args: + label: Description used in the check message + ckpt_a: First checkpoint directory (normally the reference) + ckpt_b: Second checkpoint directory + expect_differ: If True, the check passes only if the checkpoints differ + """ + for ckpt in (ckpt_a, ckpt_b): + if not (ckpt / "commit.json").exists(): + self.check(False, f"{label}: committed checkpoint " + f"{ckpt.relative_to(self.workdir)} exists") + return + cmd = [str(self.args.compare_tool), str(ckpt_a), str(ckpt_b)] + if self.args.rtol > 0: + cmd += ["--rtol", repr(self.args.rtol)] + result = subprocess.run(cmd, capture_output=True, text=True) + summary = (result.stdout.strip().splitlines() or ["(no output)"])[-1] + if expect_differ: + self.check(result.returncode == 1, f"{label}: must differ: {summary}") + return + self.check(result.returncode == 0, f"{label}: {summary}") + if result.returncode != 0: + for line in result.stdout.strip().splitlines()[:15]: + print(f" {line}") + + def compare_outputs(self, label: str, ref_basename: str, other_basename: str, + after_time: Optional[float] = None, expect_differ: bool = False) -> None: + """ + Compare every volume-average output file of two runs + + The time columns must match exactly, and the values bitwise (or within --rtol). + + Args: + label: Description used in the check message + ref_basename: Basename of the reference run + other_basename: Basename of the run to compare + after_time: If given, reference rows with time <= after_time are skipped (for runs + that restarted at that time and so only wrote the later rows) + expect_differ: If True, the check passes only if the outputs differ + """ + ref_dir = self.workdir / "results" / ref_basename + other_dir = self.workdir / "results" / other_basename + ref_files = sorted(p.name for p in ref_dir.glob("avg_*.txt")) + if not ref_files: + self.check(False, f"{label}: reference outputs exist in {ref_dir}") + return + worst = 0.0 + problems = [] + for name in ref_files: + other = other_dir / name + if not other.exists(): + problems.append(f"{name} missing") + continue + a = np.atleast_2d(np.loadtxt(ref_dir / name, comments="#")) + b = np.atleast_2d(np.loadtxt(other, comments="#")) + if after_time is not None: + a = a[a[:, 0] > after_time + 1e-9] + if a.shape != b.shape: + problems.append(f"{name}: {b.shape[0]} rows vs {a.shape[0]} expected") + continue + if not np.array_equal(a[:, 0], b[:, 0]): + problems.append(f"{name}: time columns differ") + continue + scale = np.maximum(np.maximum(np.abs(a), np.abs(b)), 1e-300) + rel = np.max(np.abs(a - b) / scale) + worst = max(worst, rel) + if (self.args.rtol == 0 and not np.array_equal(a, b)) or rel > self.args.rtol: + problems.append(f"{name}: max rel diff {rel:.3e}") + detail = "; ".join(problems) if problems else \ + f"{len(ref_files)} files, max rel diff {worst:.3e}" + if expect_differ: + self.check(bool(problems), f"{label}: outputs must differ from reference ({detail})") + return + self.check(not problems, f"{label}: outputs vs reference ({detail})") + + + def visualization_files(self, basename: str) -> Dict[str, bytes]: + """ + Every visualization file of a run, by path with the basename replaced by "BASENAME" + + Args: + basename: Basename of the run + + Returns: + {relative path: contents}, with the basename also replaced in the contents (VisIt + root files name their data files) + """ + root = self.workdir / "results" / basename / "visualizations" + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + rel = str(path.relative_to(root)).replace(basename, "BASENAME") + files[rel] = path.read_bytes().replace(basename.encode(), b"BASENAME") + return files + + def compare_visualizations(self, label: str, ref_basename: str, other_basename: str) -> None: + """ + Check that a run has exactly the visualization files of the reference, byte for byte + + Args: + label: Description used in the check message + ref_basename: Basename of the reference run + other_basename: Basename of the run to compare + """ + ref = self.visualization_files(ref_basename) + other = self.visualization_files(other_basename) + missing = sorted(set(ref) - set(other)) + extra = sorted(set(other) - set(ref)) + differ = sorted(k for k in set(ref) & set(other) if ref[k] != other[k]) + problems = [f"{len(v)} {what} (e.g. {v[0]})" for v, what in + ((missing, "missing"), (extra, "extra"), (differ, "differ")) if v] + self.check(bool(ref) and not problems, + f"{label}: visualization files vs reference (" + f"{'; '.join(problems) if problems else f'{len(ref)} files identical'})") + + def paraview_times(self, basename: str) -> Dict[str, List[float]]: + """ + Times listed in each ParaView collection (.pvd) of a run + + Args: + basename: Basename of the run + + Returns: + {collection file relative to the visualizations folder: times in file order} + """ + root = self.workdir / "results" / basename / "visualizations" + return {str(pvd.relative_to(root)): [float(t) for t in + re.findall(r'timestep="([^"]+)"', pvd.read_text())] + for pvd in sorted(root.rglob("*.pvd"))} + + +# ============================================================================ +# TEST CASE BUILDING BLOCKS +# ============================================================================ + +def parse_cycles(spec: Optional[str], available: List[int], default: List[int]) -> List[int]: + """ + Interpret a --*-restart-from option + + Args: + spec: "all", "none", a comma-separated list of cycles, or None for the default + available: Reference checkpoint cycles that can be restarted from + default: Cycles to use when spec is None + + Returns: + Cycles to restart from + """ + if spec == "all": + return list(available) + if spec == "none": + return [] + if spec is None: + return default + return [int(s) for s in spec.split(",") if s] + + +def run_reference(h: Harness, name: str, mesh: MeshSpec, bcs: str, total_steps: int, + interval: int) -> List[int]: + """ + Run a case's uninterrupted reference job, writing checkpoints along the way + + Args: + h: Test harness + name: Reference run name (also its basename and checkpoint directory suffix) + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Number of steps to run + interval: Checkpoint interval in steps + + Returns: + Cycles of the complete checkpoints the reference kept + """ + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": interval, "write_final": True, + "keep_last": h.args.keep_last, "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) + h.run(name, h.workdir / f"{name}.toml") + ref_cycles = sorted(int(p.name.split("_")[1]) + for p in (h.workdir / checkpoint_root(name)).glob("cycle_*") + if (p / "commit.json").exists()) + print(f" reference checkpoints kept: {ref_cycles}") + return ref_cycles + + +def run_chain(h: Harness, ref: str, ref_cycles: List[int], name: str, mesh: MeshSpec, bcs: str, + stops: List[int]) -> None: + """ + Run a job in segments, each restarting from the previous one, and compare with the reference + + Segment 1 starts fresh; every later segment restarts from the newest checkpoint in the + chain's own checkpoint directory and extends t_final to its stop. Every segment writes a + final checkpoint. All segments share one basename, so their volume-average outputs append + into the same files, which are then compared with the reference's in full. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + name: Chain name (its basename and checkpoint directory suffix) + mesh: Mesh of the case + bcs: [BCs] table + stops: Step at which each segment stops, increasing; the last is the end of the run + """ + for i, stop in enumerate(stops): + segment = f"{name}_seg{i + 1}" + restart = {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{name}"} + if i > 0: + restart["read"] = True + write_toml(h.workdir / f"{segment}.toml", name, mesh, h.args.num_grains, bcs, + time_of(stop), restart, rtmodel=h.args.rtmodel, assembly=h.args.assembly) + h.run(segment, h.workdir / f"{segment}.toml") + if i > 0: + h.verify_restart_log(segment, checkpoint_root(name), stops[i - 1], stop) + for stop in stops: + if stop in ref_cycles: + h.compare_checkpoints(f"{name} cycle {stop}", + h.checkpoint(checkpoint_root(ref), stop), + h.checkpoint(checkpoint_root(name), stop)) + h.compare_outputs(name, ref, name) + + +def run_from_reference(h: Harness, ref: str, ref_cycles: List[int], from_cycles: List[int], + mesh: MeshSpec, bcs: str, total_steps: int) -> None: + """ + Restart from past reference checkpoints and compare each continuation with the reference + + Each continuation reads the reference's checkpoint through [Restart] read_directory and + writes its own checkpoints to a separate directory, so the reference's checkpoints are never + touched. Its final checkpoint is compared with the reference's final one, and its outputs + with the reference rows after the restart time. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + from_cycles: Cycles to restart from + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Last step of the run + """ + for cycle in from_cycles: + if cycle not in ref_cycles: + h.check(False, f"{ref}: requested restart cycle {cycle} is among the kept " + f"checkpoints {ref_cycles}") + continue + name = f"{ref}_from{cycle}" + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "read": True, "read_cycle": cycle, "read_directory": checkpoint_root(ref), + "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) + h.run(name, h.workdir / f"{name}.toml") + h.verify_restart_log(name, checkpoint_root(ref), cycle, total_steps) + h.compare_checkpoints(f"{name} final cycle {total_steps}", + h.checkpoint(checkpoint_root(ref), total_steps), + h.checkpoint(checkpoint_root(name), total_steps)) + h.compare_outputs(name, ref, name, after_time=time_of(cycle)) + + +def run_negative_control(h: Harness, ref: str, ref_cycles: List[int], mesh: MeshSpec, bcs: str, + total_steps: int) -> None: + """ + Restart from a perturbed copy of a reference checkpoint; the result must differ + + The latest reference checkpoint before the end is copied, and every beginning-of-step state + variable on rank 0 is scaled by (1 + 1e-12). The end-of-step partners are stored as copies + of those values, so they are perturbed consistently. A continuation then restarts from the + copy, and its final checkpoint and outputs must differ from the reference. This shows that + the restored state is what the continuation actually uses, and that the comparisons pick up + even a tiny difference. Skipped with a warning if h5py is not available. + + Args: + h: Test harness + ref: Reference run name + ref_cycles: Cycles of the reference's checkpoints + mesh: Mesh of the case + bcs: [BCs] table + total_steps: Last step of the run + """ + try: + import h5py + except ImportError: + print(" WARNING: h5py is not available; skipping the negative control") + return + candidates = [c for c in ref_cycles if c < total_steps] + if not candidates: + return + cycle = candidates[-1] # the latest one keeps this extra run short + name = f"{ref}_perturbed{cycle}" + source = h.workdir / f"ckpt_{name}_src" + shutil.copytree(h.checkpoint(checkpoint_root(ref), cycle), source / f"cycle_{cycle:010d}") + with h5py.File(source / f"cycle_{cycle:010d}" / "rank_000000.hdf5", "r+") as f: + paths = [f"sidre/external/qfuncs/{g}/values" for g in f["sidre/external/qfuncs"] + if g.startswith("state_var_beg")] + if not paths: + h.check(False, f"{name}: found state_var_beg arrays to perturb") + return + for path in paths: + f[path][...] = f[path][...] * (1.0 + 1e-12) + print(f" negative control: perturbed {', '.join(p.split('/')[3] for p in paths)} by 1e-12 " + f"(relative) in a copy of {ref} cycle {cycle}") + write_toml(h.workdir / f"{name}.toml", name, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 0, "write_final": True, "keep_last": 0, + "read": True, "read_cycle": cycle, "read_directory": source.name, + "directory": f"ckpt_{name}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly) + h.run(name, h.workdir / f"{name}.toml") + h.verify_restart_log(name, source.name, cycle, total_steps) + h.compare_checkpoints(f"{name} (negative control) final cycle {total_steps}", + h.checkpoint(checkpoint_root(ref), total_steps), + h.checkpoint(checkpoint_root(name), total_steps), expect_differ=True) + h.compare_outputs(f"{name} (negative control)", ref, name, after_time=time_of(cycle), + expect_differ=True) + + +# ============================================================================ +# TEST CASES +# ============================================================================ + +def run_cyclic(h: Harness, mesh: MeshSpec) -> None: + """ + Cyclic case: fully reversed strain cycles, chained every 1 and 2 cycles + + Each cycle goes 0 -> +amp -> -amp -> 0 in z, with the top-face velocity reversing at the + peaks, so a cycle takes 4 * steps_per_quarter steps. The boundary-condition update steps + fall at the peaks, while the checkpoints fall at cycle ends, in the middle of a segment, so + restarting exercises the boundary-condition replay. + + Args: + h: Test harness + mesh: Mesh of the case + """ + args = h.args + quarter = args.steps_per_quarter + steps_per_cycle = 4 * quarter + total_steps = args.cycles * steps_per_cycle + # +v for the first quarter, then alternating -v / +v for half a cycle each. + update_steps = [1] + velocities = [TOP_VELOCITY] + step = 1 + quarter + sign = -1.0 + while step <= total_steps: + update_steps.append(step) + velocities.append(sign * TOP_VELOCITY) + step += 2 * quarter + sign = -sign + bcs = bc_block(update_steps, velocities) + amplitude = quarter * TOP_VELOCITY * DT + interval = args.cyclic_checkpoint_interval or steps_per_cycle + print(f"\n== cyclic: {args.cycles} cycles of +/-{amplitude * 100:g}% strain, {total_steps} " + f"steps, BC updates at {update_steps}, reference checkpoints every {interval} steps") + + ref = "cyclic_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + + for stride in args.cyclic_chain_strides: + stops = list(range(stride * steps_per_cycle, total_steps + 1, stride * steps_per_cycle)) + if stops[-1] != total_steps: + stops.append(total_steps) + run_chain(h, ref, ref_cycles, f"cyclic_chain{stride}", mesh, bcs, stops) + + restartable = [c for c in ref_cycles if c < total_steps] + from_cycles = parse_cycles(args.cyclic_restart_from, restartable, restartable) + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + + +def run_monotonic(h: Harness, mesh: MeshSpec) -> None: + """ + Monotonic case: long tension in z, chained halfway, plus the negative control + + Args: + h: Test harness + mesh: Mesh of the case + """ + args = h.args + total_steps = args.monotonic_steps + half = total_steps // 2 + bcs = bc_block([1], [TOP_VELOCITY]) + interval = args.monotonic_checkpoint_interval or max(1, total_steps // 10) + print(f"\n== monotonic: {total_steps * TOP_VELOCITY * DT * 100:g}% tensile strain along z, " + f"{total_steps} steps, reference checkpoints every {interval} steps") + + ref = "mono_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + run_chain(h, ref, ref_cycles, "mono_chain_half", mesh, bcs, [half, total_steps]) + + restartable = [c for c in ref_cycles if c < total_steps] + default_from = [half] if half in ref_cycles else \ + [min(restartable, key=lambda c: abs(c - half))] if restartable else [] + from_cycles = parse_cycles(args.monotonic_restart_from, restartable, default_from) + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + if args.negative_control: + run_negative_control(h, ref, ref_cycles, mesh, bcs, total_steps) + + +def run_empty_ranks(h: Harness, mesh: MeshSpec) -> None: + """ + Empty-rank case: a short tension run on a 2-element mesh + + With fewer elements than ranks, MFEM puts element i on rank i, so ranks 0 and 1 own one + element each (and share a face) while every other rank owns nothing. Those ranks must still + write and read their (empty) checkpoint files and take part in every collective step. The + reference is chained halfway and restarted from its halfway checkpoint, like the monotonic + case. + + Args: + h: Test harness + mesh: Mesh of the case (2 x 1 x 1 elements) + """ + args = h.args + total_steps = args.empty_ranks_steps + half = total_steps // 2 + bcs = bc_block([1], [TOP_VELOCITY]) + interval = max(1, total_steps // 4) + empty = max(0, args.np - mesh.num_elements) + print(f"\n== empty_ranks: {mesh.num_elements} elements on {args.np} rank(s) ({empty} without " + f"elements), {total_steps} steps, reference checkpoints every {interval} steps") + + ref = "empty_ref" + ref_cycles = run_reference(h, ref, mesh, bcs, total_steps, interval) + run_chain(h, ref, ref_cycles, "empty_chain_half", mesh, bcs, [half, total_steps]) + from_cycles = [half] if half in ref_cycles else [] + run_from_reference(h, ref, ref_cycles, from_cycles, mesh, bcs, total_steps) + + +def run_outputs(h: Harness, mesh: MeshSpec) -> None: + """ + Output case: where checkpoints go, and visualization files across restarts + + A short tension run with ParaView and VisIt output at every step. The reference runs + uninterrupted, writing checkpoints. A chain stops after step 2 and restarts to the end, then + restarts once more from its checkpoint of step 1 (an earlier one), as when a run is redone + from an older checkpoint. Checks: + - checkpoints are written inside the run's output folder, results//, + and nowhere in the work directory; + - a restart keeps the visualization files written before it: the first segment's step 0 is + not rewritten, and after each restart the visualization files (ParaView .pvd collections, + VTU pieces, VisIt root and data files) are byte for byte those of the uninterrupted run, + with every step listed once in each .pvd file. + + Args: + h: Test harness + mesh: Mesh of the case + """ + total_steps = 4 + stop = 2 + bcs = bc_block([1], [TOP_VELOCITY]) + print(f"\n== outputs: {total_steps} steps with ParaView and VisIt output, chain stopped at " + f"step {stop}, then restarted again from step 1") + + ref = "out_ref" + write_toml(h.workdir / f"{ref}.toml", ref, mesh, h.args.num_grains, bcs, + time_of(total_steps), + {"write": True, "interval": 1, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{ref}"}, + rtmodel=h.args.rtmodel, assembly=h.args.assembly, visualize=True) + h.run(ref, h.workdir / f"{ref}.toml") + inside = h.workdir / checkpoint_root(ref) / f"cycle_{total_steps:010d}" / "commit.json" + h.check(inside.exists() and not (h.workdir / f"ckpt_{ref}").exists(), + f"{ref}: checkpoints are in {checkpoint_root(ref)} (inside the output folder), " + f"not in the work directory") + expected_times = [time_of(step) for step in range(total_steps + 1)] + ref_times = h.paraview_times(ref) + h.check(bool(ref_times) and all(t == expected_times for t in ref_times.values()), + f"{ref}: every .pvd lists steps 0..{total_steps} ({ref_times})") + + chain = "out_chain" + segments = [(f"{chain}_seg1", stop, {}), + (f"{chain}_seg2", total_steps, {"read": True}), + (f"{chain}_seg3", total_steps, {"read": True, "read_cycle": 1})] + step0 = None + for i, (segment, end, read) in enumerate(segments): + restart = {"write": True, "interval": 1, "write_final": True, "keep_last": 0, + "directory": f"ckpt_{chain}", **read} + write_toml(h.workdir / f"{segment}.toml", chain, mesh, h.args.num_grains, bcs, + time_of(end), restart, rtmodel=h.args.rtmodel, assembly=h.args.assembly, + visualize=True) + h.run(segment, h.workdir / f"{segment}.toml") + files = h.visualization_files(chain) + if i == 0: + step0 = {k: v for k, v in files.items() if "Cycle000000" in k or "_000000" in k} + continue + h.verify_restart_log(segment, checkpoint_root(chain), read.get("read_cycle", stop), end) + kept = all(files.get(k) == v for k, v in step0.items()) + h.check(bool(step0) and kept, + f"{segment}: the {len(step0)} step-0 visualization files of the first segment " + f"are still there, unchanged") + times = h.paraview_times(chain) + h.check(bool(times) and all(t == expected_times for t in times.values()), + f"{segment}: every .pvd lists steps 0..{total_steps} once ({times})") + h.compare_visualizations(segment, ref, chain) + if i == 1: + # Volume-average files are only appended to, so they are compared before segment 3 + # computes steps 2..4 again + h.compare_outputs(segment, ref, chain) + + +# ============================================================================ +# MAIN +# ============================================================================ + +def main() -> int: + """ + Parse the options, run the requested cases and report the results + + Returns: + 0 if every check passed, 1 if a check or run failed, 2 for a setup error + """ + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--np", type=int, default=8, help="MPI ranks (default 8)") + p.add_argument("--rtmodel", default="CPU", choices=["CPU", "OPENMP", "GPU"], + help="[Solvers] rtmodel of every run (default CPU)") + p.add_argument("--assembly", default="FULL", choices=["FULL", "PA", "EA"], + help="[Solvers] assembly of every run (default FULL)") + p.add_argument("--mpiexec", default="mpirun", help="MPI launcher (default mpirun)") + p.add_argument("--always-mpiexec", action="store_true", + help="use the launcher even with --np 1 (default: run the binary directly)") + p.add_argument("--exe", default=str(SCRIPT_DIR / ".." / "bin" / "mechanics")) + p.add_argument("--compare-tool", + default=str(SCRIPT_DIR / ".." / "bin" / "compare_restart_checkpoints")) + p.add_argument("--data-dir", default=None, help="directory with props_cp_voce.txt etc.") + p.add_argument("--workdir", default=str(Path.cwd() / "restart_runs")) + p.add_argument("--cases", default="cyclic,monotonic,empty_ranks", + help="comma-separated cases to run (default cyclic,monotonic,empty_ranks)") + p.add_argument("--nxyz", type=int, default=16, + help="cyclic/monotonic: elements per cube edge (default 16)") + p.add_argument("--num-grains", type=int, default=500) + p.add_argument("--seed", type=int, default=12345, help="seed for the element-to-grain map") + p.add_argument("--rtol", type=float, default=0.0, help="0 = bitwise (default)") + p.add_argument("--keep-last", type=int, default=0, + help="[Restart] keep_last for the reference runs (0 = keep every checkpoint)") + p.add_argument("--cycles", type=int, default=4, help="cyclic: number of cycles (default 4)") + p.add_argument("--steps-per-quarter", type=int, default=10, + help="cyclic: steps from 0 to peak strain; 1e-3 strain per step " + "(default 10 = 1%%)") + p.add_argument("--cyclic-chain-strides", default="1,2", + help="cyclic: chained runs stopping every N cycles (default 1,2)") + p.add_argument("--cyclic-checkpoint-interval", type=int, default=0, + help="cyclic: reference checkpoint interval in steps (default: one cycle)") + p.add_argument("--cyclic-restart-from", default=None, + help="cyclic: reference cycles to restart from: list, 'all' or 'none' " + "(default all kept, except the last)") + p.add_argument("--monotonic-steps", type=int, default=100, + help="monotonic: steps; 1e-3 strain per step (default 100 = 10%%)") + p.add_argument("--monotonic-checkpoint-interval", type=int, default=0, + help="monotonic: reference checkpoint interval in steps (default steps/10)") + p.add_argument("--monotonic-restart-from", default=None, + help="monotonic: reference cycles to restart from: list, 'all' or 'none' " + "(default: the halfway checkpoint)") + p.add_argument("--empty-ranks-steps", type=int, default=20, + help="empty_ranks: steps (default 20)") + p.add_argument("--no-negative-control", dest="negative_control", action="store_false", + help="skip the perturbed-checkpoint negative control") + args = p.parse_args() + args.cyclic_chain_strides = [int(s) for s in args.cyclic_chain_strides.split(",") if s] + args.exe = Path(args.exe).resolve() + args.compare_tool = Path(args.compare_tool).resolve() + + for tool in (args.exe, args.compare_tool): + if not tool.exists(): + print(f"ERROR: {tool} not found") + return 2 + + cases: Dict[str, Tuple[Callable[[Harness, MeshSpec], None], MeshSpec]] = { + "cyclic": (run_cyclic, MeshSpec((args.nxyz,) * 3, "grains.txt")), + "monotonic": (run_monotonic, MeshSpec((args.nxyz,) * 3, "grains.txt")), + "empty_ranks": (run_empty_ranks, MeshSpec((2, 1, 1), "grains_empty_ranks.txt")), + "outputs": (run_outputs, MeshSpec((args.nxyz,) * 3, "grains.txt")), + } + requested = [c.strip() for c in args.cases.split(",") if c.strip()] + unknown = [c for c in requested if c not in cases] + if unknown: + print(f"ERROR: unknown case(s) {unknown}; choose from {list(cases)}") + return 2 + + workdir = Path(args.workdir).resolve() + if workdir.exists(): + shutil.rmtree(workdir) + try: + meshes = {cases[c][1].grain_file: cases[c][1] for c in requested} + prepare_inputs(workdir, find_data_dir(args.data_dir), list(meshes.values()), + args.num_grains, args.seed) + h = Harness(args, workdir) + print(f"Restart run tests: {args.np} rank(s), rtmodel {args.rtmodel}, assembly " + f"{args.assembly}, workdir {workdir}") + for case in requested: + run_case, mesh = cases[case] + run_case(h, mesh) + except Failure as e: + print(f"\nERROR: {e}") + return 1 + + print(f"\n{h.checks - len(h.failures)}/{h.checks} checks passed") + for failure in h.failures: + print(f" FAILED: {failure}") + return 0 if not h.failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflows/exaconstit-calibrate/examples/options.toml b/workflows/exaconstit-calibrate/examples/options.toml index 701445a0..7d1b0730 100644 --- a/workflows/exaconstit-calibrate/examples/options.toml +++ b/workflows/exaconstit-calibrate/examples/options.toml @@ -315,13 +315,6 @@ grain_file = "grains.txt" # 3. Fixed (if [Time.Fixed] exists or as default) time_type = "custom" - # ===== Restart Options (Optional) ===== - # Continue from a previous simulation - # Currently ignored but will be coming in a near-future release - # restart = true - # restart_time = 0.5 # Time to restart from - # restart_cycle = 100 # Cycle number to restart from - # ===== Automatic Time Stepping ===== # Adjusts time step based on convergence behavior [Time.Auto] @@ -560,7 +553,6 @@ grain_file = "grains.txt" # [output_directory]/[basename]/ <- Main simulation folder # [output_directory]/[basename]/avg_*.txt <- Volume average files # [output_directory]/[basename]/visualizations/ <- Visualization files - # [output_directory]/[basename]/restart/ <- Restart files (if enabled) # # Example: If output_directory = "./results" and basename = "tensile_test": # ./results/tensile_test/ <- All outputs go here