rcarson3 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026
diff --git a/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake b/cmake/thirdpartylibraries/SetupThirdPartyLibraries.cmake
@@ -9,6 +9,7 @@ set(_tpls
     exacmech
     mfem
     caliper
+    axom
     threads)
 
 foreach(_tpl ${_tpls})
@@ -122,6 +123,43 @@ if(SNLS_USE_RAJA_PORT_SUITE)
     endif()
 endif() # End SNLS_USE_RAJA_PORT_SUITE check
 
+################################
+# Axom (optional)
+################################
+# Axom installs a proper CMake package config (axom-config.cmake under
+# ${AXOM_DIR}/lib/cmake/axom). find_package CONFIG mode picks it up
+# automatically and imports the roll-up `axom` target plus per-component
+# targets (axom::core, axom::sidre, axom::slic, ...). We consume the
+# roll-up target so whatever components Axom was built with come along
+# transitively -- including sidre and its Conduit / HDF5 dependencies,
+# which axom-config.cmake locates from the paths Axom was built against.
+
+if (DEFINED AXOM_DIR)
+    find_package(axom REQUIRED CONFIG
+                 NO_DEFAULT_PATH
+                 PATHS ${AXOM_DIR})
+    if (axom_FOUND)
+        # ---- Workaround for upstream Axom export bug ----
+        # axom::slic's INTERFACE_LINK_LIBRARIES contains a bare 'lumberjack'
+        # entry inherited from BLT's internal target tracking when Axom is
+        # built with AXOM_ENABLE_LUMBERJACK=ON. Lumberjack is not in
+        # AXOM_COMPONENTS_ENABLED (it's a feature folded into slic, not a
+        # component built as its own library), so the reference is dangling.
+        # Without a stub here, every consumer of axom::slic gets -llumberjack
+        # on its link line and the linker fails to find it.
+        if (NOT TARGET lumberjack)
+            add_library(lumberjack INTERFACE IMPORTED)
+        endif()
+        option(ENABLE_AXOM "Enable Axom" ON)
+        message(STATUS "Found Axom: ${AXOM_DIR}")
+    else()
+        message(FATAL_ERROR "Unable to find Axom with given path ${AXOM_DIR}")
+    endif()
+else()
+    message(STATUS "Axom support disabled")
+endif()
+
+
 ################################
 # Caliper
 ################################

diff --git a/developers_guide.md b/developers_guide.md
@@ -67,6 +67,7 @@ For detailed installation instructions, refer to the build scripts in `scripts/i
 **Optional Dependencies:**
 - **ADIOS2**: (MFEM-based) High-performance I/O for visualization
 - **Caliper**: Performance profiling
+- **Axom** (with Sidre, Conduit and HDF5): Checkpoint/restart support, enabled with `-DAXOM_DIR=<axom install>/lib/cmake` (see `doc/restart.md`)
 
 ### Basic Build Process
 ```bash
@@ -218,6 +219,7 @@ The `src/` directory contains the core ExaConstit implementation organized into
 - **`option_solvers.cpp`**: Linear and nonlinear solver settings
 - **`option_time.cpp`**: Time-stepping parameters
 - **`option_post_processing.cpp`**: Post-processing configuration
+- **`option_restart.cpp`**: Checkpoint/restart configuration
 - **`option_enum.cpp`**: Enumeration type conversions
 - **`option_util.hpp`**: Utility functions for option parsing
 
@@ -242,6 +244,8 @@ The `src/` directory contains the core ExaConstit implementation organized into
 #### `sim_state/`
 **Purpose**: Simulation state management and field storage
 - **`simulation_state.hpp/cpp`**: Central state container class
+- **`restart_checkpoint.hpp/cpp`**: Writing checkpoints and restarting from them (requires Axom; see `doc/restart.md`)
+- **`exa_sidre_parmesh.hpp/cpp`**: Writing and rebuilding the parallel mesh for checkpoints
 
 **Manages**:
 - Finite element spaces and mesh data
@@ -594,6 +598,17 @@ cd test/data
 mpirun -np 4 ../../build/mechanics -opt example.toml
 ```
 
+### Long Tests
+Some tests take too long for CI and are only registered when configured with
+`-DEXACONSTIT_ENABLE_LONG_TESTS=ON`. They currently cover the checkpoint/restart code on real
+simulations; anyone changing that code must run them locally and make sure they pass:
+```bash
+ctest -L long --output-on-failure
+```
+GPU builds also register a GPU version of them (`test_restart_runs_gpu`; `ctest -L gpu` runs
+only that one). No test uses more MPI ranks than `EXACONSTIT_TEST_MAX_MPI_TASKS`, which defaults
+to the number of physical cores detected at configure time (Open MPI's default number of slots).
+
 ### Example Workflows
 The `test/data/` directory contains various example cases:
 - **Crystal plasticity simulations**
@@ -615,14 +630,51 @@ The `test/data/` directory contains various example cases:
 1. Create header/source in `models/mechanics_newmodel.hpp/cpp`
 2. Inherit from `ExaModel` base class
 3. Implement required virtual methods
-4. Add configuration parsing support
-5. Update `CMakeLists.txt`
+4. Register every per-point quantity that carries history between steps with `SimulationState::AddQuadratureFunction()` (see [Keeping New State Restartable](#keeping-new-state-restartable))
+5. Add configuration parsing support
+6. Update `CMakeLists.txt`
 
 #### New Boundary Condition Type
 1. Extend `BCManager` class
 2. Add parsing support in `option_boundary_conditions.cpp`
 3. Update documentation and examples
 
+#### Keeping New State Restartable
+Checkpoints (`src/sim_state/restart_checkpoint.*`, described in `doc/restart.md`) save some
+state automatically and some only because it is listed explicitly. Any new feature that
+carries state from one step to the next must fit one of the cases below, or a restarted run
+will silently continue from the wrong state.
+
+**Quadrature point data is saved automatically if it is registered.** Every function in
+SimulationState's quadrature-function registry is written and restored, whichever component
+registered it; the only exception is `tangent_stiffness`, which is recomputed before use.
+- Create history data with `SimulationState::AddQuadratureFunction()`. Data kept in a model's
+  own `mfem::Vector` or `PartialQuadratureFunction` is not saved, and nothing warns about it.
+  Scratch data recomputed every step can stay private.
+- Register it while the objects are being constructed (SimulationState, SystemDriver, the
+  models, PostProcessingDriver). The driver restores the checkpoint after all of them exist;
+  anything registered later keeps its initial value.
+- Register beginning/end-of-step pairs with `AddUpdateVariablePairNames()` as usual; the
+  checkpoint then stores an identical end-of-step function as a reference to its partner.
+- Data computed once in a constructor is fine if it depends only on the reference
+  configuration, the mesh and the options, since a restarted run reproduces those exactly.
+
+**Nodal data is not automatic.** Only the reference and current coordinates and the velocity
+are saved; the other SimulationState grid functions are derived from them after a restart. A
+new grid function or solution vector that carries history (an additional primal variable,
+Lagrange multipliers, and so on) must be added explicitly to `WriteNodalFields()` and to the
+binding in `BindFieldsAndQuadratureFunctions()`, with a check that the rebuilt space has the
+same size. Store true DOFs where possible.
+
+**Only the volume mesh is saved.** Submeshes, surface meshes and other topologies built on top
+of it must be rebuilt in the restarted run from the restored mesh, which is identical to the
+original, local numbering included. Anything that cannot be rebuilt that way needs its own
+checkpoint data.
+
+**Test it.** `test/test_restart_e2e.cpp` compares a restarted SimulationState with the
+original bitwise; add any new state to it, and run the [long tests](#long-tests) for anything
+that changes what a simulation evolves.
+
 ### Performance Considerations
 - **GPU kernels**: Use RAJA for performance portability
 - **Memory management**: Follow MFEM memory patterns

diff --git a/doc/install.md b/doc/install.md
@@ -86,7 +86,11 @@ scripts/install/
 ├── common/
 │   ├── dependency_versions.sh    # Centralized version control
 │   ├── preflight_checks.sh       # Validation and utilities
-│   └── build_functions.sh        # Shared build logic
+│   ├── build_functions.sh        # Loader that sources the files below
+│   ├── build_helpers.sh          # Logging, cloning, build-dir helpers
+│   ├── build_functions_common.sh # BLT, CAMP, RAJA, Umpire, CHAI
+│   ├── build_functions_mfem.sh   # Hypre, METIS, MFEM
+│   └── build_functions_exaconstit.sh # SNLS, ExaCMech, HDF5, Conduit, Axom, ExaConstit
 ├── configs/
 │   ├── cpu_intel_config.sh       # Intel compiler configuration
 │   ├── cpu_mac_config.sh         # macOS configuration
@@ -326,12 +330,23 @@ All dependency versions are centralized in `common/dependency_versions.sh`:
 code ExaConstit/scripts/install/common/dependency_versions.sh
 ```
 ```bash
+# Build infrastructure (shared by every RADIUSS-stack package via BLT_SOURCE_DIR)
+export BLT_VER="v0.7.2"
+
 # Portability libraries
 export CAMP_VER="v2025.12.0"
 export RAJA_VER="v2025.12.2"
 export UMPIRE_VER="v2025.12.0"
 export CHAI_VER="v2025.12.0"
 
+# Standalone SNLS (batch solvers enabled)
+export SNLS_VER="v0.4.4"
+
+# Restart / checkpoint I/O (Axom Sidre and its dependencies)
+export HDF5_VER="hdf5_1.14.6"        # Stay on the 1.14 series
+export CONDUIT_VER="v0.9.7"
+export AXOM_VER="v0.14.0"
+
 # Material models
 export EXACMECH_REPO="https://github.com/LLNL/ExaCMech.git"
 export EXACMECH_BRANCH="v0.4.3"     # Tag or branch to check out
@@ -460,18 +475,31 @@ your_build_directory/
 ├── camp/
 │   ├── build_${BUILD_SUFFIX}/        # Build artifacts
 │   └── install_${BUILD_SUFFIX}/      # Installed library
+├── blt/                               # Shared BLT checkout (no build step)
 ├── RAJA/
 │   ├── build_${BUILD_SUFFIX}/
 │   └── install_${BUILD_SUFFIX}/
-├── Umpire/                            # GPU builds only
+├── Umpire/
+│   ├── build_${BUILD_SUFFIX}/
+│   └── install_${BUILD_SUFFIX}/
+├── CHAI/
 │   ├── build_${BUILD_SUFFIX}/
 │   └── install_${BUILD_SUFFIX}/
-├── CHAI/                              # GPU builds only
+├── SNLS/
 │   ├── build_${BUILD_SUFFIX}/
 │   └── install_${BUILD_SUFFIX}/
 ├── ExaCMech/
 │   ├── build_${BUILD_SUFFIX}/
 │   └── install_${BUILD_SUFFIX}/
+├── hdf5/
+│   ├── build_${BUILD_SUFFIX}/
+│   └── install_${BUILD_SUFFIX}/
+├── conduit/
+│   ├── build_${BUILD_SUFFIX}/
+│   └── install_${BUILD_SUFFIX}/
+├── axom/
+│   ├── build_${BUILD_SUFFIX}/
+│   └── install_${BUILD_SUFFIX}/
 ├── hypre/
 │   ├── build_${BUILD_SUFFIX}/
 │   └── src/hypre_${BUILD_SUFFIX}/    # Installed library
@@ -506,10 +534,8 @@ less my_raja_build  # View the build log
 ### **Disk Space Requirements**
 
 Typical disk space usage:
-- **CPU build**: ~1 GB total
-- **GPU build (CUDA/HIP)**: ~2 GB total
-  - Includes additional Umpire and CHAI libraries
-  - GPU architectures add to binary sizes
+- **CPU build**: a few GB total (HDF5, Conduit, and Axom checkouts make up a large share)
+- **GPU build (CUDA/HIP)**: larger still, since GPU architectures add to binary sizes
 
 ---
 
@@ -747,16 +773,21 @@ If you prefer to build manually or need more control over the build process:
 ### **Prerequisites**
 
 You'll need to manually build all dependencies first:
-1. **CAMP** (v2025.12.0)
-2. **RAJA** (v2025.12.2)
-3. **Umpire** (v2025.12.0) - GPU builds only
-4. **CHAI** (v2025.12.0) - GPU builds only
-5. **ExaCMech** (v0.4.3)
+1. **BLT** (v0.7.2) - shared by every package below via `-DBLT_SOURCE_DIR`
+2. **CAMP** (v2025.12.0)
+3. **RAJA** (v2025.12.2)
+4. **Umpire** (v2025.12.0)
+5. **CHAI** (v2025.12.0)
 6. **Hypre** (v3.1.0)
 7. **METIS** (5.1.0)
 8. **MFEM** (`exaconstit-dev` branch of https://github.com/rcarson3/mfem; v0.10.0 and v0.10.1 were tested with commit `74ded90191cacabfa294b974e7d69d11ea84d364`)
+9. **SNLS** (v0.4.4) - built with `USE_BATCH_SOLVERS=ON`
+10. **ExaCMech** (v0.4.3) - built against the standalone SNLS via `SNLS_DIR`
+11. **HDF5** (1.14.6) - serial, C library only
+12. **Conduit** (v0.9.7) - with MPI and HDF5
+13. **Axom** (v0.14.0) - with Sidre (optional for ExaConstit; enabled when `AXOM_DIR` is given)
 
-See `scripts/install/common/build_functions.sh` for the exact build commands and CMake options.
+See the `scripts/install/common/build_functions_*.sh` files for the exact build commands and CMake options.
 
 ### **Building ExaConstit**
 
@@ -779,11 +810,16 @@ cmake .. \
   -DENABLE_OPENMP=OFF \
   -DENABLE_FORTRAN=OFF \
   -DPYTHON_EXECUTABLE=/usr/bin/python3 \
+  -DBLT_SOURCE_DIR=${BLT_DIR} \
   -DMFEM_DIR=${MFEM_INSTALL_DIR}/lib/cmake/mfem/ \
   -DECMECH_DIR=${EXACMECH_INSTALL_DIR}/ \
-  -DSNLS_DIR=${EXACMECH_INSTALL_DIR}/ \
+  -DSNLS_DIR=${SNLS_INSTALL_DIR}/lib/cmake/snls/ \
+  -DAXOM_DIR=${AXOM_INSTALL_DIR}/lib/cmake/ \
   -DRAJA_DIR=${RAJA_INSTALL_DIR}/lib/cmake/raja/ \
   -Dcamp_DIR=${CAMP_INSTALL_DIR}/lib/cmake/camp/ \
+  -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \
+  -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \
+  -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai \
   -DCMAKE_BUILD_TYPE=Release
 
 # 4. Build
@@ -803,10 +839,7 @@ cmake .. \
   -DCMAKE_CUDA_COMPILER=${CUDA_ROOT}/bin/nvcc \
   -DCMAKE_CUDA_HOST_COMPILER=${HOST_CXX_COMPILER} \
   -DCMAKE_CUDA_ARCHITECTURES=80 \
-  -DENABLE_CUDA=ON \
-  -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \
-  -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \
-  -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai
+  -DENABLE_CUDA=ON
 ```
 
 For HIP builds, add:
@@ -816,10 +849,7 @@ cmake .. \
   -DCMAKE_CXX_COMPILER=${ROCM_ROOT}/bin/amdclang++ \
   -DCMAKE_HIP_COMPILER=${ROCM_ROOT}/bin/amdclang++ \
   -DCMAKE_HIP_ARCHITECTURES=gfx942 \
-  -DENABLE_HIP=ON \
-  -DFMT_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/fmt \
-  -DUMPIRE_DIR=${UMPIRE_INSTALL_DIR}/lib64/cmake/umpire \
-  -DCHAI_DIR=${CHAI_INSTALL_DIR}/lib/cmake/chai
+  -DENABLE_HIP=ON
 ```
 
 ---