From 7c0e203e252110dee9fa60e5917e801be38d3448 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sat, 31 Oct 2026 02:45:12 +0700 Subject: [PATCH 1/9] fix(rpc): subtract spent output sizes in getblockstats as signed values utxo_size_inc and utxo_size_inc_actual are int64_t, but the size of each spent output is subtracted as a size_t, so the arithmetic happens unsigned and wraps whenever a block spends more UTXO bytes than it creates. The wrapped value converts back to the right negative delta, so the RPC result was correct, but the unsigned-overflow sanitizer reports it, which the pruning test's block spending 50 coinbases triggers under ASan. Cast the subtracted size to the accumulators' signed type. --- src/rpc/blockchain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d602b40ec3ad..a55f3d1ff444 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2368,8 +2368,8 @@ static RPCHelpMan getblockstats() tx_total_in += prevoutput.nValue; size_t prevout_size = GetSerializeSize(prevoutput, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; - utxo_size_inc -= prevout_size; - utxo_size_inc_actual -= prevout_size; + utxo_size_inc -= static_cast(prevout_size); + utxo_size_inc_actual -= static_cast(prevout_size); } CAmount txfee = tx_total_in - tx_total_out; From ec46d16c28bbf9a2959eea12aff9f7bff078b534 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:38:06 +0700 Subject: [PATCH 2/9] feat: add zstd as an optional dependency Block files and potentially some records in evodb [such as snapshots of masternode lists] are easy to be compressed up to 20%. In the future zstd may be used as well to compress mn-list's snapshots in evodb database and other heavy compressable data. Add the depends package (static library, headers and pkg-config file), the configure check, the CPPFLAGS/LDADD wiring and the build docs. The library is optional like ZeroMQ and BerkeleyDB, since it only saves disk space: --with-zstd defaults to auto, --without-zstd and NO_ZSTD=1 for depends turn it off, and ENABLE_ZSTD tells the code whether it is linked. The linux64_tsan CI job builds --without-zstd, so that a clang -Werror break in the no-zstd stubs does not go unnoticed: every commit is meant to build without zstd. The job shares its depends prefix with linux64_multiprocess, so zstd is disabled at configure time, which keeps zstd coverage under multiprocess's clang-tidy and debug checks. Co-authored-by: UdjinM6 --- ci/test/00_setup_env_native_tsan.sh | 2 +- configure.ac | 22 ++++++++++++++++++++++ depends/Makefile | 5 ++++- depends/README.md | 1 + depends/config.site.in | 4 ++++ depends/packages/packages.mk | 2 ++ depends/packages/zstd.mk | 19 +++++++++++++++++++ doc/build-freebsd.md | 2 +- doc/build-netbsd.md | 2 +- doc/build-openbsd.md | 2 +- doc/build-osx.md | 2 +- doc/build-unix.md | 10 +++++++++- doc/dependencies.md | 1 + src/Makefile.am | 10 +++++----- src/Makefile.bench.include | 1 + src/Makefile.qt.include | 2 +- src/Makefile.qttest.include | 2 +- src/Makefile.test.include | 3 ++- 18 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 depends/packages/zstd.mk diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 20c294584445..f138845f492d 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -32,6 +32,6 @@ export DEP_OPTS="CC=clang-19 CXX='clang++-19 -stdlib=libc++'" export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163) export TEST_RUNNER_EXTRA="${TEST_RUNNER_EXTRA} --timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-sanitizers=thread CC=clang-19 CXX=clang++-19" +export BITCOIN_CONFIG="--enable-zmq --without-zstd --with-sanitizers=thread CC=clang-19 CXX=clang++-19" export CPPFLAGS="-DARENA_DEBUG" export PYZMQ=true diff --git a/configure.ac b/configure.ac index 4ce746a94410..b41a8a7af64d 100644 --- a/configure.ac +++ b/configure.ac @@ -149,6 +149,12 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) +AC_ARG_WITH([zstd], + [AS_HELP_STRING([--without-zstd], + [disable zstd compression of block and undo files (default is enabled if libzstd is found)])], + [use_zstd=$withval], + [use_zstd=auto]) + AC_ARG_ENABLE([usdt], [AS_HELP_STRING([--enable-usdt], [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])], @@ -1695,6 +1701,21 @@ AC_CHECK_LIB([gmp], [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing)) CPPFLAGS="$TEMP_CPPFLAGS" LDFLAGS="$TEMP_LDFLAGS" +dnl zstd check +if test "$use_zstd" != "no"; then + PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0], + [AC_DEFINE([ENABLE_ZSTD], [1], [Define this symbol to enable zstd compression]) + use_zstd=yes], + [if test "$use_zstd" = "yes"; then + AC_MSG_ERROR([libzstd version 1.4.0 or greater not found.]) + fi + AC_MSG_WARN([libzstd version 1.4.0 or greater not found, disabling zstd compression]) + use_zstd=no]) +fi +if test "$use_zstd" = "yes" && test "$suppress_external_warnings" != "no"; then + ZSTD_CFLAGS=SUPPRESS_WARNINGS($ZSTD_CFLAGS) +fi + AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) dnl libmultiprocess library check @@ -2103,6 +2124,7 @@ if test $bitcoin_enable_qt != "no"; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" +echo " with zstd = $use_zstd" if test $enable_fuzz = "no"; then echo " with test = $use_tests" else diff --git a/depends/Makefile b/depends/Makefile index 5bcf0847c03b..72e0da0c6e89 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -40,6 +40,7 @@ NO_BDB ?= NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= +NO_ZSTD ?= NO_UPNP ?= NO_USDT ?= NO_NATPMP ?= @@ -170,10 +171,11 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages) natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) +zstd_packages_$(NO_ZSTD) = $(zstd_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) $(zstd_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -245,6 +247,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_qt@|$(NO_QT)|' \ -e 's|@no_qr@|$(NO_QR)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ + -e 's|@no_zstd@|$(NO_ZSTD)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_bdb@|$(NO_BDB)|' \ -e 's|@no_sqlite@|$(NO_SQLITE)|' \ diff --git a/depends/README.md b/depends/README.md index d12a56ccebd3..aef20f9c7011 100644 --- a/depends/README.md +++ b/depends/README.md @@ -82,6 +82,7 @@ The following can be set when running make: `make FOO=bar` - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ +- `NO_ZSTD`: Don't download/build/cache zstd (block files are then stored uncompressed) - `NO_WALLET`: Don't download/build/cache libs needed to enable the wallet - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite diff --git a/depends/config.site.in b/depends/config.site.in index 9f4e8d841a11..0441683c6e72 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -70,6 +70,10 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi +if test -z "$with_zstd" && test -n "@no_zstd@"; then + with_zstd=no +fi + if test -z "$enable_usdt" && test -n "@no_usdt@"; then enable_usdt=no fi diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 59232930eb38..930e4277a40a 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -18,6 +18,8 @@ sqlite_packages=sqlite zmq_packages=zeromq +zstd_packages=zstd + upnp_packages=miniupnpc natpmp_packages=libnatpmp diff --git a/depends/packages/zstd.mk b/depends/packages/zstd.mk new file mode 100644 index 000000000000..0be5db336712 --- /dev/null +++ b/depends/packages/zstd.mk @@ -0,0 +1,19 @@ +package=zstd +$(package)_version=1.5.7 +$(package)_download_path=https://github.com/facebook/zstd/releases/download/v$($(package)_version) +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" AR="$($(package)_ar)" + $(package)_build_opts+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" LDFLAGS="$($(package)_ldflags)" + $(package)_build_opts+=ZSTD_LEGACY_SUPPORT=0 ZSTD_LIB_DEPRECATED=0 ZSTD_LIB_DICTBUILDER=0 +endef + +define $(package)_build_cmds + $(MAKE) -C lib libzstd.a $($(package)_build_opts) +endef + +define $(package)_stage_cmds + $(MAKE) -C lib install-static install-includes install-pc PREFIX=$(host_prefix) DESTDIR=$($(package)_staging_dir) $($(package)_build_opts) +endef diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index d33629b551c8..768b9a06048f 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Free Run the following as root to install the base dependencies for building. ```bash -pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace +pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace zstd ``` diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 86f72e1096e4..e24fd7e49648 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -14,7 +14,7 @@ Install the required dependencies the usual way you [install software on NetBSD] The example commands below use `pkgin`. ```bash -pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp +pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp zstd ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 8b0ee196eb46..8ccc3e7e25e4 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Open Run the following as root to install the base dependencies for building. ```bash -pkg_add bash git gmake gmp libevent libtool boost +pkg_add bash git gmake gmp libevent libtool boost zstd # Select the newest version of the following packages: pkg_add autoconf automake python diff --git a/doc/build-osx.md b/doc/build-osx.md index 87914d09affc..b7590ca99b44 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -47,7 +47,7 @@ To install, run the following from your terminal: See [dependencies.md](dependencies.md) for a complete overview. ``` bash -brew install automake libtool boost gmp pkg-config libevent +brew install automake libtool boost gmp pkg-config libevent zstd ``` ### 4. Clone Dash repository diff --git a/doc/build-unix.md b/doc/build-unix.md index ac5a70783f61..c80fce711c6f 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -64,6 +64,10 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) +zstd compresses block and undo files on disk (see: `--with-zstd`, package `libzstd-dev` +on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files that +contain compressed records. + Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): ```sh @@ -147,6 +151,10 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) +zstd compresses block and undo files on disk (see: `--with-zstd`, package `libzstd-dev` +on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files that +contain compressed records. + Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): ```sh @@ -252,7 +260,7 @@ Setup and Build Example: Arch Linux This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: ```sh -pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite +pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite zstd git clone https://github.com/dashpay/dash.git cd dash/ ./autogen.sh diff --git a/doc/dependencies.md b/doc/dependencies.md index eaf67114fd32..62602df2dca8 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -27,6 +27,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | | libgmp | [link](https://gmplib.org/download/gmp/)[ \* ](#note1) | 6.3.0 | [6.2.0](https://github.com/dashpay/bls-signatures/pull/92) | No | +| [zstd](../depends/packages/zstd.mk) (block file compression) | [link](https://github.com/facebook/zstd/releases) | 1.5.7 | 1.4.0 | No | ### GUI | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/src/Makefile.am b/src/Makefile.am index c728766fb564..5dfb1a74473b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -543,7 +543,7 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) $(ZSTD_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ active/context.cpp \ @@ -1058,7 +1058,7 @@ libbitcoin_common_a_SOURCES = \ # # util # -libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZSTD_CFLAGS) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ bls/bls_batchverifier.h \ @@ -1146,7 +1146,7 @@ bitcoin_bin_ldadd = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) +bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) dashd_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp dashd_CPPFLAGS = $(bitcoin_bin_cppflags) @@ -1267,8 +1267,8 @@ if BUILD_BITCOIN_KERNEL_LIB lib_LTLIBRARIES += $(LIBBITCOINKERNEL) libdashkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) -libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS) -libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS) $(ZSTD_LIBS) +libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(ZSTD_CFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) # libdashkernel requires default symbol visibility, explicitly specify that # here so that things still work even when user configures with diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index a9949c57b682..a872d082feb3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -87,6 +87,7 @@ bench_bench_dash_LDADD = \ $(MINIUPNPC_LIBS) \ $(NATPMP_LIBS) \ $(GMP_LIBS) \ + $(ZSTD_LIBS) \ $(BACKTRACE_LIBS) if ENABLE_ZMQ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 9dc7d11ba75f..0b1831ae0090 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -489,7 +489,7 @@ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(BACKTRACE_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) bitcoin_qt_ldflags = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 4690c07a5ea0..fcf7957e1590 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -90,7 +90,7 @@ endif qt_test_test_dash_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBMEMENV) $(BACKTRACE_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) qt_test_test_dash_qt_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_dash_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a2194169c846..73f39c029934 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -60,6 +60,7 @@ FUZZ_SUITE_LD_COMMON = \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) \ $(GMP_LIBS) \ + $(ZSTD_LIBS) \ $(BACKTRACE_LIBS) if USE_UPNP @@ -288,7 +289,7 @@ test_test_dash_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON $(LIBDASHBLS) $(LIBLEVELDB) $(LIBMEMENV) $(BACKTRACE_LIBS) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) +test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) test_test_dash_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ From 23e5cdab47baff33f09551c0fe84652ecd6886fe Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:38:23 +0700 Subject: [PATCH 3/9] feat: add zstd compress/decompress helpers Thin wrappers around the one-shot libzstd API so callers never include themselves. Decompression handles the first frame of its input one-shot, into a buffer of the content size recorded in the frame header, or of the caller-supplied maximum when the header leaves it out, so frames from a streaming compressor are read as well. The maximum bounds the output either way, which lets the block store reject malformed records without over-allocating, and one-shot decompression keeps its history in the output buffer, so memory stays bounded by that maximum whatever window the frame declares. Builds without zstd keep the same interface: ZstdAvailable() reports the difference, compression returns nothing and decompression always fails, so callers store data uncompressed and report compressed data they cannot read. Co-authored-by: UdjinM6 --- src/Makefile.am | 3 ++ src/test/util_tests.cpp | 49 ++++++++++++++++++++++ src/util/zstd.cpp | 70 +++++++++++++++++++++++++++++++ src/util/zstd.h | 33 +++++++++++++++ test/util/data/non-backported.txt | 1 + 5 files changed, 156 insertions(+) create mode 100644 src/util/zstd.cpp create mode 100644 src/util/zstd.h diff --git a/src/Makefile.am b/src/Makefile.am index 5dfb1a74473b..33f7c8f06354 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -495,6 +495,7 @@ BITCOIN_CORE_H = \ util/vecdeque.h \ util/vector.h \ util/wpipe.h \ + util/zstd.h \ validation.h \ validationinterface.h \ versionbits.h \ @@ -1108,6 +1109,7 @@ libbitcoin_util_a_SOURCES = \ util/time.cpp \ util/tokenpipe.cpp \ util/wpipe.cpp \ + util/zstd.cpp \ $(BITCOIN_CORE_H) # @@ -1421,6 +1423,7 @@ libdashkernel_la_SOURCES = \ util/threadnames.cpp \ util/time.cpp \ util/tokenpipe.cpp \ + util/zstd.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 61bb8c88fe5b..ebb1870493da 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1976,4 +1977,52 @@ BOOST_AUTO_TEST_CASE(padding_test) BOOST_CHECK_EQUAL(PadString("", 1), " "); } +BOOST_AUTO_TEST_CASE(util_zstd_roundtrip) +{ + std::vector data(10000); + for (size_t i{0}; i < data.size(); ++i) data[i] = std::byte(i % 7); + if (!util::ZstdAvailable()) { + BOOST_CHECK(util::ZstdCompress(data, 3).empty()); + BOOST_CHECK(!util::ZstdDecompress(data, data.size())); + BOOST_CHECK_EQUAL(util::ZstdMaxLevel(), 0); + return; + } + + const auto compressed{util::ZstdCompress(data, 3)}; + BOOST_CHECK(!compressed.empty()); + BOOST_CHECK(compressed.size() < data.size()); + + const auto restored{util::ZstdDecompress(compressed, data.size())}; + BOOST_REQUIRE(restored); + BOOST_CHECK(*restored == data); + + const auto empty{util::ZstdCompress({}, 3)}; + BOOST_CHECK(!empty.empty()); + const auto restored_empty{util::ZstdDecompress(empty, 0)}; + BOOST_REQUIRE(restored_empty); + BOOST_CHECK(restored_empty->empty()); + + // content size above the caller's bound, truncated frame, garbage + BOOST_CHECK(!util::ZstdDecompress(compressed, data.size() - 1)); + BOOST_CHECK(!util::ZstdDecompress(Span{compressed}.first(compressed.size() / 2), data.size())); + BOOST_CHECK(!util::ZstdDecompress(data, data.size())); + + // a frame whose header does not record the content size: 128 KiB window, one raw last block + DataStream unsized; + unsized << uint32_t{0xFD2FB528} << uint8_t{0x00} << uint8_t{(17 - 10) << 3}; + const uint32_t block_header{1 | (static_cast(data.size()) << 3)}; + unsized << uint8_t(block_header) << uint8_t(block_header >> 8) << uint8_t(block_header >> 16); + unsized.write(data); + const auto restored_unsized{util::ZstdDecompress(MakeByteSpan(unsized), data.size())}; + BOOST_REQUIRE(restored_unsized); + BOOST_CHECK(*restored_unsized == data); + BOOST_CHECK(!util::ZstdDecompress(MakeByteSpan(unsized), data.size() - 1)); + BOOST_CHECK(!util::ZstdDecompress(MakeByteSpan(unsized).first(unsized.size() - 1), data.size())); + // only the first frame is decompressed, like ZstdFileReader does + unsized << uint32_t{0xdeadbeef}; + BOOST_CHECK(util::ZstdDecompress(MakeByteSpan(unsized), data.size()) == data); + + BOOST_CHECK(util::ZstdMaxLevel() >= 19); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/zstd.cpp b/src/util/zstd.cpp new file mode 100644 index 000000000000..3613eb285509 --- /dev/null +++ b/src/util/zstd.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include + +#ifdef ENABLE_ZSTD +#include +#endif + +namespace util { + +bool ZstdAvailable() +{ +#ifdef ENABLE_ZSTD + return true; +#else + return false; +#endif +} + +#ifdef ENABLE_ZSTD +int ZstdMaxLevel() +{ + return ZSTD_maxCLevel(); +} + +std::vector ZstdCompress(Span in, int level) +{ + std::vector out(ZSTD_compressBound(in.size())); + const size_t written{ZSTD_compress(out.data(), out.size(), in.data(), in.size(), level)}; + if (ZSTD_isError(written)) return {}; + out.resize(written); + return out; +} + +std::optional> ZstdDecompress(Span in, size_t max_out) +{ + const size_t frame_size{ZSTD_findFrameCompressedSize(in.data(), in.size())}; + if (ZSTD_isError(frame_size)) return std::nullopt; + // zstd.h spells these sentinels as 0ULL - 1 and 0ULL - 2, which the unsigned-overflow sanitizer flags unless they are constant-folded + constexpr unsigned long long content_size_unknown{ZSTD_CONTENTSIZE_UNKNOWN}; + constexpr unsigned long long content_size_error{ZSTD_CONTENTSIZE_ERROR}; + const unsigned long long content_size{ZSTD_getFrameContentSize(in.data(), frame_size)}; + if (content_size == content_size_error || (content_size != content_size_unknown && content_size > max_out)) { + return std::nullopt; + } + // one-shot decompression keeps its history in the output buffer, so memory stays bounded by max_out whatever window the frame declares + std::vector out(content_size == content_size_unknown ? max_out : content_size); + const size_t written{ZSTD_decompress(out.data(), out.size(), in.data(), frame_size)}; + if (ZSTD_isError(written)) return std::nullopt; + out.resize(written); + return out; +} + +#else + +int ZstdMaxLevel() { return 0; } + +std::vector ZstdCompress(Span, int) { return {}; } + +std::optional> ZstdDecompress(Span, size_t) { return std::nullopt; } + +#endif // ENABLE_ZSTD + +} // namespace util diff --git a/src/util/zstd.h b/src/util/zstd.h new file mode 100644 index 000000000000..774186a61188 --- /dev/null +++ b/src/util/zstd.h @@ -0,0 +1,33 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ZSTD_H +#define BITCOIN_UTIL_ZSTD_H + +#include + +#include +#include +#include + +namespace util { + +/** Whether this build links libzstd. Without it nothing is compressed and compressed data cannot be read. */ +bool ZstdAvailable(); + +/** Highest compression level accepted by the linked libzstd, 0 without zstd. */ +int ZstdMaxLevel(); + +/** Compress `in` into a single zstd frame. Returns an empty vector on library error. */ +std::vector ZstdCompress(Span in, int level); + +/** + * Decompress the zstd frame at the start of `in`, whether or not its header records the content size. + * Returns nullopt for a malformed or truncated frame, or one that decompresses to more than max_out bytes. + */ +std::optional> ZstdDecompress(Span in, size_t max_out); + +} // namespace util + +#endif // BITCOIN_UTIL_ZSTD_H diff --git a/test/util/data/non-backported.txt b/test/util/data/non-backported.txt index 1c868c6d1400..24212bbd2fe1 100644 --- a/test/util/data/non-backported.txt +++ b/test/util/data/non-backported.txt @@ -106,6 +106,7 @@ src/util/helpers.h src/util/ranges_set.* src/util/std23.h src/util/wpipe.* +src/util/zstd.* src/wallet/bip39* src/wallet/coinjoin.* src/wallet/hdchain.* From cd2f2c61e69dbff9fded3f2f9f37b15d970b12f1 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:42:03 +0700 Subject: [PATCH 4/9] feat: read block records through a decompressing record reader Every reader of a block record deserialized straight from the file at the indexed position and never looked at the 8-byte record header, and txindex seeked to a transaction offset inside the stored bytes. Neither can work once a stored record may be a zstd frame of the block. Add util::ZstdFileReader, a stream over an AutoFile that passes a plain record through and decompresses a flagged one chunk by chunk while the deserializer pulls bytes, so neither the record nor its contents are ever held in memory as a whole. The reader bounds the compressed input by the record size and the decompressed output by a caller-supplied limit, which block readers set to MaxBlockSize(), so a small corrupt frame on disk cannot expand into an arbitrarily large block before ReadBlockFromDisk rejects it. OpenRecordPayload reads the record header, masks the new BLOCK_RECORD_COMPRESSED flag out of the size field and returns such a reader, and RecordHeaderPos maps an indexed payload position back to the header. ReadBlockFromDisk and TxIndex::FindTx keep their shape and read through that reader instead of the file; txindex skips to the transaction offset inside the decompressed stream, so the index format is unchanged and offsets stay relative to the serialized block. The reindex scanner reads the whole record from its buffered file, decompresses it when flagged, capped at MaxBlockSize() as well, and deserializes from memory, since it needs the record as a whole for out-of-order blocks. A build without zstd skips flagged records there, logging each under -debug=reindex and warning once per file with the number of blocks skipped. Nothing writes flagged records yet, so behaviour is unchanged. Co-authored-by: UdjinM6 --- src/index/txindex.cpp | 16 ++++---- src/node/blockstorage.cpp | 28 ++++++++++++- src/node/blockstorage.h | 12 ++++++ src/test/util_tests.cpp | 62 +++++++++++++++++++++++++++++ src/util/zstd.cpp | 84 +++++++++++++++++++++++++++++++++++++++ src/util/zstd.h | 43 ++++++++++++++++++++ src/validation.cpp | 47 ++++++++++++++++------ 7 files changed, 271 insertions(+), 21 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index ae563b5f2c40..67caf35574d1 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -9,8 +9,12 @@ #include #include #include +#include #include +using node::OpenRecordPayload; +using node::RecordHeaderPos; + constexpr uint8_t DB_TXINDEX{'t'}; std::unique_ptr g_txindex; @@ -79,19 +83,17 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - AutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; + AutoFile file{m_chainstate->m_blockman.OpenBlockFile(RecordHeaderPos(postx), true)}; if (file.IsNull()) { LogError("%s: OpenBlockFile failed\n", __func__); return false; } CBlockHeader header; try { - file >> header; - if (FileSeek64(file.Get(), postx.nTxOffset, SEEK_CUR)) { - LogError("%s: fseek(...) failed\n", __func__); - return false; - } - file >> tx; + util::ZstdFileReader block_in{OpenRecordPayload(file)}; + block_in >> header; + block_in.ignore(postx.nTxOffset); + block_in >> tx; } catch (const std::exception& e) { LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what()); return false; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index a944a753ca26..3addba746051 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include @@ -827,12 +829,33 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid return true; } +FlatFilePos RecordHeaderPos(const FlatFilePos& pos) +{ + if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return {}; + return {pos.nFile, pos.nPos - BLOCK_SERIALIZATION_HEADER_SIZE}; +} + +/** Size field of the record the file is positioned at; the file is left at the payload */ +static uint32_t ReadRecordSizeField(AutoFile& filein) +{ + uint32_t size_field; + filein.ignore(CMessageHeader::MESSAGE_START_SIZE); + filein >> size_field; + return size_field; +} + +util::ZstdFileReader OpenRecordPayload(AutoFile& filein) +{ + const uint32_t size_field{ReadRecordSizeField(filein)}; + return {filein, size_field & BLOCK_RECORD_SIZE_MASK, (size_field & BLOCK_RECORD_COMPRESSED) != 0, MaxBlockSize()}; +} + std::optional BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const { block.SetNull(); // Open history file to read - AutoFile filein{OpenBlockFile(pos, true)}; + AutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)}; if (filein.IsNull()) { LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString()); return std::nullopt; @@ -840,7 +863,8 @@ std::optional BlockManager::ReadBlockFromDisk(CBlock& block, const Flat // Read block try { - filein >> block; + util::ZstdFileReader block_in{OpenRecordPayload(filein)}; + block_in >> block; } catch (const std::exception& e) { LogError("ReadBlockFromDisk: Deserialize or I/O error - %s at %s\n", e.what(), pos.ToString()); return std::nullopt; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index f70b1b6731b4..5ec4e6b2ef3e 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -20,6 +20,7 @@ #include #include +class AutoFile; class BlockValidationState; class CBlock; class CBlockUndo; @@ -31,6 +32,9 @@ struct FlatFilePos; namespace Consensus { struct Params; } +namespace util { +class ZstdFileReader; +} namespace node { @@ -43,6 +47,9 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Size of header written by WriteBlockToDisk before a serialized CBlock */ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); +/** Set in the size field of a block record whose payload is one zstd frame of the serialized CBlock */ +static constexpr uint32_t BLOCK_RECORD_COMPRESSED{0x80000000}; +static constexpr uint32_t BLOCK_RECORD_SIZE_MASK{~BLOCK_RECORD_COMPRESSED}; extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; @@ -267,6 +274,11 @@ class BlockManager void CleanupBlockRevFiles() const; }; +/** Position of the 8-byte header of the record whose payload starts at pos, null when pos leaves no room for one */ +FlatFilePos RecordHeaderPos(const FlatFilePos& pos); +/** Read the header of the block record the file is positioned at and return a reader for its payload; a compressed payload is decompressed to at most MaxBlockSize() bytes */ +util::ZstdFileReader OpenRecordPayload(AutoFile& filein); + void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles); } // namespace node diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index ebb1870493da..fec5b9cb5384 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -5,6 +5,7 @@ #include #include // For Hash() #include // For CKey +#include #include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -2023,6 +2025,66 @@ BOOST_AUTO_TEST_CASE(util_zstd_roundtrip) BOOST_CHECK(util::ZstdDecompress(MakeByteSpan(unsized), data.size()) == data); BOOST_CHECK(util::ZstdMaxLevel() >= 19); + + // streaming reader over a file holding the frame followed by other data + const fs::path path{m_args.GetDataDirBase() / "zstd_frame.dat"}; + { + AutoFile file{fsbridge::fopen(path, "wb")}; + file.write(compressed); + file << uint32_t{0xdeadbeef}; + } + { + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true}; + std::vector streamed(data.size()); + for (size_t pos{0}; pos < streamed.size(); pos += 1000) reader.read(Span{streamed}.subspan(pos, 1000)); + BOOST_CHECK(streamed == data); + BOOST_CHECK_THROW(reader.read(Span{streamed}.first(1)), std::ios_base::failure); + uint32_t trailer; + file >> trailer; + BOOST_CHECK_EQUAL(trailer, 0xdeadbeef); + } + { + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true}; + std::vector tail(data.size() - 1234); + reader.ignore(1234); + reader.read(tail); + BOOST_CHECK(std::ranges::equal(tail, Span{data}.subspan(1234))); + } + { + // a frame that decompresses to more than the caller allows + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true, /*max_out=*/data.size() - 1}; + std::vector streamed(data.size()); + BOOST_CHECK_THROW(reader.read(streamed), std::ios_base::failure); + } + { + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true, /*max_out=*/data.size()}; + std::vector streamed(data.size()); + reader.read(streamed); + BOOST_CHECK(streamed == data); + } + { + // the frame cut short by the recorded size + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size() / 2, /*compressed=*/true}; + std::vector streamed(data.size()); + BOOST_CHECK_THROW(reader.read(streamed), std::ios_base::failure); + } + { + // a plain record is passed through + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size() + sizeof(uint32_t), /*compressed=*/false}; + reader.ignore(compressed.size()); + uint32_t trailer; + reader >> trailer; + BOOST_CHECK_EQUAL(trailer, 0xdeadbeef); + // neither reads nor skips may leave the record + BOOST_CHECK_THROW(reader >> trailer, std::ios_base::failure); + BOOST_CHECK_THROW(reader.ignore(1), std::ios_base::failure); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/zstd.cpp b/src/util/zstd.cpp index 3613eb285509..ca42657b6d8b 100644 --- a/src/util/zstd.cpp +++ b/src/util/zstd.cpp @@ -6,8 +6,16 @@ #include #endif +#include #include +#include +#include + +#include +#include +#include + #ifdef ENABLE_ZSTD #include #endif @@ -23,6 +31,29 @@ bool ZstdAvailable() #endif } +void ZstdFileReader::ReadPlain(Span dst) +{ + if (dst.size() > m_remaining) throw std::ios_base::failure("ZstdFileReader::read: end of data"); + m_file.read(dst); + m_remaining -= dst.size(); +} + +void ZstdFileReader::ignore(size_t n) +{ + if (!m_compressed) { + if (n > m_remaining) throw std::ios_base::failure("ZstdFileReader::ignore: end of data"); + if (FileSeek64(m_file.Get(), n, SEEK_CUR) != 0) throw std::ios_base::failure("ZstdFileReader::ignore: fseek failed"); + m_remaining -= n; + return; + } + std::byte buf[1024]; + while (n > 0) { + const size_t now{std::min(n, sizeof(buf))}; + read({buf, now}); + n -= now; + } +} + #ifdef ENABLE_ZSTD int ZstdMaxLevel() { @@ -57,6 +88,50 @@ std::optional> ZstdDecompress(Span in, s return out; } +ZstdFileReader::ZstdFileReader(AutoFile& file, size_t size, bool compressed, size_t max_out) + : m_file{file}, + m_compressed{compressed}, + m_remaining{size}, + m_max_out{max_out}, + m_stream{compressed ? ZSTD_createDStream() : nullptr, [](ZSTD_DCtx_s* stream) { ZSTD_freeDStream(stream); }} +{ + if (!m_compressed) return; + if (!m_stream) throw std::ios_base::failure("ZstdFileReader: ZSTD_createDStream failed"); + m_in.resize(ZSTD_DStreamInSize()); + m_out.resize(ZSTD_DStreamOutSize()); +} + +void ZstdFileReader::read(Span dst) +{ + if (!m_compressed) return ReadPlain(dst); + while (!dst.empty()) { + if (!m_unread.empty()) { + const size_t n{std::min(dst.size(), m_unread.size())}; + std::copy_n(m_unread.begin(), n, dst.begin()); + m_unread = m_unread.subspan(n); + dst = dst.subspan(n); + continue; + } + if (m_frame_done) throw std::ios_base::failure("ZstdFileReader::read: end of data"); + if (m_pending.empty()) { + if (m_remaining == 0) throw std::ios_base::failure("ZstdFileReader::read: truncated frame"); + const Span chunk{Span{m_in}.first(std::min(m_remaining, m_in.size()))}; + m_file.read(chunk); + m_remaining -= chunk.size(); + m_pending = chunk; + } + ZSTD_inBuffer in{m_pending.data(), m_pending.size(), 0}; + ZSTD_outBuffer out{m_out.data(), m_out.size(), 0}; + const size_t ret{ZSTD_decompressStream(m_stream.get(), &out, &in)}; + if (ZSTD_isError(ret)) throw std::ios_base::failure(strprintf("ZstdFileReader::read: %s", ZSTD_getErrorName(ret))); + if (out.pos > m_max_out - m_decompressed) throw std::ios_base::failure("ZstdFileReader::read: record decompresses to more than its limit"); + m_decompressed += out.pos; + m_pending = m_pending.subspan(in.pos); + m_unread = Span{m_out}.first(out.pos); + m_frame_done = ret == 0; + } +} + #else int ZstdMaxLevel() { return 0; } @@ -65,6 +140,15 @@ std::vector ZstdCompress(Span, int) { return {}; } std::optional> ZstdDecompress(Span, size_t) { return std::nullopt; } +ZstdFileReader::ZstdFileReader(AutoFile& file, size_t size, bool compressed, size_t max_out) + : m_file{file}, m_compressed{compressed}, m_remaining{size}, m_max_out{max_out}, m_stream{nullptr, [](ZSTD_DCtx_s*) {}} {} + +void ZstdFileReader::read(Span dst) +{ + if (m_compressed) throw std::ios_base::failure("ZstdFileReader::read: compressed record, but this build has no zstd support"); + ReadPlain(dst); +} + #endif // ENABLE_ZSTD } // namespace util diff --git a/src/util/zstd.h b/src/util/zstd.h index 774186a61188..a4f3603ec7b0 100644 --- a/src/util/zstd.h +++ b/src/util/zstd.h @@ -5,12 +5,18 @@ #ifndef BITCOIN_UTIL_ZSTD_H #define BITCOIN_UTIL_ZSTD_H +#include #include #include +#include +#include #include #include +class AutoFile; +struct ZSTD_DCtx_s; + namespace util { /** Whether this build links libzstd. Without it nothing is compressed and compressed data cannot be read. */ @@ -28,6 +34,43 @@ std::vector ZstdCompress(Span in, int level); */ std::optional> ZstdDecompress(Span in, size_t max_out); +/** + * Reads a record of size bytes from the current position of a file. A compressed record is one + * zstd frame, decompressed chunk by chunk while the caller reads, so neither the frame nor its + * contents are ever held in memory as a whole; a plain record is passed through. Throws + * std::ios_base::failure on a truncated or corrupt frame, when reading past its end, when a + * compressed record decompresses to more than max_out bytes, and for a compressed record in builds + * without zstd support. + */ +class ZstdFileReader +{ + AutoFile& m_file; + const bool m_compressed; + size_t m_remaining; //!< bytes of the record not read from the file yet + [[maybe_unused]] const size_t m_max_out; //!< most bytes a compressed record may decompress to + [[maybe_unused]] size_t m_decompressed{0}; + std::unique_ptr m_stream; + std::vector m_in; + std::vector m_out; + Span m_pending; //!< compressed bytes read from the file, not consumed by zstd yet + Span m_unread; //!< decompressed bytes not returned to the caller yet + [[maybe_unused]] bool m_frame_done{false}; + + void ReadPlain(Span dst); + +public: + ZstdFileReader(AutoFile& file, size_t size, bool compressed, size_t max_out = std::numeric_limits::max()); + void read(Span dst); + void ignore(size_t n); + + template + ZstdFileReader& operator>>(T&& obj) + { + ::Unserialize(*this, obj); + return *this; + } +}; + } // namespace util #endif // BITCOIN_UTIL_ZSTD_H diff --git a/src/validation.cpp b/src/validation.cpp index 9b7c0c4a0905..054a482e606e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,8 @@ using kernel::ComputeUTXOStats; using kernel::Notifications; using fsbridge::FopenFn; +using node::BLOCK_RECORD_COMPRESSED; +using node::BLOCK_RECORD_SIZE_MASK; using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; @@ -5630,6 +5633,7 @@ void ChainstateManager::LoadExternalBlockFile( const CChainParams& params{GetParams()}; int nLoaded = 0; + int skipped_compressed{0}; try { unsigned int nMaxBlockSize = MaxBlockSize(); // This takes over fileIn and calls fclose() on it in the BufferedFile destructor @@ -5644,6 +5648,7 @@ void ChainstateManager::LoadExternalBlockFile( nRewind++; // start one byte further next time, in case of failure blkdat.SetLimit(); // remove former limit unsigned int nSize = 0; + bool compressed{false}; try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; @@ -5654,8 +5659,11 @@ void ChainstateManager::LoadExternalBlockFile( continue; } // read size - blkdat >> nSize; - if (nSize < 80 || nSize > nMaxBlockSize) + uint32_t size_field{0}; + blkdat >> size_field; + compressed = (size_field & BLOCK_RECORD_COMPRESSED) != 0; + nSize = size_field & BLOCK_RECORD_SIZE_MASK; + if ((!compressed && nSize < 80) || nSize > nMaxBlockSize) continue; } catch (const std::exception&) { // no valid block header found; don't complain @@ -5663,18 +5671,32 @@ void ChainstateManager::LoadExternalBlockFile( break; } try { - // read block header + // read block record const uint64_t nBlockPos{blkdat.GetPos()}; if (dbp) dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); + std::vector block_data(nSize); + blkdat.read(block_data); + // Position to the marker before the next block, but it's still possible to rewind to the start of the + // current block (without a disk read). + nRewind = nBlockPos + nSize; + if (compressed) { + auto decompressed{util::ZstdDecompress(block_data, nMaxBlockSize)}; + if (!decompressed) { + if (util::ZstdAvailable()) { + LogPrint(BCLog::REINDEX, "%s: Could not decompress block at file offset 0x%x\n", __func__, nBlockPos); + } else { + LogPrint(BCLog::REINDEX, "%s: Skipping zstd-compressed block at file offset 0x%x, this build has no zstd support\n", __func__, nBlockPos); + ++skipped_compressed; + } + continue; + } + block_data = std::move(*decompressed); + } CBlockHeader header; - blkdat >> header; + SpanReader{SER_DISK, CLIENT_VERSION, MakeUCharSpan(block_data)} >> header; const uint256 hash{header.GetHash()}; - // Skip the rest of this block (this may read from disk into memory); position to the marker before the - // next block, but it's still possible to rewind to the start of the current block (without a disk read). - nRewind = nBlockPos + nSize; - blkdat.SkipTo(nRewind); std::shared_ptr pblock{}; // needs to remain available after the cs_main lock is released to avoid duplicate reads from disk @@ -5693,11 +5715,9 @@ void ChainstateManager::LoadExternalBlockFile( // process in case the block isn't known yet const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { - // This block can be processed immediately; rewind to its start, read and deserialize it. - blkdat.SetPos(nBlockPos); + // This block can be processed immediately; deserialize it. pblock = std::make_shared(); - blkdat >> *pblock; - nRewind = blkdat.GetPos(); + SpanReader{SER_DISK, CLIENT_VERSION, MakeUCharSpan(block_data)} >> *pblock; BlockValidationState state; if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, &hash)) { @@ -5797,6 +5817,9 @@ void ChainstateManager::LoadExternalBlockFile( AbortNode(std::string("System error: ") + e.what()); } LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks(SteadyClock::now() - start)); + if (skipped_compressed > 0) { + LogWarning("Skipped %i zstd-compressed blocks from external file, this build has no zstd support\n", skipped_compressed); + } } void ChainstateManager::CheckBlockIndex() From 052e4b3ea737572f27a95eb1d4819767026bd55a Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:43:56 +0700 Subject: [PATCH 5/9] feat: write new block records zstd-compressed Block files are the bulk of a node's disk usage. zstd level 3 saves 19% per block at ~400 MB/s, and higher levels add nothing per block. Compress each newly written block into one zstd frame and flag the record through the high bit of its size field, so the record stays self-describing for reindex and old records are untouched. Blocks that would not shrink are stored as before, which keeps the stored payload within MaxBlockSize(). The level comes from the new -blockcompression option: 0 stores blocks uncompressed, levels above 5 produce a warning, and the default is set in a later commit. ApplyArgsManOptions reads it once and hands it to the BlockManager through its options like the other node settings, so the block store has no argument dependency and tests pick the level when they construct a BlockManager. During reindex the block file accounting takes the stored record size rather than the serialized size, so pruning usage and the next write offset stay exact for compressed files. A build without zstd stores blocks uncompressed: its default level is 0 and any other value is an init error. It cannot read compressed records, so the block index remembers once any have been written, through a compressedrecords flag set when a compressed record is written or met during reindex, and such a build refuses to start on that index while loading the chainstate, before anything reads block data, naming both ways out: a build with zstd, or -reindex, which skips the compressed records and re-downloads those blocks. Co-authored-by: UdjinM6 --- doc/release-notes-7939.md | 16 +++++ src/init.cpp | 14 +++++ src/kernel/blockmanager_opts.h | 3 + src/node/blockmanager_args.cpp | 1 + src/node/blockstorage.cpp | 51 ++++++++++++++-- src/node/blockstorage.h | 9 ++- src/node/chainstate.cpp | 6 ++ src/test/blockmanager_tests.cpp | 102 ++++++++++++++++++++++++++++++-- src/util/zstd.cpp | 9 +++ src/util/zstd.h | 7 ++- 10 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 doc/release-notes-7939.md diff --git a/doc/release-notes-7939.md b/doc/release-notes-7939.md new file mode 100644 index 000000000000..90ed9460f127 --- /dev/null +++ b/doc/release-notes-7939.md @@ -0,0 +1,16 @@ +Block storage +------------- + +New blocks are written to `blocks/blk*.dat` zstd-compressed, one frame per block, which saves +about a fifth of the space new block files take. Block files written by earlier versions are read +as before and are not rewritten. The new `-blockcompression=` option selects the zstd level +(default 0, stores blocks uncompressed for backwards compatibility). + +zstd is an optional build dependency (`--without-zstd`, `NO_ZSTD=1` for depends). A build without +it stores everything uncompressed, accepts only `-blockcompression=0`, and refuses to start on +block files written with compression until `-reindex` re-downloads those blocks. + +Downgrading to a version without this feature requires a `-reindex` if blockcompression > 0 had ever +been used, since older versions cannot read the compressed records. +`contrib/linearize/linearize-data.py` needs the `zstandard` Python module (or Python 3.14+) to read +block files containing compressed records. diff --git a/src/init.cpp b/src/init.cpp index fc6f22d7617e..a270082c14bc 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -100,6 +100,7 @@ #include #include #include +#include #include #include #include @@ -183,6 +184,7 @@ using node::DEFAULT_PRINTPRIORITY; using node::g_indexes_ready_to_sync; using node::KernelNotifications; using node::LoadChainstate; +using node::MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL; using node::MempoolPath; using node::NodeContext; using node::ShouldPersistMempool; @@ -614,6 +616,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-alertnotify=", "Execute command when an alert is raised (%s in cmd is replaced by message)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif argsman.AddArg("-assumevalid=", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, devnet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), devnetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-blockcompression=", strprintf("zstd compression level for newly written block files, 0 to store blocks uncompressed; builds without zstd support accept 0 only (default: %d)", util::DefaultZstdLevel()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksdir=", "Specify directory to hold blocks subdirectory for *.dat files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-tinyblk", "Use smaller block files for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -1333,6 +1336,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) result = init::SetLoggingLevel(args); if (!result) return InitError(util::ErrorString(result)); + const int64_t block_compression{args.GetIntArg("-blockcompression", util::DefaultZstdLevel())}; + if (!util::ZstdAvailable() && block_compression != 0) { + return InitError(_("-blockcompression requires a build with zstd support.")); + } + if (block_compression < 0 || block_compression > util::ZstdMaxLevel()) { + return InitError(strprintf(_("-blockcompression must be between 0 and %d."), util::ZstdMaxLevel())); + } + if (block_compression > MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL) { + InitWarning(strprintf(_("-blockcompression=%d: levels above %d cost much more CPU per block for no measurable gain."), block_compression, MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL)); + } + nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 83f68cc8d17d..a3deacf36e09 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -6,6 +6,7 @@ #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H #include +#include #include @@ -25,6 +26,8 @@ struct BlockManagerOpts { bool fast_prune{false}; bool tiny_blk{false}; bool stop_after_block_import{DEFAULT_STOPAFTERBLOCKIMPORT}; + //! zstd compression level for newly written block and undo records, 0 to store them uncompressed + int block_compression{util::DefaultZstdLevel()}; const fs::path blocks_dir; }; diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index d6ac6f82182a..ac54c62cdd75 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -41,6 +41,7 @@ std::optional ApplyArgsManOptions(const ArgsManager& args, BlockM if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; if (auto value{args.GetBoolArg("-tinyblk")}) opts.tiny_blk = *value; if (auto value{args.GetBoolArg("-stopafterblockimport")}) opts.stop_after_block_import = *value; + if (auto value{args.GetIntArg("-blockcompression")}) opts.block_compression = *value; return std::nullopt; } diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 3addba746051..bfa6a646afc5 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -25,6 +25,7 @@ #include #include +#include #include namespace node { @@ -345,6 +346,9 @@ bool BlockManager::WriteBlockIndexDB() vBlocks.push_back(*it); m_dirty_blockindex.erase(it++); } + if (m_have_compressed_records && !m_block_tree_db->WriteFlag("compressedrecords", true)) { + return false; + } if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) { return false; } @@ -391,6 +395,7 @@ bool BlockManager::LoadBlockIndexDB() // Check whether we have ever pruned block & undo files m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned); + m_block_tree_db->ReadFlag("compressedrecords", m_have_compressed_records); if (m_have_pruned) { LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); } @@ -765,7 +770,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const +bool BlockManager::WriteBlockToDisk(Span payload, uint32_t size_field, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append AutoFile fileout{OpenBlockFile(pos)}; @@ -775,8 +780,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const } // Write index header - unsigned int nSize = GetSerializeSize(block, CLIENT_VERSION); - fileout << messageStart << nSize; + fileout << messageStart << size_field; // Write block const int64_t fileOutPos{FileTell64(fileout.Get())}; @@ -785,11 +789,27 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const return false; } pos.nPos = fileOutPos; - fileout << block; + fileout.write(payload); return true; } +/** Serialize the block for storage, as a zstd frame when that is smaller. Returns the payload and its size field. */ +static std::pair BuildBlockRecordPayload(const CBlock& block, int level) +{ + DataStream ss; + ss << block; + if (level > 0) { + const std::vector compressed{util::ZstdCompress(MakeByteSpan(ss), level)}; + if (!compressed.empty() && compressed.size() < ss.size()) { + const uint32_t size_field{static_cast(compressed.size()) | BLOCK_RECORD_COMPRESSED}; + return {DataStream{compressed}, size_field}; + } + } + const uint32_t size_field{static_cast(ss.size())}; + return {std::move(ss), size_field}; +} + bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); @@ -896,14 +916,34 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) co return true; } +std::optional BlockManager::StoredBlockSizeField(const FlatFilePos& pos) const +{ + AutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)}; + if (filein.IsNull()) return std::nullopt; + try { + return ReadRecordSizeField(filein); + } catch (const std::exception&) { + return std::nullopt; + } +} + FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; const auto position_known {dbp != nullptr}; + DataStream payload; + uint32_t size_field{0}; if (position_known) { blockPos = *dbp; + // the stored record may be smaller than the serialized block + if (const auto stored{StoredBlockSizeField(blockPos)}) { + nBlockSize = *stored & BLOCK_RECORD_SIZE_MASK; + if (*stored & BLOCK_RECORD_COMPRESSED) m_have_compressed_records = true; + } } else { + std::tie(payload, size_field) = BuildBlockRecordPayload(block, m_opts.block_compression); + nBlockSize = payload.size(); // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. @@ -914,7 +954,8 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, cons return FlatFilePos(); } if (!position_known) { - if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) { + if (size_field & BLOCK_RECORD_COMPRESSED) m_have_compressed_records = true; + if (!WriteBlockToDisk(MakeByteSpan(payload), size_field, blockPos, GetParams().MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 5ec4e6b2ef3e..393c26d2c10d 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -37,6 +37,8 @@ class ZstdFileReader; } namespace node { +/** Higher levels cost several times more CPU per block for no measurable gain */ +static constexpr int MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL{5}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -114,7 +116,9 @@ class BlockManager FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; - bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; + bool WriteBlockToDisk(Span payload, uint32_t size_field, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; + /** Size field of the record already stored at pos */ + std::optional StoredBlockSizeField(const FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const; /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ @@ -248,6 +252,9 @@ class BlockManager /** True if any block files have ever been pruned. */ bool m_have_pruned = false; + /** True if any zstd-compressed block or undo record has ever been written, which a build without zstd cannot read. */ + bool m_have_compressed_records = false; + //! Check whether the block associated with this index entry is pruned or not. bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index ea001791d571..d333263e65e0 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -192,6 +193,11 @@ static ChainstateLoadResult CompleteChainstateInitialization(ChainstateManager& return {ChainstateLoadStatus::FAILURE, _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain")}; } + // Before anything reads block data, e.g. ReplayBlocks after an unclean shutdown + if (chainman.m_blockman.m_have_compressed_records && !util::ZstdAvailable()) { + return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("The block files contain zstd-compressed records, but this build has no zstd support. Use a build with zstd support, or restart with -reindex to re-download the blocks.")}; + } + // At this point blocktree args are consistent with what's on disk. // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 7379f302697a..c8bf6829753b 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -3,19 +3,37 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include +#include #include #include +#include +#include