From abf9d4c84c898ebb06b13ee4ab9aa8b694e04d6b Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Mon, 9 Nov 2026 13:30:45 -0600 Subject: [PATCH 1/6] Merge bitcoin/bitcoin#28207: mempool: Persist with XOR fa6b053b5c964fb35935fa994cb782c0731a56f8 mempool: persist with XOR (MarcoFalke) Pull request description: Currently the `mempool.dat` file stores data received from remote peers as-is. This may be problematic when a program other than Bitcoin Core tries to interpret them by accident. For example, an anti-virus program or other program may scan the file and move it into quarantine, or delete it, or corrupt it. While the local wallet is expected to re-submit any pending transactions, unrelated transactions may be missing from the mempool after a restart. This may cause fee estimates to be off, or may cause block relay to be slower. Fix this, similar to https://github.com/bitcoin/bitcoin/pull/6650, by rolling a random XOR pattern over the dat file when writing or reading it. Obviously this can only protect against programs that accidentally and unintentionally are trying to mess with the dat file. Any program that intentionally wants to mess with the dat file can still trivially do so. ACKs for top commit: achow101: re-ACK fa6b053b5c964fb35935fa994cb782c0731a56f8 glozow: reACK fa6b053b5c964fb35935fa994cb782c0731a56f8 ismaelsadeeq: ACK fa6b053b5c964fb35935fa994cb782c0731a56f8 Tree-SHA512: ded2ce3d81bc944b828263534e3178a1e45a914fe8e024f4a14c6561a73e301820944ecc75dd704b3d4221a7a3a5c0597ccab79546250c1197609ee981fe324e Dash adaptations: - src/kernel/mempool_persist.cpp: opens mempool.dat as AutoFile{fopen(...)} instead of CAutoFile{..., CLIENT_VERSION}, because Dash already uses the version-less AutoFile here - src/kernel/mempool_persist.cpp: file.SetXor(xor_key) goes before Dash's progress-reporting load loop (total_txns_to_load/txns_tried) instead of upstream's simple num countdown loop - src/kernel/mempool_options.h, src/txmempool.{h,cpp}, src/node/mempool_args.cpp: persist_v1_dat / m_persist_v1_dat sit after Dash's address/spent index fields; Dash has no full_rbf - test/functional/mempool_compatibility.py: keeps Dash's old-release version 19030000 instead of upstream's 200100 and takes upstream's new comment on that line --- doc/release-notes-28207.md | 7 +++++ src/init.cpp | 5 ++++ src/kernel/mempool_options.h | 3 ++ src/kernel/mempool_persist.cpp | 36 ++++++++++++++++-------- src/node/mempool_args.cpp | 2 ++ src/streams.h | 5 +++- src/txmempool.cpp | 1 + src/txmempool.h | 1 + test/functional/mempool_compatibility.py | 4 +-- 9 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 doc/release-notes-28207.md diff --git a/doc/release-notes-28207.md b/doc/release-notes-28207.md new file mode 100644 index 000000000000..56b88da16ab2 --- /dev/null +++ b/doc/release-notes-28207.md @@ -0,0 +1,7 @@ +mempool.dat compatibility +======================== + +The `mempool.dat` file created by -persistmempool or the savemempool RPC will +be written in a new format, which can not be read by previous software +releases. To allow for a downgrade, a temporary setting `-persistmempoolv1` has +been added to fall back to the legacy format. diff --git a/src/init.cpp b/src/init.cpp index fc6f22d7617e..e4d8606afa22 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -643,6 +643,11 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-parbls=", strprintf("Set the number of BLS verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)", llmq::MAX_BLSCHECK_THREADS, llmq::DEFAULT_BLSCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-persistmempoolv1", + strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format " + "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)", + DEFAULT_PERSIST_V1_DAT), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-prune=", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -addressindex and -spentindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index f9282f0646a0..24d7c3a3f333 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -27,6 +27,8 @@ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; +/** Whether to fall back to legacy V1 serialization when writing mempool.dat */ +static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false}; @@ -65,6 +67,7 @@ struct MemPoolOptions { bool require_standard{true}; bool address_index_enabled{false}; bool spent_index_enabled{false}; + bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index 2e9d26a87472..2e4875e246c0 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -34,14 +35,14 @@ using fsbridge::FopenFn; namespace kernel { -static const uint64_t MEMPOOL_DUMP_VERSION = 1; +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; - AutoFile file(filestr); + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); return false; @@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - if (version != MEMPOOL_DUMP_VERSION) { + std::vector xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { return false; } + file.SetXor(xor_key); uint64_t total_txns_to_load; file >> total_txns_to_load; uint64_t txns_tried = 0; @@ -161,17 +168,22 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock auto mid = SteadyClock::now(); - try { - FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; - if (!filestr) { - return false; - } - - AutoFile file(filestr); + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } - uint64_t version = MEMPOOL_DUMP_VERSION; + try { + const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; + std::vector xor_key(8); + if (!pool.m_persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { file << *(i.tx); diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index 26b5b895b851..1ae4c69935f2 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -93,6 +93,8 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString()); } + mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); + ApplyArgsManOptions(argsman, mempool_opts.limits); mempool_opts.address_index_enabled = argsman.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); diff --git a/src/streams.h b/src/streams.h index 4fd854b8ef8b..cf0588ea6e0e 100644 --- a/src/streams.h +++ b/src/streams.h @@ -495,7 +495,7 @@ class AutoFile { protected: std::FILE* m_file; - const std::vector m_xor; + std::vector m_xor; public: explicit AutoFile(std::FILE* file, std::vector data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} @@ -535,6 +535,9 @@ class AutoFile */ bool IsNull() const { return m_file == nullptr; } + /** Continue with a different XOR key */ + void SetXor(std::vector data_xor) { m_xor = data_xor; } + /** Implementation detail, only used internally. */ std::size_t detail_fread(Span dst); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 4b1199e69114..77acc4840277 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -405,6 +405,7 @@ CTxMemPool::CTxMemPool(const Options& opts) m_require_standard{opts.require_standard}, m_address_index_enabled{opts.address_index_enabled}, m_spent_index_enabled{opts.spent_index_enabled}, + m_persist_v1_dat{opts.persist_v1_dat}, m_limits{opts.limits} { } diff --git a/src/txmempool.h b/src/txmempool.h index 3fd0fb4e170b..67ca9c46b91b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -463,6 +463,7 @@ class CTxMemPool const bool m_require_standard; const bool m_address_index_enabled; const bool m_spent_index_enabled; + const bool m_persist_v1_dat; const Limits m_limits; diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 0624a58947be..f13e758a88cf 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -27,7 +27,7 @@ def skip_test_if_missing_module(self): def setup_network(self): self.add_nodes(self.num_nodes, versions=[ - 19030000, # Last release with previous mempool format + 19030000, # Last release without unbroadcast serialization and without XOR None, ]) self.extra_args = [ @@ -62,7 +62,7 @@ def run_test(self): old_node_mempool.rename(new_node_mempool) self.log.info("Start new node and verify mempool contains the tx") - self.start_node(1) + self.start_node(1, extra_args=["-persistmempoolv1=1"]) assert old_tx_hash in new_node.getrawmempool() self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") From 42e0abe737405cc7df5be91dd6e3b7739b3aba79 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Mon, 9 Nov 2026 20:02:44 -0600 Subject: [PATCH 2/6] Merge bitcoin/bitcoin#28721: multiprocess compatibility updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3b70f7b6156cb110c47a6e482791cf337bb6ad6d doc: fix broken doc/design/multiprocess.md links after #24352 (Ryan Ofsky) 6d43aad742c7ea28303cf2799528188938e7ce32 span: Make Span template deduction guides work in SFINAE context (Ryan Ofsky) 8062c3bdb9dd3062597ed8299e99151b612d32b7 util: Add ArgsManager SetConfigFilePath method (Ryan Ofsky) 441d00c60f0a67889d23f8556190ff99dde488bc interfaces: Rename CalculateBumpFees methods to be compatible with capn'proto (Ryan Ofsky) 156f49d682ef025fb942c997a6c5475e18eef9cf interfaces: Change getUnspentOutput return type to avoid multiprocess segfault (Ryan Ofsky) 4978754c0058bbdfbcd492f25fa49ef211e11d6e interfaces: Add schedulerMockForward method so mockscheduler RPC can work across processes (Ryan Ofsky) 924327eaf3ada45a603e80aa4a3ab38a0f8c8673 interfaces: Fix const virtual method that breaks multiprocess support (Ryan Ofsky) 82a379eca8251c736b4de6e7a2516582641ce397 streams: Add SpanReader ignore method (Russell Yanofsky) Pull request description: This is a collection of small changes to interfaces and code which were needed as part of multiprocess PR #10102, but have been moved here to make that PR smaller. All of these changes are refactoring changes which do not affect behavior of current code --- This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/issues/28722). ACKs for top commit: achow101: ACK 3b70f7b6156cb110c47a6e482791cf337bb6ad6d naumenkogs: ACK 3b70f7b6156cb110c47a6e482791cf337bb6ad6d maflcko: re-ACK 3b70f7b6156cb110c47a6e482791cf337bb6ad6d 🎆 Tree-SHA512: 2368772b887056ad8a9f84c299cfde76ba45943770e3b5353130580900afa9611302195b899ced7b6e303b11f053ff204cae7c28ff4e12c55562fcc81119ba4c Dash adaptations: - doc/multiprocess.md: libmultiprocess install link changed to .../blob/master/doc/install.md here, because Dash's bitcoin#28978 backport already moved the Installation section into this file - doc/design/multiprocess.md: kept Dash's version (the bitcoin#28978 rewrite), which already has the ../../src/interfaces link fixes; no other part of upstream's changes here applies to it - src/qt/transactiondesc.cpp: uses prev->out but keeps Dash's local variable name txout (upstream calls it vout) - src/qt/clientfeeds.cpp: Dash-only masternode collateral lookup updated to the std::optional getUnspentOutput signature - src/qt/sharedmndialogs.cpp: Dash-only txInputsUnspent updated to the std::optional getUnspentOutput signature - src/qt/sharedmncreatedialog.cpp: four Dash-only shared-masternode call sites updated to the std::optional getUnspentOutput signature - src/wallet/interfaces.cpp: include added next to Dash's