backport: bitcoin#28207, bitcoin#28721, bitcoin#28372, bitcoin#28994, bitcoin#27877, bitcoin#29462

doc/multiprocess.md

@@ -25,7 +25,7 @@ DASHD=dash-node test/functional/test_runner.py
 
 The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option).
 
-Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/bitcoin-core/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general.
+Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/bitcoin-core/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general.
 
 ## Usage
 

doc/release-notes-28207.md

@@ -0,0 +1,7 @@
+mempool.dat compatibility
+========================
+
+The `mempool.dat` file created by -persistmempool or the savemempool RPC will
+be written in a new format, which can not be read by previous software
+releases. To allow for a downgrade, a temporary setting `-persistmempoolv1` has
+been added to fall back to the legacy format.

src/Makefile.test.include

@@ -204,6 +204,7 @@ BITCOIN_TESTS =\
   test/sigopcount_tests.cpp \
   test/skiplist_tests.cpp \
   test/sock_tests.cpp \
+  test/span_tests.cpp \
   test/spork_tests.cpp \
   test/streams_tests.cpp \
   test/subsidy_tests.cpp \

src/common/args.cpp

@@ -772,6 +772,13 @@ fs::path ArgsManager::GetConfigFilePath() const
     return *Assert(m_config_path);
 }
 
+void ArgsManager::SetConfigFilePath(fs::path path)
+{
+    LOCK(cs_args);
+    assert(!m_config_path);
+    m_config_path = path;
+}
+
 ChainType ArgsManager::GetChainType() const
 {
     std::variant<ChainType, std::string> arg = GetChainArg();

src/common/args.h

@@ -186,6 +186,7 @@ class ArgsManager
      * Return config file path (read-only)
      */
     fs::path GetConfigFilePath() const;
+    void SetConfigFilePath(fs::path);
     [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false);
 
     /**

src/init.cpp

@@ -643,6 +643,11 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-parbls=<n>", strprintf("Set the number of BLS verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)",
         llmq::MAX_BLSCHECK_THREADS, llmq::DEFAULT_BLSCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-persistmempoolv1",
+                   strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format "
+                             "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)",
+                             DEFAULT_PERSIST_V1_DAT),
+                   ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -addressindex and -spentindex. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "

src/interfaces/chain.h

@@ -398,6 +398,9 @@ class ChainClient
 
     //! Set mock time.
     virtual void setMockTime(int64_t time) = 0;
+
+    //! Mock the scheduler to fast forward in time.
+    virtual void schedulerMockForward(std::chrono::seconds delta_seconds) = 0;
 };
 
 //! Return implementation of Chain interface.

src/interfaces/node.h

@@ -513,8 +513,8 @@ class Node
     //! Unset RPC timer interface.
     virtual void rpcUnsetTimerInterface(RPCTimerInterface* iface) = 0;
 
-    //! Get unspent outputs associated with a transaction.
-    virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0;
+    //! Get unspent output associated with a transaction.
+    virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0;
 
     //! Broadcast transaction.
     virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, bilingual_str& err_string) = 0;

src/interfaces/wallet.h

@@ -197,7 +197,7 @@ class Wallet
         wallet::AddressPurpose* purpose) = 0;
 
     //! Get wallet address list.
-    virtual std::vector<WalletAddress> getAddresses() const = 0;
+    virtual std::vector<WalletAddress> getAddresses() = 0;
 
     //! Get receive requests.
     virtual std::vector<std::string> getAddressReceiveRequests() = 0;

src/kernel/mempool_options.h

@@ -27,6 +27,8 @@ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300};
 static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5};
 /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */
 static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336};
+/** Whether to fall back to legacy V1 serialization when writing mempool.dat */
+static constexpr bool DEFAULT_PERSIST_V1_DAT{false};
 /** Default for -acceptnonstdtxn */
 static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false};
 
@@ -65,6 +67,7 @@ struct MemPoolOptions {
     bool require_standard{true};
     bool address_index_enabled{false};
     bool spent_index_enabled{false};
+    bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT};
     MemPoolLimits limits{};
 };
 } // namespace kernel

src/kernel/mempool_persist.cpp

@@ -8,6 +8,7 @@
 #include <consensus/amount.h>
 #include <logging.h>
 #include <primitives/transaction.h>
+#include <random.h>
 #include <serialize.h>
 #include <shutdown.h>
 #include <streams.h>
@@ -34,14 +35,14 @@ using fsbridge::FopenFn;
 
 namespace kernel {
 
-static const uint64_t MEMPOOL_DUMP_VERSION = 1;
+static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1};
+static const uint64_t MEMPOOL_DUMP_VERSION{2};
 
 bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts)
 {
     if (load_path.empty()) return false;
 
-    FILE* filestr{opts.mockable_fopen_function(load_path, "rb")};
-    AutoFile file(filestr);
+    AutoFile file{opts.mockable_fopen_function(load_path, "rb")};
     if (file.IsNull()) {
         LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n");
         return false;
@@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
     try {
         uint64_t version;
         file >> version;
-        if (version != MEMPOOL_DUMP_VERSION) {
+        std::vector<std::byte> xor_key;
+        if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
+            // Leave XOR-key empty
+        } else if (version == MEMPOOL_DUMP_VERSION) {
+            file >> xor_key;
+        } else {
             return false;
         }
+        file.SetXor(xor_key);
         uint64_t total_txns_to_load;
         file >> total_txns_to_load;
         uint64_t txns_tried = 0;
@@ -161,17 +168,22 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
 
     auto mid = SteadyClock::now();
 
-    try {
-        FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")};
-        if (!filestr) {
-            return false;
-        }
-
-        AutoFile file(filestr);
+    AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")};
+    if (file.IsNull()) {
+        return false;
+    }
 
-        uint64_t version = MEMPOOL_DUMP_VERSION;
+    try {
+        const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
         file << version;
 
+        std::vector<std::byte> xor_key(8);
+        if (!pool.m_persist_v1_dat) {
+            FastRandomContext{}.fillrand(xor_key);
+            file << xor_key;
+        }
+        file.SetXor(xor_key);
+
         file << (uint64_t)vinfo.size();
         for (const auto& i : vinfo) {
             file << *(i.tx);

src/node/interfaces.cpp

@@ -1166,10 +1166,12 @@ class NodeImpl : public Node
     std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); }
     void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); }
     void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); }
-    bool getUnspentOutput(const COutPoint& output, Coin& coin) override
+    std::optional<Coin> getUnspentOutput(const COutPoint& output) override
     {
         LOCK(::cs_main);
-        return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin);
+        Coin coin;
+        if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin;
+        return {};
     }
     TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, bilingual_str& err_string) override
     {

src/node/mempool_args.cpp

@@ -93,6 +93,8 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con
         return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString());
     }
 
+    mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat);
+
     ApplyArgsManOptions(argsman, mempool_opts.limits);
 
     mempool_opts.address_index_enabled = argsman.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX);

src/qt/clientfeeds.cpp

@@ -176,10 +176,9 @@ void MasternodeFeed::fetch()
 
     dmn->forEachMN(/*only_valid=*/false, [&](const auto& dmn) {
         CTxDestination collateralDest;
-        Coin coin;
         QString collateralStr = QObject::tr("UNKNOWN");
-        if (m_client_model.node().getUnspentOutput(dmn->getCollateralOutpoint(), coin) &&
-            ExtractDestination(coin.out.scriptPubKey, collateralDest)) {
+        if (auto coin{m_client_model.node().getUnspentOutput(dmn->getCollateralOutpoint())};
+            coin && ExtractDestination(coin->out.scriptPubKey, collateralDest)) {
             collateralStr = QString::fromStdString(EncodeDestination(collateralDest));
         }
         int nNextPayment{0};

src/qt/sharedmncreatedialog.cpp

@@ -2654,7 +2654,7 @@ QString SharedMnCreateDialog::ownContributionError() const
     for (const auto& input : mine->inputs) {
         const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout};
         CTxOut txout{wallet.getCoins({outpoint}).front().txout};
-        if (Coin coin; txout.IsNull() && m_node.getUnspentOutput(outpoint, coin)) txout = coin.out;
+        if (txout.IsNull()) { if (auto coin{m_node.getUnspentOutput(outpoint)}) txout = coin->out; }
         if (!txout.IsNull() && wallet.isSpendable(txout.scriptPubKey)) ours += txout.nValue;
     }
     if (ours == 0) return {};
@@ -3637,8 +3637,7 @@ QString SharedMnCreateDialog::deadReason(const MnShareSession& session) const
     for (const auto& contribution : session.contributions()) {
         for (const auto& input : contribution.inputs) {
             const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout};
-            Coin coin;
-            if (m_node.getUnspentOutput(outpoint, coin)) continue;
+            if (m_node.getUnspentOutput(outpoint)) continue;
             // Absence from the confirmed UTXO set proves nothing on its own:
             // another participant's unconfirmed coin looks exactly the same.
             // Only a coin this wallet tracks and reports spent is proof that
@@ -3677,8 +3676,7 @@ std::optional<CAmount> SharedMnCreateDialog::resolveInputValue(const MnShareSess
             return known.front().txout.nValue;
         }
     }
-    Coin coin;
-    if (m_node.getUnspentOutput(outpoint, coin)) return coin.out.nValue;
+    if (auto coin{m_node.getUnspentOutput(outpoint)}) return coin->out.nValue;
     return std::nullopt;
 }
 
@@ -3713,7 +3711,7 @@ SharedMnOwnFunding SharedMnCreateDialog::ownFunding() const
         for (const auto& input : contribution.inputs) {
             const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout};
             CTxOut txout{wallet.getCoins({outpoint}).front().txout};
-            if (Coin coin; txout.IsNull() && m_node.getUnspentOutput(outpoint, coin)) txout = coin.out;
+            if (txout.IsNull()) { if (auto coin{m_node.getUnspentOutput(outpoint)}) txout = coin->out; }
             if (txout.IsNull()) {
                 if (&contribution == mine) ++funding.unvalued;
                 continue;

src/qt/sharedmndialogs.cpp

@@ -957,8 +957,7 @@ bool SharedMnDialog::txInputsUnspent(const QString& tx_hex, QString& error)
         return false;
     }
     for (const CTxIn& txin : tx.vin) {
-        Coin coin;
-        if (m_node.getUnspentOutput(txin.prevout, coin)) continue;
+        if (m_node.getUnspentOutput(txin.prevout)) continue;
         // Not in the confirmed UTXO set. A fee input funded from a
         // still-unconfirmed output of this wallet (e.g. fresh change selected
         // by the prepare call) is unspent and merely waiting for a

src/qt/transactiondesc.cpp

@@ -354,12 +354,10 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall
         {
             COutPoint prevout = txin.prevout;
 
-            Coin prev;
-            if(node.getUnspentOutput(prevout, prev))
-            {
+            if (auto prev{node.getUnspentOutput(prevout)}) {
                 {
                     strHTML += "<li>";
-                    const CTxOut& txout = prev.out;
+                    const CTxOut& txout = prev->out;
                     CTxDestination address;
                     if (ExtractDestination(txout.scriptPubKey, address))
                     {

src/rpc/node.cpp

@@ -918,6 +918,9 @@ static RPCHelpMan mockscheduler()
     const NodeContext& node_context{EnsureAnyNodeContext(request.context)};
     CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds});
     SyncWithValidationInterfaceQueue();
+    for (const auto& chain_client : node_context.chain_clients) {
+        chain_client->schedulerMockForward(std::chrono::seconds(delta_seconds));
+    }
 
     return UniValue::VNULL;
 },

src/span.h

@@ -223,15 +223,32 @@ class Span
     template <typename O> friend class Span;
 };
 
+// Return result of calling .data() method on type T. This is used to be able to
+// write template deduction guides for the single-parameter Span constructor
+// below that will work if the value that is passed has a .data() method, and if
+// the data method does not return a void pointer.
+//
+// It is important to check for the void type specifically below, so the
+// deduction guides can be used in SFINAE contexts to check whether objects can
+// be converted to spans. If the deduction guides did not explicitly check for
+// void, and an object was passed that returned void* from data (like
+// std::vector<bool>), the template deduction would succeed, but the Span<void>
+// object instantiation would fail, resulting in a hard error, rather than a
+// SFINAE error.
+// https://stackoverflow.com/questions/68759148/sfinae-to-detect-the-explicitness-of-a-ctad-deduction-guide
+// https://stackoverflow.com/questions/16568986/what-happens-when-you-call-data-on-a-stdvectorbool
+template<typename T>
+using DataResult = std::remove_pointer_t<decltype(std::declval<T&>().data())>;
+
 // Deduction guides for Span
 // For the pointer/size based and iterator based constructor:
 template <typename T, typename EndOrSize> Span(T*, EndOrSize) -> Span<T>;
 // For the array constructor:
 template <typename T, std::size_t N> Span(T (&)[N]) -> Span<T>;
 // For the temporaries/rvalue references constructor, only supporting const output.
-template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T>, const std::remove_pointer_t<decltype(std::declval<T&&>().data())>>>;
+template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T> && !std::is_void_v<DataResult<T&&>>, const DataResult<T&&>>>;
 // For (lvalue) references, supporting mutable output.
-template <typename T> Span(T&) -> Span<std::remove_pointer_t<decltype(std::declval<T&>().data())>>;
+template <typename T> Span(T&) -> Span<std::enable_if_t<!std::is_void_v<DataResult<T&>>, DataResult<T&>>>;
 
 /** Pop the last element off a span, and return a reference to that element. */
 template <typename T>

src/streams.h

@@ -198,6 +198,11 @@ class SpanReader
         memcpy(dst.data(), m_data.data(), dst.size());
         m_data = m_data.subspan(dst.size());
     }
+
+    void ignore(size_t n)
+    {
+        m_data = m_data.subspan(n);
+    }
 };
 
 /** Double ended buffer combining vector and stream-like interfaces.
@@ -495,7 +500,7 @@ class AutoFile
 {
 protected:
     std::FILE* m_file;
-    const std::vector<std::byte> m_xor;
+    std::vector<std::byte> m_xor;
 
 public:
     explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {}
@@ -535,6 +540,9 @@ class AutoFile
      */
     bool IsNull() const { return m_file == nullptr; }
 
+    /** Continue with a different XOR key */
+    void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; }
+
     /** Implementation detail, only used internally. */
     std::size_t detail_fread(Span<std::byte> dst);