diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 18c2c2254700..c37eca134960 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -25,7 +25,7 @@ DASHD=dash-node test/functional/test_runner.py The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). -Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/bitcoin-core/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/bitcoin-core/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. ## Usage diff --git a/doc/release-notes-28207.md b/doc/release-notes-28207.md new file mode 100644 index 000000000000..56b88da16ab2 --- /dev/null +++ b/doc/release-notes-28207.md @@ -0,0 +1,7 @@ +mempool.dat compatibility +======================== + +The `mempool.dat` file created by -persistmempool or the savemempool RPC will +be written in a new format, which can not be read by previous software +releases. To allow for a downgrade, a temporary setting `-persistmempoolv1` has +been added to fall back to the legacy format. diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a2194169c846..d3f8ecd8a9ca 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -204,6 +204,7 @@ BITCOIN_TESTS =\ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/sock_tests.cpp \ + test/span_tests.cpp \ test/spork_tests.cpp \ test/streams_tests.cpp \ test/subsidy_tests.cpp \ diff --git a/src/common/args.cpp b/src/common/args.cpp index d5ebb8cd4c70..229ab17173be 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -772,6 +772,13 @@ fs::path ArgsManager::GetConfigFilePath() const return *Assert(m_config_path); } +void ArgsManager::SetConfigFilePath(fs::path path) +{ + LOCK(cs_args); + assert(!m_config_path); + m_config_path = path; +} + ChainType ArgsManager::GetChainType() const { std::variant arg = GetChainArg(); diff --git a/src/common/args.h b/src/common/args.h index 0fbdfe9af687..956039cb17af 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -186,6 +186,7 @@ class ArgsManager * Return config file path (read-only) */ fs::path GetConfigFilePath() const; + void SetConfigFilePath(fs::path); [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); /** diff --git a/src/init.cpp b/src/init.cpp index fc6f22d7617e..e4d8606afa22 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -643,6 +643,11 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-parbls=", strprintf("Set the number of BLS verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)", llmq::MAX_BLSCHECK_THREADS, llmq::DEFAULT_BLSCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-persistmempoolv1", + strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format " + "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)", + DEFAULT_PERSIST_V1_DAT), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-prune=", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -addressindex and -spentindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 47c1199deeb5..46d2d38013cb 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -398,6 +398,9 @@ class ChainClient //! Set mock time. virtual void setMockTime(int64_t time) = 0; + + //! Mock the scheduler to fast forward in time. + virtual void schedulerMockForward(std::chrono::seconds delta_seconds) = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 559746221f13..8ce7ac05b6be 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -513,8 +513,8 @@ class Node //! Unset RPC timer interface. virtual void rpcUnsetTimerInterface(RPCTimerInterface* iface) = 0; - //! Get unspent outputs associated with a transaction. - virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; + //! Get unspent output associated with a transaction. + virtual std::optional getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, bilingual_str& err_string) = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 7f08ba98b510..b5b9331c40e2 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -197,7 +197,7 @@ class Wallet wallet::AddressPurpose* purpose) = 0; //! Get wallet address list. - virtual std::vector getAddresses() const = 0; + virtual std::vector getAddresses() = 0; //! Get receive requests. virtual std::vector getAddressReceiveRequests() = 0; diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index f9282f0646a0..24d7c3a3f333 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -27,6 +27,8 @@ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; +/** Whether to fall back to legacy V1 serialization when writing mempool.dat */ +static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false}; @@ -65,6 +67,7 @@ struct MemPoolOptions { bool require_standard{true}; bool address_index_enabled{false}; bool spent_index_enabled{false}; + bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index 2e9d26a87472..2e4875e246c0 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -34,14 +35,14 @@ using fsbridge::FopenFn; namespace kernel { -static const uint64_t MEMPOOL_DUMP_VERSION = 1; +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; - AutoFile file(filestr); + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); return false; @@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - if (version != MEMPOOL_DUMP_VERSION) { + std::vector xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { return false; } + file.SetXor(xor_key); uint64_t total_txns_to_load; file >> total_txns_to_load; uint64_t txns_tried = 0; @@ -161,17 +168,22 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock auto mid = SteadyClock::now(); - try { - FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; - if (!filestr) { - return false; - } - - AutoFile file(filestr); + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } - uint64_t version = MEMPOOL_DUMP_VERSION; + try { + const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; + std::vector xor_key(8); + if (!pool.m_persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { file << *(i.tx); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 09d9b824642b..29876ddb3b8b 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -1166,10 +1166,12 @@ class NodeImpl : public Node std::vector listRpcCommands() override { return ::tableRPC.listCommands(); } void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } - bool getUnspentOutput(const COutPoint& output, Coin& coin) override + std::optional getUnspentOutput(const COutPoint& output) override { LOCK(::cs_main); - return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); + Coin coin; + if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin; + return {}; } TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, bilingual_str& err_string) override { diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index 26b5b895b851..1ae4c69935f2 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -93,6 +93,8 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString()); } + mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); + ApplyArgsManOptions(argsman, mempool_opts.limits); mempool_opts.address_index_enabled = argsman.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); diff --git a/src/qt/clientfeeds.cpp b/src/qt/clientfeeds.cpp index 3383c67a3b81..83a84aec2a43 100644 --- a/src/qt/clientfeeds.cpp +++ b/src/qt/clientfeeds.cpp @@ -176,10 +176,9 @@ void MasternodeFeed::fetch() dmn->forEachMN(/*only_valid=*/false, [&](const auto& dmn) { CTxDestination collateralDest; - Coin coin; QString collateralStr = QObject::tr("UNKNOWN"); - if (m_client_model.node().getUnspentOutput(dmn->getCollateralOutpoint(), coin) && - ExtractDestination(coin.out.scriptPubKey, collateralDest)) { + if (auto coin{m_client_model.node().getUnspentOutput(dmn->getCollateralOutpoint())}; + coin && ExtractDestination(coin->out.scriptPubKey, collateralDest)) { collateralStr = QString::fromStdString(EncodeDestination(collateralDest)); } int nNextPayment{0}; diff --git a/src/qt/sharedmncreatedialog.cpp b/src/qt/sharedmncreatedialog.cpp index 5588e7236022..595dea02425b 100644 --- a/src/qt/sharedmncreatedialog.cpp +++ b/src/qt/sharedmncreatedialog.cpp @@ -2654,7 +2654,7 @@ QString SharedMnCreateDialog::ownContributionError() const for (const auto& input : mine->inputs) { const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout}; CTxOut txout{wallet.getCoins({outpoint}).front().txout}; - if (Coin coin; txout.IsNull() && m_node.getUnspentOutput(outpoint, coin)) txout = coin.out; + if (txout.IsNull()) { if (auto coin{m_node.getUnspentOutput(outpoint)}) txout = coin->out; } if (!txout.IsNull() && wallet.isSpendable(txout.scriptPubKey)) ours += txout.nValue; } if (ours == 0) return {}; @@ -3637,8 +3637,7 @@ QString SharedMnCreateDialog::deadReason(const MnShareSession& session) const for (const auto& contribution : session.contributions()) { for (const auto& input : contribution.inputs) { const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout}; - Coin coin; - if (m_node.getUnspentOutput(outpoint, coin)) continue; + if (m_node.getUnspentOutput(outpoint)) continue; // Absence from the confirmed UTXO set proves nothing on its own: // another participant's unconfirmed coin looks exactly the same. // Only a coin this wallet tracks and reports spent is proof that @@ -3677,8 +3676,7 @@ std::optional SharedMnCreateDialog::resolveInputValue(const MnShareSess return known.front().txout.nValue; } } - Coin coin; - if (m_node.getUnspentOutput(outpoint, coin)) return coin.out.nValue; + if (auto coin{m_node.getUnspentOutput(outpoint)}) return coin->out.nValue; return std::nullopt; } @@ -3713,7 +3711,7 @@ SharedMnOwnFunding SharedMnCreateDialog::ownFunding() const for (const auto& input : contribution.inputs) { const COutPoint outpoint{uint256S(input.txid.toStdString()), input.vout}; CTxOut txout{wallet.getCoins({outpoint}).front().txout}; - if (Coin coin; txout.IsNull() && m_node.getUnspentOutput(outpoint, coin)) txout = coin.out; + if (txout.IsNull()) { if (auto coin{m_node.getUnspentOutput(outpoint)}) txout = coin->out; } if (txout.IsNull()) { if (&contribution == mine) ++funding.unvalued; continue; diff --git a/src/qt/sharedmndialogs.cpp b/src/qt/sharedmndialogs.cpp index afd83188146a..97a96f1d1940 100644 --- a/src/qt/sharedmndialogs.cpp +++ b/src/qt/sharedmndialogs.cpp @@ -957,8 +957,7 @@ bool SharedMnDialog::txInputsUnspent(const QString& tx_hex, QString& error) return false; } for (const CTxIn& txin : tx.vin) { - Coin coin; - if (m_node.getUnspentOutput(txin.prevout, coin)) continue; + if (m_node.getUnspentOutput(txin.prevout)) continue; // Not in the confirmed UTXO set. A fee input funded from a // still-unconfirmed output of this wallet (e.g. fresh change selected // by the prepare call) is unspent and merely waiting for a diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 9901478c8e33..df7e3970665d 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -354,12 +354,10 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall { COutPoint prevout = txin.prevout; - Coin prev; - if(node.getUnspentOutput(prevout, prev)) - { + if (auto prev{node.getUnspentOutput(prevout)}) { { strHTML += "
  • "; - const CTxOut& txout = prev.out; + const CTxOut& txout = prev->out; CTxDestination address; if (ExtractDestination(txout.scriptPubKey, address)) { diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 0ef365ede78c..13575fd6ba69 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -918,6 +918,9 @@ static RPCHelpMan mockscheduler() const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds}); SyncWithValidationInterfaceQueue(); + for (const auto& chain_client : node_context.chain_clients) { + chain_client->schedulerMockForward(std::chrono::seconds(delta_seconds)); + } return UniValue::VNULL; }, diff --git a/src/span.h b/src/span.h index 0eb909ef9a3e..147b06b41ff9 100644 --- a/src/span.h +++ b/src/span.h @@ -223,15 +223,32 @@ class Span template friend class Span; }; +// Return result of calling .data() method on type T. This is used to be able to +// write template deduction guides for the single-parameter Span constructor +// below that will work if the value that is passed has a .data() method, and if +// the data method does not return a void pointer. +// +// It is important to check for the void type specifically below, so the +// deduction guides can be used in SFINAE contexts to check whether objects can +// be converted to spans. If the deduction guides did not explicitly check for +// void, and an object was passed that returned void* from data (like +// std::vector), the template deduction would succeed, but the Span +// object instantiation would fail, resulting in a hard error, rather than a +// SFINAE error. +// https://stackoverflow.com/questions/68759148/sfinae-to-detect-the-explicitness-of-a-ctad-deduction-guide +// https://stackoverflow.com/questions/16568986/what-happens-when-you-call-data-on-a-stdvectorbool +template +using DataResult = std::remove_pointer_t().data())>; + // Deduction guides for Span // For the pointer/size based and iterator based constructor: template Span(T*, EndOrSize) -> Span; // For the array constructor: template Span(T (&)[N]) -> Span; // For the temporaries/rvalue references constructor, only supporting const output. -template Span(T&&) -> Span, const std::remove_pointer_t().data())>>>; +template Span(T&&) -> Span && !std::is_void_v>, const DataResult>>; // For (lvalue) references, supporting mutable output. -template Span(T&) -> Span().data())>>; +template Span(T&) -> Span>, DataResult>>; /** Pop the last element off a span, and return a reference to that element. */ template diff --git a/src/streams.h b/src/streams.h index 4fd854b8ef8b..58f4e769585e 100644 --- a/src/streams.h +++ b/src/streams.h @@ -198,6 +198,11 @@ class SpanReader memcpy(dst.data(), m_data.data(), dst.size()); m_data = m_data.subspan(dst.size()); } + + void ignore(size_t n) + { + m_data = m_data.subspan(n); + } }; /** Double ended buffer combining vector and stream-like interfaces. @@ -495,7 +500,7 @@ class AutoFile { protected: std::FILE* m_file; - const std::vector m_xor; + std::vector m_xor; public: explicit AutoFile(std::FILE* file, std::vector data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} @@ -535,6 +540,9 @@ class AutoFile */ bool IsNull() const { return m_file == nullptr; } + /** Continue with a different XOR key */ + void SetXor(std::vector data_xor) { m_xor = data_xor; } + /** Implementation detail, only used internally. */ std::size_t detail_fread(Span dst); diff --git a/src/test/span_tests.cpp b/src/test/span_tests.cpp new file mode 100644 index 000000000000..f6cac10b09bc --- /dev/null +++ b/src/test/span_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include + +namespace { +struct Ignore +{ + template Ignore(T&&) {} +}; +template +bool Spannable(T&& value, decltype(Span{value})* enable = nullptr) +{ + return true; +} +bool Spannable(Ignore) +{ + return false; +} + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wunneeded-member-function" +# pragma clang diagnostic ignored "-Wunused-member-function" +#endif +struct SpannableYes +{ + int* data(); + size_t size(); +}; +struct SpannableNo +{ + void* data(); + size_t size(); +}; +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +} // namespace + +BOOST_AUTO_TEST_SUITE(span_tests) + +// Make sure template Span template deduction guides accurately enable calls to +// Span constructor overloads that work, and disable calls to constructor overloads that +// don't work. This makes it is possible to use the Span constructor in a SFINAE +// contexts like in the Spannable function above to detect whether types are or +// aren't compatible with Spans at compile time. +// +// Previously there was a bug where writing a SFINAE check for vector was +// not possible, because in libstdc++ vector has a data() memeber +// returning void*, and the Span template guide ignored the data() return value, +// so the template substitution would succeed, but the constructor would fail, +// resulting in a fatal compile error, rather than a SFINAE error that could be +// handled. +BOOST_AUTO_TEST_CASE(span_constructor_sfinae) +{ + BOOST_CHECK(Spannable(std::vector{})); + BOOST_CHECK(!Spannable(std::set{})); + BOOST_CHECK(!Spannable(std::vector{})); + BOOST_CHECK(Spannable(std::array{})); + BOOST_CHECK(Spannable(Span{})); + BOOST_CHECK(Spannable("char array")); + BOOST_CHECK(Spannable(SpannableYes{})); + BOOST_CHECK(!Spannable(SpannableNo{})); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 4b1199e69114..77acc4840277 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -405,6 +405,7 @@ CTxMemPool::CTxMemPool(const Options& opts) m_require_standard{opts.require_standard}, m_address_index_enabled{opts.address_index_enabled}, m_spent_index_enabled{opts.spent_index_enabled}, + m_persist_v1_dat{opts.persist_v1_dat}, m_limits{opts.limits} { } diff --git a/src/txmempool.h b/src/txmempool.h index 3fd0fb4e170b..67ca9c46b91b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -463,6 +463,7 @@ class CTxMemPool const bool m_require_standard; const bool m_address_index_enabled; const bool m_spent_index_enabled; + const bool m_persist_v1_dat; const Limits m_limits; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 858faf860f9c..0545b4fc590e 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -39,14 +39,30 @@ static int GetSelectionWeight(const SelectionResult& result) }); } -// Descending order comparator +// Sort by descending (effective) value prefer lower waste on tie struct { bool operator()(const OutputGroup& a, const OutputGroup& b) const { + if (a.GetSelectionAmount() == b.GetSelectionAmount()) { + // Lower waste is better when effective_values are tied + return (a.fee - a.long_term_fee) < (b.fee - b.long_term_fee); + } return a.GetSelectionAmount() > b.GetSelectionAmount(); } } descending; +// Sort by descending (effective) value prefer lower weight on tie +struct { + bool operator()(const OutputGroup& a, const OutputGroup& b) const + { + if (a.GetSelectionAmount() == b.GetSelectionAmount()) { + // Sort lower weight to front on tied effective_value + return a.m_weight < b.m_weight; + } + return a.GetSelectionAmount() > b.GetSelectionAmount(); + } +} descending_effval_weight; + /* * This is the Branch and Bound Coin Selection algorithm designed by Murch. It searches for an input * set that can pay for the spending target and does not exceed the spending target by more than the @@ -82,6 +98,7 @@ struct { * bound of the range. * @param const CAmount& cost_of_change This is the cost of creating and spending a change output. * This plus selection_target is the upper bound of the range. + * @param int max_weight The maximum weight available for the input set. * @returns The result of this coin selection algorithm, or std::nullopt */ @@ -199,6 +216,331 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool return result; } +/* + * TL;DR: Coin Grinder is a DFS-based algorithm that deterministically searches for the minimum-weight input set to fund + * the transaction. The algorithm is similar to the Branch and Bound algorithm, but will produce a transaction _with_ a + * change output instead of a changeless transaction. + * + * Full description: CoinGrinder can be thought of as a graph walking algorithm. It explores a binary tree + * representation of the powerset of the UTXO pool. Each node in the tree represents a candidate input set. The tree’s + * root is the empty set. Each node in the tree has two children which are formed by either adding or skipping the next + * UTXO ("inclusion/omission branch"). Each level in the tree after the root corresponds to a decision about one UTXO in + * the UTXO pool. + * + * Example: + * We represent UTXOs as _alias=[effective_value/weight]_ and indicate omitted UTXOs with an underscore. Given a UTXO + * pool {A=[10/2], B=[7/1], C=[5/1], D=[4/2]} sorted by descending effective value, our search tree looks as follows: + * + * _______________________ {} ________________________ + * / \ + * A=[10/2] __________ {A} _________ __________ {_} _________ + * / \ / \ + * B=[7/1] {AB} _ {A_} _ {_B} _ {__} _ + * / \ / \ / \ / \ + * C=[5/1] {ABC} {AB_} {A_C} {A__} {_BC} {_B_} {__C} {___} + * / \ / \ / \ / \ / \ / \ / \ / \ + * D=[4/2] {ABCD} {ABC_} {AB_D} {AB__} {A_CD} {A_C_} {A__D} {A___} {_BCD} {_BC_} {_B_D} {_B__} {__CD} {__C_} {___D} {____} + * + * + * CoinGrinder uses a depth-first search to walk this tree. It first tries inclusion branches, then omission branches. A + * naive exploration of a tree with four UTXOs requires visiting all 31 nodes: + * + * {} {A} {AB} {ABC} {ABCD} {ABC_} {AB_} {AB_D} {AB__} {A_} {A_C} {A_CD} {A_C_} {A__} {A__D} {A___} {_} {_B} {_BC} + * {_BCD} {_BC_} {_B_} {_B_D} {_B__} {__} {__C} {__CD} {__C} {___} {___D} {____} + * + * As powersets grow exponentially with the set size, walking the entire tree would quickly get computationally + * infeasible with growing UTXO pools. Thanks to traversing the tree in a deterministic order, we can keep track of the + * progress of the search solely on basis of the current selection (and the best selection so far). We visit as few + * nodes as possible by recognizing and skipping any branches that can only contain solutions worse than the best + * solution so far. This makes CoinGrinder a branch-and-bound algorithm + * (https://en.wikipedia.org/wiki/Branch_and_bound). + * CoinGrinder is searching for the input set with lowest weight that can fund a transaction, so for example we can only + * ever find a _better_ candidate input set in a node that adds a UTXO, but never in a node that skips a UTXO. After + * visiting {A} and exploring the inclusion branch {AB} and its descendants, the candidate input set in the omission + * branch {A_} is equivalent to the parent {A} in effective value and weight. While CoinGrinder does need to visit the + * descendants of the omission branch {A_}, it is unnecessary to evaluate the candidate input set in the omission branch + * itself. By skipping evaluation of all nodes on an omission branch we reduce the visited nodes to 15: + * + * {A} {AB} {ABC} {ABCD} {AB_D} {A_C} {A_CD} {A__D} {_B} {_BC} {_BCD} {_B_D} {__C} {__CD} {___D} + * + * _______________________ {} ________________________ + * / \ + * A=[10/2] __________ {A} _________ ___________\____________ + * / \ / \ + * B=[7/1] {AB} __ __\_____ {_B} __ __\_____ + * / \ / \ / \ / \ + * C=[5/1] {ABC} \ {A_C} \ {_BC} \ {__C} \ + * / / / / / / / / + * D=[4/2] {ABCD} {AB_D} {A_CD} {A__D} {_BCD} {_B_D} {__CD} {___D} + * + * + * We refer to the move from the inclusion branch {AB} via the omission branch {A_} to its inclusion-branch child {A_C} + * as _shifting to the omission branch_ or just _SHIFT_. (The index of the ultimate element in the candidate input set + * shifts right by one: {AB} ⇒ {A_C}.) + * When we reach a leaf node in the last level of the tree, shifting to the omission branch is not possible. Instead we + * go to the omission branch of the node’s last ancestor on an inclusion branch: from {ABCD}, we go to {AB_D}. From + * {AB_D}, we go to {A_C}. We refer to this operation as a _CUT_. (The ultimate element in + * the input set is deselected, and the penultimate element is shifted right by one: {AB_D} ⇒ {A_C}.) + * If a candidate input set in a node has not selected sufficient funds to build the transaction, we continue directly + * along the next inclusion branch. We call this operation _EXPLORE_. (We go from one inclusion branch to the next + * inclusion branch: {_B} ⇒ {_BC}.) + * Further, any prefix that already has selected sufficient effective value to fund the transaction cannot be improved + * by adding more UTXOs. If for example the candidate input set in {AB} is a valid solution, all potential descendant + * solutions {ABC}, {ABCD}, and {AB_D} must have a higher weight, thus instead of exploring the descendants of {AB}, we + * can SHIFT from {AB} to {A_C}. + * + * Given the above UTXO set, using a target of 11, and following these initial observations, the basic implementation of + * CoinGrinder visits the following 10 nodes: + * + * Node [eff_val/weight] Evaluation + * --------------------------------------------------------------- + * {A} [10/2] Insufficient funds: EXPLORE + * {AB} [17/3] Solution: SHIFT to omission branch + * {A_C} [15/3] Better solution: SHIFT to omission branch + * {A__D} [14/4] Worse solution, shift impossible due to leaf node: CUT to omission branch of {A__D}, + * i.e. SHIFT to omission branch of {A} + * {_B} [7/1] Insufficient funds: EXPLORE + * {_BC} [12/2] Better solution: SHIFT to omission branch + * {_B_D} [11/3] Worse solution, shift impossible due to leaf node: CUT to omission branch of {_B_D}, + * i.e. SHIFT to omission branch of {_B} + * {__C} [5/1] Insufficient funds: EXPLORE + * {__CD} [9/3] Insufficient funds, leaf node: CUT + * {___D} [4/2] Insufficient funds, leaf node, cannot CUT since only one UTXO selected: done. + * + * _______________________ {} ________________________ + * / \ + * A=[10/2] __________ {A} _________ ___________\____________ + * / \ / \ + * B=[7/1] {AB} __\_____ {_B} __ __\_____ + * / \ / \ / \ + * C=[5/1] {A_C} \ {_BC} \ {__C} \ + * / / / / + * D=[4/2] {A__D} {_B_D} {__CD} {___D} + * + * + * We implement this tree walk in the following algorithm: + * 1. Add `next_utxo` + * 2. Evaluate candidate input set + * 3. Determine `next_utxo` by deciding whether to + * a) EXPLORE: Add next inclusion branch, e.g. {_B} ⇒ {_B} + `next_uxto`: C + * b) SHIFT: Replace last selected UTXO by next higher index, e.g. {A_C} ⇒ {A__} + `next_utxo`: D + * c) CUT: deselect last selected UTXO and shift to omission branch of penultimate UTXO, e.g. {AB_D} ⇒ {A_} + `next_utxo: C + * + * The implementation then adds further optimizations by discovering further situations in which either the inclusion + * branch can be skipped, or both the inclusion and omission branch can be skipped after evaluating the candidate input + * set in the node. + * + * @param std::vector& utxo_pool The UTXOs that we are choosing from. These UTXOs will be sorted in + * descending order by effective value, with lower weight preferred as a tie-breaker. (We can think of an output + * group with multiple as a heavier UTXO with the combined amount here.) + * @param const CAmount& selection_target This is the minimum amount that we need for the transaction without considering change. + * @param const CAmount& change_target The minimum budget for creating a change output, by which we increase the selection_target. + * @param int max_weight The maximum permitted weight for the input set. + * @returns The result of this coin selection algorithm, or std::nullopt + */ +util::Result CoinGrinder(std::vector& utxo_pool, const CAmount& selection_target, CAmount change_target, int max_weight) +{ + std::sort(utxo_pool.begin(), utxo_pool.end(), descending_effval_weight); + // The sum of UTXO amounts after this UTXO index, e.g. lookahead[5] = Σ(UTXO[6+].amount) + std::vector lookahead(utxo_pool.size()); + // The minimum UTXO weight among the remaining UTXOs after this UTXO index, e.g. min_tail_weight[5] = min(UTXO[6+].weight) + std::vector min_tail_weight(utxo_pool.size()); + + // Calculate lookahead values, min_tail_weights, and check that there are sufficient funds + CAmount total_available = 0; + int min_group_weight = std::numeric_limits::max(); + for (size_t i = 0; i < utxo_pool.size(); ++i) { + size_t index = utxo_pool.size() - 1 - i; // Loop over every element in reverse order + lookahead[index] = total_available; + min_tail_weight[index] = min_group_weight; + // UTXOs with non-positive effective value must have been filtered + Assume(utxo_pool[index].GetSelectionAmount() > 0); + total_available += utxo_pool[index].GetSelectionAmount(); + min_group_weight = std::min(min_group_weight, utxo_pool[index].m_weight); + } + + const CAmount total_target = selection_target + change_target; + if (total_available < total_target) { + // Insufficient funds + return util::Error(); + } + + // The current selection and the best input set found so far, stored as the utxo_pool indices of the UTXOs forming them + std::vector curr_selection; + std::vector best_selection; + + // The currently selected effective amount, and the effective amount of the best selection so far + CAmount curr_amount = 0; + CAmount best_selection_amount = MAX_MONEY; + + // The weight of the currently selected input set, and the weight of the best selection + int curr_weight = 0; + int best_selection_weight = max_weight; // Tie is fine, because we prefer lower selection amount + + // Whether the input sets generated during this search have exceeded the maximum transaction weight at any point + bool max_tx_weight_exceeded = false; + + // Index of the next UTXO to consider in utxo_pool + size_t next_utxo = 0; + + /* + * You can think of the current selection as a vector of booleans that has decided inclusion or exclusion of all + * UTXOs before `next_utxo`. When we consider the next UTXO, we extend this hypothetical boolean vector either with + * a true value if the UTXO is included or a false value if it is omitted. The equivalent state is stored more + * compactly as the list of indices of the included UTXOs and the `next_utxo` index. + * + * We can never find a new solution by deselecting a UTXO, because we then revisit a previously evaluated + * selection. Therefore, we only need to check whether we found a new solution _after adding_ a new UTXO. + * + * Each iteration of CoinGrinder starts by selecting the `next_utxo` and evaluating the current selection. We + * use three state transitions to progress from the current selection to the next promising selection: + * + * - EXPLORE inclusion branch: We do not have sufficient funds, yet. Add `next_utxo` to the current selection, then + * nominate the direct successor of the just selected UTXO as our `next_utxo` for the + * following iteration. + * + * Example: + * Current Selection: {0, 5, 7} + * Evaluation: EXPLORE, next_utxo: 8 + * Next Selection: {0, 5, 7, 8} + * + * - SHIFT to omission branch: Adding more UTXOs to the current selection cannot produce a solution that is better + * than the current best, e.g. the current selection weight exceeds the max weight or + * the current selection amount is equal to or greater than the target. + * We designate our `next_utxo` the one after the tail of our current selection, then + * deselect the tail of our current selection. + * + * Example: + * Current Selection: {0, 5, 7} + * Evaluation: SHIFT, next_utxo: 8, omit last selected: {0, 5} + * Next Selection: {0, 5, 8} + * + * - CUT entire subtree: We have exhausted the inclusion branch for the penultimately selected UTXO, both the + * inclusion and the omission branch of the current prefix are barren. E.g. we have + * reached the end of the UTXO pool, so neither further EXPLORING nor SHIFTING can find + * any solutions. We designate our `next_utxo` the one after our penultimate selected, + * then deselect both the last and penultimate selected. + * + * Example: + * Current Selection: {0, 5, 7} + * Evaluation: CUT, next_utxo: 6, omit two last selected: {0} + * Next Selection: {0, 6} + */ + auto deselect_last = [&]() { + OutputGroup& utxo = utxo_pool[curr_selection.back()]; + curr_amount -= utxo.GetSelectionAmount(); + curr_weight -= utxo.m_weight; + curr_selection.pop_back(); + }; + + SelectionResult result(selection_target, SelectionAlgorithm::CG); + bool is_done = false; + size_t curr_try = 0; + while (!is_done) { + bool should_shift{false}, should_cut{false}; + // Select `next_utxo` + OutputGroup& utxo = utxo_pool[next_utxo]; + curr_amount += utxo.GetSelectionAmount(); + curr_weight += utxo.m_weight; + curr_selection.push_back(next_utxo); + ++next_utxo; + ++curr_try; + + // EVALUATE current selection: check for solutions and see whether we can CUT or SHIFT before EXPLORING further + auto curr_tail = curr_selection.back(); + if (curr_amount + lookahead[curr_tail] < total_target) { + // Insufficient funds with lookahead: CUT + should_cut = true; + } else if (curr_weight > best_selection_weight) { + // best_selection_weight is initialized to max_weight + if (curr_weight > max_weight) max_tx_weight_exceeded = true; + // Worse weight than best solution. More UTXOs only increase weight: + // CUT if last selected group had minimal weight, else SHIFT + if (utxo_pool[curr_tail].m_weight <= min_tail_weight[curr_tail]) { + should_cut = true; + } else { + should_shift = true; + } + } else if (curr_amount >= total_target) { + // Success, adding more weight cannot be better: SHIFT + should_shift = true; + if (curr_weight < best_selection_weight || (curr_weight == best_selection_weight && curr_amount < best_selection_amount)) { + // New lowest weight, or same weight with fewer funds tied up + best_selection = curr_selection; + best_selection_weight = curr_weight; + best_selection_amount = curr_amount; + } + } else if (!best_selection.empty() && curr_weight + int64_t{min_tail_weight[curr_tail]} * ((total_target - curr_amount + utxo_pool[curr_tail].GetSelectionAmount() - 1) / utxo_pool[curr_tail].GetSelectionAmount()) > best_selection_weight) { + // Compare minimal tail weight and last selected amount with the amount missing to gauge whether a better weight is still possible. + if (utxo_pool[curr_tail].m_weight <= min_tail_weight[curr_tail]) { + should_cut = true; + } else { + should_shift = true; + } + } + + if (curr_try >= TOTAL_TRIES) { + // Solution is not guaranteed to be optimal if `curr_try` hit TOTAL_TRIES + result.SetAlgoCompleted(false); + break; + } + + if (next_utxo == utxo_pool.size()) { + // Last added UTXO was end of UTXO pool, nothing left to add on inclusion or omission branch: CUT + should_cut = true; + } + + if (should_cut) { + // Neither adding to the current selection nor exploring the omission branch of the last selected UTXO can + // find any solutions. Redirect to exploring the Omission branch of the penultimate selected UTXO (i.e. + // set `next_utxo` to one after the penultimate selected, then deselect the last two selected UTXOs) + should_cut = false; + deselect_last(); + should_shift = true; + } + + while (should_shift) { + // Set `next_utxo` to one after last selected, then deselect last selected UTXO + if (curr_selection.empty()) { + // Exhausted search space before running into attempt limit + is_done = true; + result.SetAlgoCompleted(true); + break; + } + next_utxo = curr_selection.back() + 1; + deselect_last(); + should_shift = false; + + // After SHIFTing to an omission branch, the `next_utxo` might have the same effective value as the UTXO we + // just omitted. Since lower weight is our tiebreaker on UTXOs with equal effective value for sorting, if it + // ties on the effective value, it _must_ have the same weight (i.e. be a "clone" of the prior UTXO) or a + // higher weight. If so, selecting `next_utxo` would produce an equivalent or worse selection as one we + // previously evaluated. In that case, increment `next_utxo` until we find a UTXO with a differing amount. + while (utxo_pool[next_utxo - 1].GetSelectionAmount() == utxo_pool[next_utxo].GetSelectionAmount()) { + if (next_utxo >= utxo_pool.size() - 1) { + // Reached end of UTXO pool skipping clones: SHIFT instead + should_shift = true; + break; + } + // Skip clone: previous UTXO is equivalent and unselected + ++next_utxo; + } + } + } + + result.SetSelectionsEvaluated(curr_try); + + if (best_selection.empty()) { + return max_tx_weight_exceeded ? ErrorMaxWeightExceeded() : util::Error(); + } + + for (const size_t& i : best_selection) { + result.AddInput(utxo_pool[i]); + } + + return result; +} + class MinOutputGroupComparator { public: @@ -583,6 +925,26 @@ void SelectionResult::ComputeAndSetWaste(const CAmount min_viable_change, const } } +void SelectionResult::SetAlgoCompleted(bool algo_completed) +{ + m_algo_completed = algo_completed; +} + +bool SelectionResult::GetAlgoCompleted() const +{ + return m_algo_completed; +} + +void SelectionResult::SetSelectionsEvaluated(size_t attempts) +{ + m_selections_evaluated = attempts; +} + +size_t SelectionResult::GetSelectionsEvaluated() const +{ + return m_selections_evaluated; +} + CAmount SelectionResult::GetWaste() const { return *Assert(m_waste); @@ -671,6 +1033,7 @@ std::string GetAlgorithmName(const SelectionAlgorithm algo) case SelectionAlgorithm::BNB: return "bnb"; case SelectionAlgorithm::KNAPSACK: return "knapsack"; case SelectionAlgorithm::SRD: return "srd"; + case SelectionAlgorithm::CG: return "cg"; case SelectionAlgorithm::MANUAL: return "manual"; // No default case to allow for compiler to warn } diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index a8ee26a594e2..03ba78020b28 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -140,8 +140,8 @@ struct CoinSelectionParams { size_t change_output_size = 0; /** Size of the input to spend a change output in virtual bytes. */ size_t change_spend_size = 0; - /** Mininmum change to target in Knapsack solver: select coins to cover the payment and - * at least this value of change. */ + /** Mininmum change to target in Knapsack solver and CoinGrinder: + * select coins to cover the payment and at least this value of change. */ CAmount m_min_change_target{0}; /** Minimum amount for creating a change output. * If change budget is smaller than min_change then we forgo creation of change output. @@ -329,7 +329,8 @@ enum class SelectionAlgorithm : uint8_t BNB = 0, KNAPSACK = 1, SRD = 2, - MANUAL = 3, + CG = 3, + MANUAL = 4, }; std::string GetAlgorithmName(const SelectionAlgorithm algo); @@ -347,6 +348,10 @@ struct SelectionResult bool m_use_effective{false}; /** The computed waste */ std::optional m_waste; + /** False if algorithm was cut short by hitting limit of attempts and solution is non-optimal */ + bool m_algo_completed{true}; + /** The count of selections that were evaluated by this coin selection attempt */ + size_t m_selections_evaluated; /** Total weight of the selected inputs */ int m_weight{0}; @@ -381,6 +386,18 @@ struct SelectionResult void ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); [[nodiscard]] CAmount GetWaste() const; + /** Tracks that algorithm was able to exhaustively search the entire combination space before hitting limit of tries */ + void SetAlgoCompleted(bool algo_completed); + + /** Get m_algo_completed */ + bool GetAlgoCompleted() const; + + /** Record the number of selections that were evaluated */ + void SetSelectionsEvaluated(size_t attempts); + + /** Get selections_evaluated */ + size_t GetSelectionsEvaluated() const ; + /** * Combines the @param[in] other selection result into 'this' selection result. * @@ -423,6 +440,8 @@ struct SelectionResult util::Result SelectCoinsBnB(std::vector& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change, int max_weight); +util::Result CoinGrinder(std::vector& utxo_pool, const CAmount& selection_target, CAmount change_target, int max_weight); + /** Select coins by Single Random Draw. OutputGroups are selected randomly from the eligible * outputs until the target is satisfied * diff --git a/src/wallet/context.h b/src/wallet/context.h index 877301715bc0..e99cd86f0d4e 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -13,6 +13,7 @@ #include class ArgsManager; +class CScheduler; namespace interfaces { class Chain; class Wallet; @@ -38,6 +39,7 @@ using LoadWalletFn = std::function wall //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + CScheduler* scheduler{nullptr}; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct // It is unsafe to lock this after locking a CWallet::cs_wallet mutex because // this could introduce inconsistent lock ordering and cause deadlocks. diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index c6d755e9d95d..6bbf7ef7d5ec 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include