diff --git a/doc/release-notes-7861.md b/doc/release-notes-7861.md new file mode 100644 index 000000000000..cfa126788d91 --- /dev/null +++ b/doc/release-notes-7861.md @@ -0,0 +1,12 @@ +Pruning +------- + +- Pruned nodes on mainnet, testnet and devnets now keep at least four days of + blocks (2304) below the tip instead of 288. Validation reads older blocks + from disk: the credit pool is rebuilt from its newest daily snapshot after a + restart, and rotated quorum membership reads the work blocks of earlier DKG + cycles. `pruneblockchain` clamps its height accordingly, and on testnet and + devnets pruning starts once the chain is longer than 3000 blocks + (previously 1000). Blocks that an earlier version already pruned are not + recovered; a node that fails to validate a block after upgrading needs + `-reindex`. diff --git a/src/Makefile.am b/src/Makefile.am index 44fb7be39ced..add830dc7a55 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -31,6 +31,14 @@ check_PROGRAMS = TESTS = BENCHMARKS = +# The first make in dashbls after configure reruns its config.status, which +# replaces relic_conf.h. Let that finish before compiling anything that +# includes it. +BUILT_SOURCES = dashbls/depends/relic/include/relic_conf.h + +dashbls/depends/relic/include/relic_conf.h: FORCE + $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C dashbls depends/relic/include/relic_conf.h + if ENABLE_STACKTRACES if ENABLE_CRASH_HOOKS if CRASH_HOOKS_WRAPPED_CXX_ABI diff --git a/src/chainparams.cpp b/src/chainparams.cpp index c46bc0a738b8..82b68c08d547 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -243,6 +243,7 @@ class CMainParams : public CChainParams { nDefaultPlatformP2PPort = 26656; nDefaultPlatformHTTPPort = 443; nPruneAfterHeight = 100000; + m_min_blocks_to_keep = 2304; // four days m_assumed_blockchain_size = 57; m_assumed_chain_state_size = 1; @@ -446,7 +447,8 @@ class CTestNetParams : public CChainParams { nDefaultPort = 19999; nDefaultPlatformP2PPort = 22000; nDefaultPlatformHTTPPort = 22001; - nPruneAfterHeight = 1000; + nPruneAfterHeight = 3000; + m_min_blocks_to_keep = 2304; // four days m_assumed_blockchain_size = 10; m_assumed_chain_state_size = 1; @@ -628,7 +630,8 @@ class CDevNetParams : public CChainParams { nDefaultPort = 19799; nDefaultPlatformP2PPort = 22100; nDefaultPlatformHTTPPort = 22101; - nPruneAfterHeight = 1000; + nPruneAfterHeight = 3000; + m_min_blocks_to_keep = 2304; // four days m_assumed_blockchain_size = 0; m_assumed_chain_state_size = 0; @@ -873,6 +876,7 @@ class CRegTestParams : public CChainParams { nDefaultPlatformP2PPort = 22200; nDefaultPlatformHTTPPort = 22201; nPruneAfterHeight = args.GetBoolArg("-fastprune", false) ? 100 : 1000; + m_min_blocks_to_keep = 288; m_assumed_blockchain_size = 0; m_assumed_chain_state_size = 0; diff --git a/src/chainparams.h b/src/chainparams.h index dc21b75f60f9..22e86ef13169 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -114,6 +114,8 @@ class CChainParams /** If this chain is exclusively used for testing */ bool IsTestChain() const { return m_is_test_chain; } uint64_t PruneAfterHeight() const { return nPruneAfterHeight; } + /** Block files containing a block-height within MinBlocksToKeep() of ActiveChain().Tip() will not be pruned. */ + int MinBlocksToKeep() const { return m_min_blocks_to_keep; } /** Minimum free space (in GB) needed for data directory */ uint64_t AssumedBlockchainSize() const { return m_assumed_blockchain_size; } /** Minimum free space (in GB) needed for data directory when pruned; Does not include prune target*/ @@ -166,6 +168,11 @@ class CChainParams CMessageHeader::MessageStartChars pchMessageStart; uint16_t nDefaultPort; uint64_t nPruneAfterHeight; + /// Four days of blocks (2304) on every network but regtest. Validation reads older blocks from disk: + /// a credit pool is rebuilt from its newest daily snapshot, reading the block one withdrawal window + /// before each block since it, and rotated LLMQ modifiers read the work block of earlier cycles. + /// Regtest keeps Bitcoin Core's 288 for the upstream pruning tests. + int m_min_blocks_to_keep{288}; uint64_t m_assumed_blockchain_size; uint64_t m_assumed_chain_state_size; std::vector vSeeds; diff --git a/src/evo/creditpool.h b/src/evo/creditpool.h index e7676beace54..060a570e17ca 100644 --- a/src/evo/creditpool.h +++ b/src/evo/creditpool.h @@ -123,9 +123,8 @@ class CCreditPoolManager CEvoDB& evoDb; const ChainstateManager& m_chainman; - static constexpr int DISK_SNAPSHOT_PERIOD = 576; // once per day - public: + static constexpr int DISK_SNAPSHOT_PERIOD = 576; // once per day static constexpr CAmount LimitAmountLow = 100 * COIN; static constexpr CAmount LimitAmountHigh = 1000 * COIN; static constexpr CAmount LimitAmountV22 = 2000 * COIN; diff --git a/src/init.cpp b/src/init.cpp index aa4d77b639ed..170936cbf62f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -2099,9 +2099,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } if (status == node::ChainstateLoadStatus::SUCCESS) { uiInterface.InitMessage(_("Verifying blocks…").translated); - if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { + if (chainman.m_blockman.m_have_pruned && options.check_blocks > chainman.GetParams().MinBlocksToKeep()) { LogWarning("pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); + chainman.GetParams().MinBlocksToKeep()); } std::tie(status, error) = catch_exceptions([&]{ return VerifyLoadedChainstate(chainman, options, *Assert(node.evodb), [](bool bls_state) { LogPrintf("AppInitMain: bls_legacy_scheme=%d\n", bls_state); diff --git a/src/net.cpp b/src/net.cpp index 4e9d3457cb7f..a8bb48cdc6d0 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2599,7 +2599,7 @@ size_t CConnman::SocketRecvData(CNode *pnode) if (!pnode->m_sock) return 0; nBytes = recv(pnode->m_sock->Get(), (char*)pchBuf, sizeof(pchBuf), MSG_DONTWAIT); - if (nBytes < (int)sizeof(pchBuf)) { + if (nBytes < 0 && WSAGetLastError() == WSAEWOULDBLOCK) { pnode->fHasRecvData = false; } } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c41876bc606c..d315c6f2a2cb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -160,7 +160,7 @@ static constexpr auto BLOCK_STALLING_TIMEOUT_MAX{64s}; static const int MAX_CMPCTBLOCK_DEPTH = 5; /** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ static const int MAX_BLOCKTXN_DEPTH = 10; -static_assert(MAX_BLOCKTXN_DEPTH <= MIN_BLOCKS_TO_KEEP, "MAX_BLOCKTXN_DEPTH too high"); +static_assert(MAX_BLOCKTXN_DEPTH <= BLOCKS_NEAR_TIP, "MAX_BLOCKTXN_DEPTH too high"); /** Size of the "block download window": how far ahead of our current height do we fetch? * Larger windows tolerate larger download speed differences between peer, but increase the potential * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably @@ -4691,7 +4691,7 @@ void PeerManagerImpl::ProcessMessage( } // If pruning, don't inv blocks unless we have on disk and are likely to still have // for some reasonable time window (1 hour) that block relay might require. - const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / m_chainparams.GetConsensus().nPowTargetSpacing; + const int nPrunedBlocksLikelyToHave = BLOCKS_NEAR_TIP - 3600 / m_chainparams.GetConsensus().nPowTargetSpacing; if (m_chainman.m_blockman.IsPruneMode() && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= m_chainman.ActiveChain().Tip()->nHeight - nPrunedBlocksLikelyToHave)) { LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 1f2680fa0a6c..249a6dc2e259 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -171,8 +171,9 @@ void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nM return; } - // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) - unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); + // last block to prune is the lesser of (user-specified height, MinBlocksToKeep() from the tip) + unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, + static_cast(chain_tip_height - GetParams().MinBlocksToKeep())); int count = 0; for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { @@ -195,7 +196,7 @@ void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPr return; } - unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP))}; + unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - GetParams().MinBlocksToKeep())}; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -225,7 +226,7 @@ void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPr break; } - // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning + // don't prune files that could have a block within MinBlocksToKeep() of the main chain's tip but keep scanning if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { continue; } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 70f1547fb1c4..ec7ba9c76e81 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -112,7 +112,7 @@ class BlockManager * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set. * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight). - * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * Pruning will never delete a block within CChainParams::MinBlocksToKeep() of the active chain's tip. * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. * A db flag records the fact that at least some block files have been pruned. * diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 3b82e308a89d..4f98fa02fca0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1120,9 +1120,9 @@ static RPCHelpMan pruneblockchain() throw JSONRPCError(RPC_MISC_ERROR, "Blockchain is too short for pruning."); } else if (height > chainHeight) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); - } else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { + } else if (height > chainHeight - chainman.GetParams().MinBlocksToKeep()) { LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks.\n"); - height = chainHeight - MIN_BLOCKS_TO_KEEP; + height = chainHeight - chainman.GetParams().MinBlocksToKeep(); } PruneBlockFilesManual(active_chainstate, height); diff --git a/src/test/evo_assetlocks_tests.cpp b/src/test/evo_assetlocks_tests.cpp index 7bc4c09a5e85..b9780d3e4866 100644 --- a/src/test/evo_assetlocks_tests.cpp +++ b/src/test/evo_assetlocks_tests.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -858,6 +859,17 @@ BOOST_FIXTURE_TEST_CASE(credit_pool_snapshot_persisted_after_transactionless_con BOOST_CHECK_EQUAL(snapshot.currentLimit, pool.currentLimit); } +BOOST_FIXTURE_TEST_CASE(min_blocks_to_keep_covers_credit_pool_rebuild, BasicTestingSetup) +{ + // A pool is rebuilt from a snapshot up to one period old, reading the block one window before + // each block since it. Regtest deliberately keeps Bitcoin Core's 288 and is not checked. + for (const auto& chain : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::DEVNET}) { + const auto params = CreateChainParams(*m_node.args, chain); + BOOST_CHECK_GE(params->MinBlocksToKeep(), + CCreditPoolManager::DISK_SNAPSHOT_PERIOD + params->CreditPoolPeriodBlocks()); + } +} + BOOST_AUTO_TEST_CASE(credit_pool_unlock_limit_v24) { // From v24 the limit is relative and net: 20% of the balance at the window start, never less diff --git a/src/validation.cpp b/src/validation.cpp index 85d41ae6d7c6..e20e6e9b824f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4797,7 +4797,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr& pblock, // blocks which are too close in height to the tip. Apply this test // regardless of whether pruning is enabled; it should generally be safe to // not process unrequested blocks. - bool fTooFarAhead{pindex->nHeight > ActiveHeight() + int(MIN_BLOCKS_TO_KEEP)}; + bool fTooFarAhead{pindex->nHeight > ActiveHeight() + int(BLOCKS_NEAR_TIP)}; // TODO: Decouple this function from the block download logic by removing fRequested // This requires some new chain data structure to efficiently look up if a @@ -5200,6 +5200,9 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in BlockValidationState state; const CAmount blockSubsidy = GetBlockSubsidy(pindex, m_chainman.GetConsensus()); MNListUpdates mnlist_updates; + // Use the scheme ConnectBlock used for this block instead of whatever replay inherited + ScopedBLSLegacyScheme bls_scheme_guard{ + !DeploymentActiveAt(*pindex, m_chainman.GetConsensus(), Consensus::DEPLOYMENT_V19)}; if (!m_chain_helper->special_tx->ProcessSpecialTxsInBlock(*this, m_chain, block, pindex, GetSpecialTxRules(pindex->pprev, m_chainman), inputs, blockSubsidy, /*fJustCheck=*/false, @@ -5298,6 +5301,11 @@ bool Chainstate::ReplayBlocks() LogPrintf("Rolling forward %s (%i)\n", pindex.GetBlockHash().ToString(), nHeight); uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); + // Special transaction and quorum commitment checks read the chain, which + // LoadChainTip() has not set yet; give them the view ConnectBlock had. + // Stopping at the parent also keeps LoadChainTip() from skipping its + // candidate pruning. + m_chain.SetTip(*Assert(pindex.pprev)); if (!RollforwardBlock(&pindex, cache)) return false; } diff --git a/src/validation.h b/src/validation.h index 2b6e8e6ce9f9..de5c2fc754e8 100644 --- a/src/validation.h +++ b/src/validation.h @@ -84,8 +84,9 @@ static const unsigned int MAX_HEADERS_COMPRESSED_RESULT = 8000; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ -static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +/** Blocks within this distance of ActiveChain().Tip() are accepted unrequested and announced by pruned nodes. + * Bitcoin Core's MIN_BLOCKS_TO_KEEP; pruning retention is CChainParams::MinBlocksToKeep(). */ +static const unsigned int BLOCKS_NEAR_TIP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static constexpr int DEFAULT_CHECKLEVEL{3}; diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py index 56536643ba39..493a0c74fdf5 100755 --- a/test/functional/feature_asset_locks.py +++ b/test/functional/feature_asset_locks.py @@ -5,7 +5,9 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import copy +import http.client import struct +import subprocess from decimal import Decimal from io import BytesIO @@ -46,6 +48,7 @@ assert_equal, assert_greater_than, assert_greater_than_or_equal, + assert_raises, assert_raises_rpc_error, softfork_active, ) @@ -354,6 +357,7 @@ def run_test(self): self.test_v24_fork(node_wallet, node, pubkey) self.test_asset_unlock_v2(node_wallet, node, pubkey) self.test_non_standard(node_wallet, node, pubkey) + self.test_replay_after_crash(node_wallet, node, pubkey) def test_asset_locks(self, node_wallet, node, pubkey): @@ -1438,6 +1442,49 @@ def test_non_standard(self, node_wallet, node, pubkey): self.restart_node(1, self.extra_args[1]) self.connect_nodes(1, 0) + def test_replay_after_crash(self, node_wallet, node, pubkey): + self.log.info("Testing recovery from a crash mid-flush with a quorum commitment and an asset unlock to replay...") + # Every flush now dies after its first partial write, so nothing mined from here on is flushed + self.restart_with_crashing_flush() + + self.mine_quorum_2_nodes() + quorum_hash = node_wallet.quorum("list", 1)["llmq_test_platform"][0] + commitment_block = node_wallet.quorum("info", llmq_type_test, quorum_hash)["minedBlock"] + txid = self.send_tx(self.create_assetunlock(900, COIN, pubkey)) + unlock_block = self.generate(node_wallet, 1)[0] + assert txid in node_wallet.getblock(unlock_block)["tx"] + + self.log.info("Restart without peers: the commitment block is replayed before the asset unlock block") + self.crash_and_replay(node_wallet, node, [commitment_block, unlock_block]) + assert quorum_hash in node_wallet.quorum("list")["llmq_test_platform"] + + self.log.info("Replay an asset unlock as the first block rolled forward, with the quorum already mature and flushed") + self.restart_node(0, self.extra_args[0]) + self.restart_with_crashing_flush() + txid = self.send_tx(self.create_assetunlock(901, COIN, pubkey)) + unlock_block = self.generate(node_wallet, 1)[0] + assert txid in node_wallet.getblock(unlock_block)["tx"] + self.crash_and_replay(node_wallet, node, [unlock_block]) + + self.restart_node(0, self.extra_args[0]) + for i in range(1, len(self.nodes)): + self.connect_nodes(0, i) + + def restart_with_crashing_flush(self): + self.restart_node(0, self.extra_args[0] + ["-dbcrashratio=1", "-dbbatchsize=1"]) + for i in range(1, len(self.nodes)): + self.connect_nodes(0, i) + + def crash_and_replay(self, node_wallet, node, replayed_blocks): + tip = node_wallet.getbestblockhash() + self.log.info("Force a flush, which crashes the node") + assert_raises((ConnectionError, http.client.CannotSendRequest, subprocess.CalledProcessError), node_wallet.gettxoutsetinfo) + self.wait_for_node_exit(0, timeout=10) + with node_wallet.assert_debug_log([f"Rolling forward {block}" for block in replayed_blocks]): + self.start_node(0, extra_args=self.extra_args[0] + ["-connect=0"]) + assert_equal(node_wallet.getbestblockhash(), tip) + assert_equal(node_wallet.gettxoutsetinfo()["hash_serialized_2"], node.gettxoutsetinfo()["hash_serialized_2"]) + if __name__ == '__main__': AssetLocksTest().main()