backport: bitcoin#27609, #27850, #28291, #28392, #28428, #28450, #28458, #28471, #28483, #28545

.github/workflows/test-src.yml

@@ -41,7 +41,8 @@ jobs:
       image: ${{ inputs.container-path }} # zizmor: ignore[unpinned-images] built earlier in this run by build-container.yml
       # SYS_PTRACE is required by LeakSanitizer's stop-the-world
       # (https://github.com/google/sanitizers/issues/764)
-      options: --user root --cap-add SYS_PTRACE
+      # LINUX_IMMUTABLE lets feature_reindex_readonly.py run chattr +i
+      options: --user root --cap-add LINUX_IMMUTABLE --cap-add SYS_PTRACE
     steps:
       - name: Checkout code
         uses: actions/checkout@v6

contrib/containers/ci/ci-slim.Dockerfile

@@ -40,6 +40,7 @@ RUN set -ex; \
     build-essential \
     ca-certificates \
     curl \
+    e2fsprogs \
     g++ \
     git \
     jq \

doc/release-notes-27609.md

@@ -0,0 +1,13 @@
+- A new RPC, `submitpackage`, has been added. It can be used to submit a list of raw hex
+  transactions to the mempool to be evaluated as a package using consensus and mempool policy rules.
+These policies include package CPFP, allowing a child with high fees to bump a parent below the
+mempool minimum feerate (but not minimum relay feerate).
+
+    - Warning: successful submission does not mean the transactions will propagate throughout the
+      network, as package relay is not supported.
+
+    - Not all features are available. The package is limited to a child with all of its
+      unconfirmed parents, and no parent may spend the output of another parent. Refer to
+      doc/policy/packages.md for more details on package policies and limitations.
+
+    - This RPC is experimental. Its interface may change.

src/Makefile.test.include

@@ -363,6 +363,7 @@ test_fuzz_fuzz_SOURCES = \
  test/fuzz/netbase_dns_lookup.cpp \
  test/fuzz/node_eviction.cpp \
  test/fuzz/p2p_transport_serialization.cpp \
+ test/fuzz/package_eval.cpp \
  test/fuzz/parse_hd_keypath.cpp \
  test/fuzz/parse_numbers.cpp \
  test/fuzz/parse_script.cpp \

src/bench/load_external.cpp

@@ -5,6 +5,7 @@
 #include <bench/bench.h>
 #include <bench/data.h>
 #include <chainparams.h>
+#include <clientversion.h>
 #include <test/util/setup_common.h>
 #include <util/chaintype.h>
 #include <validation.h>
@@ -54,7 +55,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench)
     bench.run([&] {
         // "rb" is "binary, O_RDONLY", positioned to the start of the file.
         // The file will be closed by LoadExternalBlockFile().
-        FILE* file{fsbridge::fopen(blkfile, "rb")};
+        CAutoFile file{fsbridge::fopen(blkfile, "rb"), CLIENT_VERSION};
         testing_setup->m_node.chainman->LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
     });
     fs::remove(blkfile);

src/bench/streams_findbyte.cpp

@@ -4,27 +4,31 @@
 
 #include <bench/bench.h>
 
-#include <util/fs.h>
 #include <streams.h>
+#include <util/fs.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
 
 static void FindByte(benchmark::Bench& bench)
 {
     // Setup
-    FILE* file = fsbridge::fopen("streams_tmp", "w+b");
+    CAutoFile file{fsbridge::fopen("streams_tmp", "w+b"), 0};
     const size_t file_size = 200;
     uint8_t data[file_size] = {0};
     data[file_size-1] = 1;
-    fwrite(&data, sizeof(uint8_t), file_size, file);
-    rewind(file);
-    BufferedFile bf{file, /*nBufSize=*/file_size + 1, /*nRewindIn=*/file_size, 0};
+    file << data;
+    std::rewind(file.Get());
+    BufferedFile bf{file, /*nBufSize=*/file_size + 1, /*nRewindIn=*/file_size};
 
     bench.run([&] {
         bf.SetPos(0);
         bf.FindByte(std::byte(1));
     });
 
     // Cleanup
-    bf.fclose();
+    file.fclose();
     fs::remove("streams_tmp");
 }
 

src/chain.h

@@ -372,6 +372,14 @@ const CBlockIndex* LastCommonAncestor(const CBlockIndex* pa, const CBlockIndex*
 /** Used to marshal pointers into hashes for db storage. */
 class CDiskBlockIndex : public CBlockIndex
 {
+    /** Historically CBlockLocator's version field has been written to disk
+     * streams as the client version, but the value has never been used.
+     *
+     * Hard-code to the highest client version ever written.
+     * SerParams can be used if the field requires any meaning in the future.
+     **/
+    static constexpr int DUMMY_VERSION = 240100;
+
 public:
     uint256 hash;
     uint256 hashPrev;
@@ -391,8 +399,8 @@ class CDiskBlockIndex : public CBlockIndex
     SERIALIZE_METHODS(CDiskBlockIndex, obj)
     {
         LOCK(::cs_main);
-        int _nVersion = s.GetVersion();
-        if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
+        int _nVersion = DUMMY_VERSION;
+        READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
 
         READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED));
         READWRITE(VARINT(obj.nStatus));

src/dbwrapper.cpp

@@ -4,7 +4,6 @@
 
 #include <dbwrapper.h>
 
-#include <clientversion.h>
 #include <common/system.h>
 #include <logging.h>
 #include <random.h>
@@ -157,9 +156,9 @@ struct CDBBatch::WriteBatchImpl {
     leveldb::WriteBatch batch;
 };
 
-CDBBatch::CDBBatch(const CDBWrapper& _parent) : parent(_parent),
-                                                m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()},
-                                                ssValue(SER_DISK, CLIENT_VERSION){};
+CDBBatch::CDBBatch(const CDBWrapper& _parent)
+    : parent{_parent},
+      m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()} {};
 
 CDBBatch::~CDBBatch() = default;
 
@@ -169,7 +168,7 @@ void CDBBatch::Clear()
     size_estimate = 0;
 }
 
-void CDBBatch::WriteImpl(Span<const std::byte> key, CDataStream& ssValue)
+void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
 {
     leveldb::Slice slKey(CharCast(key.data()), key.size());
     ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));

src/dbwrapper.h

@@ -91,11 +91,11 @@ class CDBBatch
     const std::unique_ptr<WriteBatchImpl> m_impl_batch;
 
     DataStream ssKey{};
-    CDataStream ssValue;
+    DataStream ssValue{};
 
     size_t size_estimate{0};
 
-    void WriteImpl(Span<const std::byte> key, CDataStream& ssValue);
+    void WriteImpl(Span<const std::byte> key, DataStream& ssValue);
     void EraseImpl(Span<const std::byte> key);
 
 public:
@@ -199,7 +199,7 @@ class CDBIterator
 
     template<typename V> bool GetValue(V& value) {
         try {
-            CDataStream ssValue{GetValueImpl(), SER_DISK, CLIENT_VERSION};
+            DataStream ssValue{GetValueImpl()};
             ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
             ssValue >> value;
         } catch (const std::exception&) {
@@ -252,21 +252,21 @@ class CDBWrapper
     CDBWrapper& operator=(const CDBWrapper&) = delete;
 
     template <typename K>
-    bool ReadDataStream(const K& key, CDataStream& ssValue) const
+    bool ReadDataStream(const K& key, DataStream& ssValue) const
     {
         DataStream ssKey{};
         ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
         ssKey << key;
         return ReadDataStream(ssKey, ssValue);
     }
 
-    bool ReadDataStream(const DataStream& ssKey, CDataStream& ssValue) const
+    bool ReadDataStream(const DataStream& ssKey, DataStream& ssValue) const
     {
         std::optional<std::string> strValue{ReadImpl(ssKey)};
         if (!strValue) {
             return false;
         }
-        CDataStream ssValueTmp{MakeByteSpan(*strValue), SER_DISK, CLIENT_VERSION};
+        DataStream ssValueTmp{MakeByteSpan(*strValue)};
         ssValueTmp.Xor(obfuscate_key);
         ssValue = std::move(ssValueTmp);
         return true;
@@ -284,7 +284,7 @@ class CDBWrapper
     template <typename V>
     bool Read(const DataStream& ssKey, V& value) const
     {
-        CDataStream ssValue(SER_DISK, CLIENT_VERSION);
+        DataStream ssValue{};
         if (!ReadDataStream(ssKey, ssValue)) {
             return false;
         }

src/evo/deterministicmns.cpp

@@ -1068,7 +1068,7 @@ bool CDeterministicMNManager::MigrateLegacyDiffs(const CBlockIndex* const tip_in
         }
 
         // Use legacy-aware deserialization for DB_LIST_DIFF_LEGACY entries
-        CDataStream s(SER_DISK, CLIENT_VERSION);
+        DataStream s{};
         if (!m_evoDb.GetRawDB().ReadDataStream(key, s)) {
             break;
         }

src/evo/deterministicmns.h

@@ -73,14 +73,7 @@ class CDeterministicMN
         READWRITE(obj.collateralOutpoint);
         READWRITE(obj.nOperatorReward);
         READWRITE(obj.pdmnState);
-        // We can't know if we are serialising for the Disk or for the Network here (s.GetType() is not accessible)
-        // Therefore if s.GetVersion() == CLIENT_VERSION -> Then we know we are serialising for the Disk
-        // Otherwise, we can safely check with protocol versioning logic so we won't break old clients
-        if (s.GetVersion() == CLIENT_VERSION || s.GetVersion() >= DMN_TYPE_PROTO_VERSION) {
-            READWRITE(obj.nType);
-        } else {
-            SER_READ(obj, obj.nType = MnType::Regular);
-        }
+        READWRITE(obj.nType);
     }
 
     [[nodiscard]] bool HasInternalId() const { return internalId != std::numeric_limits<uint64_t>::max(); }

src/index/blockfilterindex.cpp

@@ -4,6 +4,7 @@
 
 #include <map>
 
+#include <clientversion.h>
 #include <common/args.h>
 #include <dbwrapper.h>
 #include <hash.h>

src/index/txindex.cpp

@@ -4,6 +4,7 @@
 
 #include <index/txindex.h>
 
+#include <clientversion.h>
 #include <common/args.h>
 #include <index/disktxpos.h>
 #include <logging.h>
@@ -83,7 +84,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe
         return false;
     }
 
-    AutoFile file{m_chainstate->m_blockman.OpenBlockFile(RecordHeaderPos(postx), true)};
+    CAutoFile file{m_chainstate->m_blockman.OpenBlockFile(RecordHeaderPos(postx), true)};
     if (file.IsNull()) {
         LogError("%s: OpenBlockFile failed\n", __func__);
         return false;

src/init.cpp

@@ -21,6 +21,7 @@
 #include <blockfilter.h>
 #include <chain.h>
 #include <chainparams.h>
+#include <clientversion.h>
 #include <common/args.h>
 #include <common/system.h>
 #include <context.h>

src/llmq/dkgsessionmgr.cpp

@@ -159,7 +159,7 @@ bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType,
             ContributionsCacheKey cacheKey = {llmqType, pQuorumBaseBlockIndex->GetBlockHash(), proTxHash};
             auto it = contributionsCache.find(cacheKey);
             if (it == contributionsCache.end()) {
-                CDataStream s(SER_DISK, CLIENT_VERSION);
+                DataStream s{};
                 if (!db->ReadDataStream(std::make_tuple(DB_VVEC, llmqType, pQuorumBaseBlockIndex->GetBlockHash(), proTxHash), s)) {
                     LogPrint(BCLog::LLMQ, "%s -- this node does not have vvec for llmq=%d block=%s protx=%s\n",
                              __func__, std23::to_underlying(llmqType), pQuorumBaseBlockIndex->GetBlockHash().ToString(),

src/llmq/quorums.cpp

@@ -212,7 +212,7 @@ void CQuorum::WriteContributions(CDBWrapper& db) const
 bool CQuorum::ReadContributions(const CDBWrapper& db)
 {
     uint256 dbKey = MakeQuorumKey(*this);
-    CDataStream s(SER_DISK, CLIENT_VERSION);
+    DataStream s{};
 
     if (!db.ReadDataStream(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s)) {
         return false;

src/llmq/quorumsman.cpp

@@ -643,7 +643,7 @@ void CQuorumManager::MigrateOldQuorumDB(CEvoDB& evoDb) const
         int count{0};
         while (pcursor->Valid()) {
             decltype(start) k;
-            CDataStream s(SER_DISK, CLIENT_VERSION);
+            DataStream s{};
             CBLSSecretKey sk;
 
             if (!pcursor->GetKey(k) || std::get<0>(k) != prefix) {

src/llmq/signing.cpp

@@ -96,7 +96,7 @@ bool CRecoveredSigsDb::ReadRecoveredSig(Consensus::LLMQType llmqType, const uint
 {
     auto k = std::make_tuple(std::string("rs_r"), llmqType, id);
 
-    CDataStream ds(SER_DISK, CLIENT_VERSION);
+    DataStream ds{};
     if (!db->ReadDataStream(k, ds)) {
         return false;
     }
@@ -239,7 +239,7 @@ struct RecoveredSigMarker {
 
 std::optional<RecoveredSigMarker> ReadRecoveredSigMarker(const CDBWrapper& db, const std::string& prefix, const uint256& hash)
 {
-    CDataStream value{SER_DISK, CLIENT_VERSION};
+    DataStream value{};
     if (!db.ReadDataStream(std::make_pair(prefix, hash), value)) return std::nullopt;
     RecoveredSigMarker marker;
     try {

src/node/blockstorage.cpp

@@ -390,7 +390,7 @@ bool BlockManager::LoadBlockIndexDB()
     }
     for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
         FlatFilePos pos(*it, 0);
-        if (AutoFile{OpenBlockFile(pos, true)}.IsNull()) {
+        if (OpenBlockFile(pos, true).IsNull()) {
             return false;
         }
     }
@@ -509,7 +509,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
 bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, const std::optional<std::vector<std::byte>>& compressed, uint32_t size_field, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const
 {
     // Open history file to append
-    AutoFile fileout{OpenUndoFile(pos)};
+    CAutoFile fileout{OpenUndoFile(pos)};
     if (fileout.IsNull()) {
         LogError("%s: OpenUndoFile failed\n", __func__);
         return false;
@@ -550,7 +550,7 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in
     }
 
     // Open history file to read
-    AutoFile filein{OpenUndoFile(RecordHeaderPos(pos), true)};
+    CAutoFile filein{OpenUndoFile(RecordHeaderPos(pos), true)};
     if (filein.IsNull()) {
         LogError("%s: OpenUndoFile failed\n", __func__);
         return false;
@@ -662,15 +662,15 @@ FlatFileSeq BlockManager::UndoFileSeq() const
     return FlatFileSeq(m_opts.blocks_dir, "rev", m_opts.tiny_blk ? 0x10000 /* 64kb */ : UNDOFILE_CHUNK_SIZE);
 }
 
-FILE* BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
+CAutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
 {
-    return BlockFileSeq().Open(pos, fReadOnly);
+    return CAutoFile{BlockFileSeq().Open(pos, fReadOnly), CLIENT_VERSION};
 }
 
 /** Open an undo file (rev?????.dat) */
-FILE* BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
+CAutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
 {
-    return UndoFileSeq().Open(pos, fReadOnly);
+    return CAutoFile{UndoFileSeq().Open(pos, fReadOnly), CLIENT_VERSION};
 }
 
 fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
@@ -782,7 +782,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP
 bool BlockManager::WriteBlockToDisk(Span<const std::byte> payload, uint32_t size_field, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const
 {
     // Open history file to append
-    AutoFile fileout{OpenBlockFile(pos)};
+    CAutoFile fileout{OpenBlockFile(pos)};
     if (fileout.IsNull()) {
         LogError("WriteBlockToDisk: OpenBlockFile failed\n");
         return false;
@@ -904,7 +904,7 @@ std::optional<uint256> BlockManager::ReadBlockFromDisk(CBlock& block, const Flat
     block.SetNull();
 
     // Open history file to read
-    AutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)};
+    CAutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)};
     if (filein.IsNull()) {
         LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString());
         return std::nullopt;
@@ -1027,8 +1027,8 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile
                 if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) {
                     break; // No block files left to reindex
                 }
-                FILE* file = chainman.m_blockman.OpenBlockFile(pos, true);
-                if (!file) {
+                CAutoFile file = chainman.m_blockman.OpenBlockFile(pos, true);
+                if (file.IsNull()) {
                     break; // This error is logged in OpenBlockFile
                 }
                 LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
@@ -1048,8 +1048,8 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile
 
         // -loadblock=
         for (const fs::path& path : vImportFiles) {
-            FILE *file = fsbridge::fopen(path, "rb");
-            if (file) {
+            CAutoFile file{fsbridge::fopen(path, "rb"), CLIENT_VERSION};
+            if (!file.IsNull()) {
                 LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
                 chainman.LoadExternalBlockFile(file);
                 if (ShutdownRequested()) {